1 /* $NetBSD: locore.s,v 1.439 2026/03/07 20:48:26 palle Exp $ */ 2 3 /* 4 * Copyright (c) 2006-2010 Matthew R. Green 5 * Copyright (c) 1996-2002 Eduardo Horvath 6 * Copyright (c) 1996 Paul Kranenburg 7 * Copyright (c) 1996 8 * The President and Fellows of Harvard College. 9 * All rights reserved. 10 * Copyright (c) 1992, 1993 11 * The Regents of the University of California. 12 * All rights reserved. 13 * 14 * This software was developed by the Computer Systems Engineering group 15 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 16 * contributed to Berkeley. 17 * 18 * All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the University of 21 * California, Lawrence Berkeley Laboratory. 22 * This product includes software developed by Harvard University. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 1. Redistributions of source code must retain the above copyright 28 * notice, this list of conditions and the following disclaimer. 29 * 2. Redistributions in binary form must reproduce the above copyright 30 * notice, this list of conditions and the following disclaimer in the 31 * documentation and/or other materials provided with the 32 * distribution. 33 * 3. All advertising materials mentioning features or use of this 34 * software must display the following acknowledgement: 35 * This product includes software developed by the University of 36 * California, Berkeley and its contributors. 37 * This product includes software developed by Harvard University. 38 * This product includes software developed by Paul Kranenburg. 39 * 4. Neither the name of the University nor the names of its 40 * contributors may be used to endorse or promote products derived 41 * from this software without specific prior written permission. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' 44 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 45 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 46 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 47 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 48 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 49 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 50 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 51 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 52 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 53 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 54 * DAMAGE. 55 * 56 * @(#)locore.s 8.4 (Berkeley) 12/10/93 57 */ 58 59 #undef PARANOID /* Extremely expensive consistency checks */ 60 #undef NO_VCACHE /* Map w/D$ disabled */ 61 #undef TRAPSTATS /* Count traps */ 62 #undef TRAPS_USE_IG /* Use Interrupt Globals for all traps */ 63 #define HWREF /* Track ref/mod bits in trap handlers */ 64 #undef DCACHE_BUG /* Flush D$ around ASI_PHYS accesses */ 65 #undef NO_TSB /* Don't use TSB */ 66 #define BB_ERRATA_1 /* writes to TICK_CMPR may fail */ 67 #undef TLB_FLUSH_LOWVA /* also flush 32-bit entries from the MMU */ 68 69 #include "opt_ddb.h" 70 #include "opt_kgdb.h" 71 #include "opt_multiprocessor.h" 72 #include "opt_compat_netbsd.h" 73 #include "opt_compat_netbsd32.h" 74 #include "opt_lockdebug.h" 75 76 #include "assym.h" 77 #include <machine/param.h> 78 #include <machine/types.h> 79 #include <sparc64/sparc64/intreg.h> 80 #include <sparc64/sparc64/timerreg.h> 81 #include <machine/ctlreg.h> 82 #include <machine/psl.h> 83 #include <machine/signal.h> 84 #include <machine/trap.h> 85 #include <machine/frame.h> 86 #include <machine/pmap.h> 87 #include <machine/intr.h> 88 #include <machine/asm.h> 89 #include <machine/locore.h> 90 #ifdef SUN4V 91 #include <machine/hypervisor.h> 92 #endif 93 #include <sys/syscall.h> 94 95 #define BLOCK_SIZE SPARC64_BLOCK_SIZE 96 #define BLOCK_ALIGN SPARC64_BLOCK_ALIGN 97 98 #ifdef SUN4V 99 #define SUN4V_N_REG_WINDOWS 8 /* As per UA2005 spec */ 100 #define SUN4V_NWINDOWS (SUN4V_N_REG_WINDOWS-1) /* This is an index number, so subtract one */ 101 #endif 102 103 #include "ksyms.h" 104 105 /* Misc. macros */ 106 107 .macro GET_MAXCWP reg 108 #ifdef SUN4V 109 sethi %hi(cputyp), \reg 110 ld [\reg + %lo(cputyp)], \reg 111 cmp \reg, CPU_SUN4V 112 bne,pt %icc, 2f 113 nop 114 /* sun4v */ 115 ba 3f 116 mov SUN4V_NWINDOWS, \reg 117 2: 118 #endif 119 /* sun4u */ 120 rdpr %ver, \reg 121 and \reg, CWP, \reg 122 3: 123 .endm 124 125 .macro SET_MMU_CONTEXTID_SUN4U ctxid,ctx 126 stxa \ctxid, [\ctx] ASI_DMMU; 127 .endm 128 129 #ifdef SUN4V 130 .macro SET_MMU_CONTEXTID_SUN4V ctxid,ctx 131 stxa \ctxid, [\ctx] ASI_MMU_CONTEXTID; 132 .endm 133 #endif 134 135 .macro SET_MMU_CONTEXTID ctxid,ctx,scratch 136 #ifdef SUN4V 137 sethi %hi(cputyp), \scratch 138 ld [\scratch + %lo(cputyp)], \scratch 139 cmp \scratch, CPU_SUN4V 140 bne,pt %icc, 2f 141 nop 142 /* sun4v */ 143 SET_MMU_CONTEXTID_SUN4V \ctxid,\ctx 144 ba 3f 145 nop 146 2: 147 #endif 148 /* sun4u */ 149 SET_MMU_CONTEXTID_SUN4U \ctxid,\ctx 150 3: 151 .endm 152 153 .macro GET_MMU_CONTEXTID_SUN4U ctxid,ctx 154 ldxa [\ctx] ASI_DMMU, \ctxid 155 .endm 156 157 #ifdef SUN4V 158 .macro GET_MMU_CONTEXTID_SUN4V ctxid,ctx 159 ldxa [\ctx] ASI_MMU_CONTEXTID, \ctxid 160 .endm 161 #endif 162 163 .macro GET_MMU_CONTEXTID ctxid,ctx,scratch 164 #ifdef SUN4V 165 sethi %hi(cputyp), \scratch 166 ld [\scratch + %lo(cputyp)], \scratch 167 cmp \scratch, CPU_SUN4V 168 bne,pt %icc, 2f 169 nop 170 /* sun4v */ 171 GET_MMU_CONTEXTID_SUN4V \ctxid,\ctx 172 ba 3f 173 nop 174 2: 175 #endif 176 /* sun4u */ 177 GET_MMU_CONTEXTID_SUN4U \ctxid,\ctx 178 3: 179 .endm 180 181 #ifdef SUN4V 182 .macro NORMAL_GLOBALS_SUN4V 183 wrpr %g0, 0, %gl ! Set globals to level 0 184 .endm 185 #endif 186 .macro NORMAL_GLOBALS_SUN4U 187 wrpr %g0, PSTATE_KERN, %pstate ! Alternate Globals (AG) bit set to zero 188 .endm 189 190 #ifdef SUN4V 191 .macro ALTERNATE_GLOBALS_SUN4V 192 wrpr %g0, 1, %gl ! Set globals to level 1 193 .endm 194 #endif 195 .macro ALTERNATE_GLOBALS_SUN4U 196 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Alternate Globals (AG) bit set to one 197 .endm 198 199 .macro ENABLE_INTERRUPTS scratch 200 rdpr %pstate, \scratch 201 or \scratch, PSTATE_IE, \scratch ! Interrupt Enable (IE) bit set to one 202 wrpr %g0, \scratch, %pstate 203 .endm 204 205 .macro DISABLE_INTERRUPTS scratch 206 rdpr %pstate, \scratch 207 and \scratch, ~PSTATE_IE, \scratch ! Interrupt Enable (IE) bit set to zero 208 wrpr %g0, \scratch, %pstate 209 .endm 210 211 212 #ifdef SUN4V 213 /* Misc. sun4v macros */ 214 215 .macro GET_MMFSA reg 216 sethi %hi(CPUINFO_VA + CI_MMUFSA), \reg 217 LDPTR [\reg + %lo(CPUINFO_VA + CI_MMUFSA)], \reg 218 .endm 219 220 .macro GET_CTXBUSY reg 221 sethi %hi(CPUINFO_VA + CI_CTXBUSY), \reg 222 LDPTR [\reg + %lo(CPUINFO_VA + CI_CTXBUSY)], \reg 223 .endm 224 225 .macro GET_TSB_DMMU reg 226 sethi %hi(CPUINFO_VA + CI_TSB_DMMU), \reg 227 LDPTR [\reg + %lo(CPUINFO_VA + CI_TSB_DMMU)], \reg 228 .endm 229 230 .macro sun4v_tl1_uspill_normal 231 ba,a,pt %xcc, spill_normal_to_user_stack 232 nop 233 .align 128 234 .endm 235 236 .macro sun4v_tl1_uspill_other 237 ba,a,pt %xcc, pcbspill_other 238 nop 239 .align 128 240 .endm 241 242 #endif 243 244 #if 1 245 /* 246 * Try to issue an elf note to ask the Solaris 247 * bootloader to align the kernel properly. 248 */ 249 .section .note 250 .word 0x0d 251 .word 4 ! Dunno why 252 .word 1 253 0: .asciz "SUNW Solaris" 254 1: 255 .align 4 256 .word 0x0400000 257 #endif 258 259 .register %g2,#scratch 260 .register %g3,#scratch 261 262 263 .data 264 .globl _C_LABEL(data_start) 265 _C_LABEL(data_start): ! Start of data segment 266 267 #ifdef KGDB 268 /* 269 * Another item that must be aligned, easiest to put it here. 270 */ 271 KGDB_STACK_SIZE = 2048 272 .globl _C_LABEL(kgdb_stack) 273 _C_LABEL(kgdb_stack): 274 .space KGDB_STACK_SIZE ! hope this is enough 275 #endif 276 277 #ifdef NOTDEF_DEBUG 278 /* 279 * This stack is used when we detect kernel stack corruption. 280 */ 281 .space USPACE 282 .align 16 283 panicstack: 284 #endif 285 286 /* 287 * romp is the prom entry pointer 288 * romtba is the prom trap table base address 289 */ 290 .globl romp 291 romp: POINTER 0 292 .globl romtba 293 romtba: POINTER 0 294 295 .globl cputyp 296 cputyp: .word CPU_SUN4U ! Default to sun4u 297 298 _ALIGN 299 .text 300 301 /* 302 * The v9 trap frame is stored in the special trap registers. The 303 * register window is only modified on window overflow, underflow, 304 * and clean window traps, where it points to the register window 305 * needing service. Traps have space for 8 instructions, except for 306 * the window overflow, underflow, and clean window traps which are 307 * 32 instructions long, large enough to in-line. 308 * 309 * The spitfire CPU (Ultra I) has 4 different sets of global registers. 310 * (blah blah...) 311 * 312 * I used to generate these numbers by address arithmetic, but gas's 313 * expression evaluator has about as much sense as your average slug 314 * (oddly enough, the code looks about as slimy too). Thus, all the 315 * trap numbers are given as arguments to the trap macros. This means 316 * there is one line per trap. Sigh. 317 * 318 * Hardware interrupt vectors can be `linked'---the linkage is to regular 319 * C code---or rewired to fast in-window handlers. The latter are good 320 * for unbuffered hardware like the Zilog serial chip and the AMD audio 321 * chip, where many interrupts can be handled trivially with pseudo-DMA 322 * or similar. Only one `fast' interrupt can be used per level, however, 323 * and direct and `fast' interrupts are incompatible. Routines in intr.c 324 * handle setting these, with optional paranoia. 325 */ 326 327 /* 328 * TA8 -- trap align for 8 instruction traps 329 * TA32 -- trap align for 32 instruction traps 330 */ 331 #define TA8 .align 32 332 #define TA32 .align 128 333 334 /* 335 * v9 trap macros: 336 * 337 * We have a problem with v9 traps; we have no registers to put the 338 * trap type into. But we do have a %tt register which already has 339 * that information. Trap types in these macros are all dummys. 340 */ 341 /* regular vectored traps */ 342 343 #define VTRAP(type, label) \ 344 ba,a,pt %icc,label; nop; NOTREACHED; TA8 345 346 /* hardware interrupts (can be linked or made `fast') */ 347 #define HARDINT4U(lev) \ 348 VTRAP(lev, _C_LABEL(sparc_interrupt)) 349 #ifdef SUN4V 350 #define HARDINT4V(lev) HARDINT4U(lev) 351 #endif 352 353 /* software interrupts (may not be made direct, sorry---but you 354 should not be using them trivially anyway) */ 355 #define SOFTINT4U(lev, bit) \ 356 HARDINT4U(lev) 357 358 /* traps that just call trap() */ 359 #define TRAP(type) VTRAP(type, slowtrap) 360 361 /* architecturally undefined traps (cause panic) */ 362 #ifndef DEBUG 363 #define UTRAP(type) sir; VTRAP(type, slowtrap) 364 #else 365 #define UTRAP(type) VTRAP(type, slowtrap) 366 #endif 367 368 /* software undefined traps (may be replaced) */ 369 #define STRAP(type) VTRAP(type, slowtrap) 370 371 /* breakpoint acts differently under kgdb */ 372 #ifdef KGDB 373 #define BPT VTRAP(T_BREAKPOINT, bpt) 374 #define BPT_KGDB_EXEC VTRAP(T_KGDB_EXEC, bpt) 375 #else 376 #define BPT TRAP(T_BREAKPOINT) 377 #define BPT_KGDB_EXEC TRAP(T_KGDB_EXEC) 378 #endif 379 380 #define SYSCALL VTRAP(0x100, syscall_setup) 381 #ifdef notyet 382 #define ZS_INTERRUPT ba,a,pt %icc, zshard; nop; TA8 383 #else 384 #define ZS_INTERRUPT4U HARDINT4U(12) 385 #endif 386 387 388 /* 389 * Macro to clear %tt so we don't get confused with old traps. 390 */ 391 #ifdef DEBUG 392 #define CLRTT wrpr %g0,0x1ff,%tt 393 #else 394 #define CLRTT 395 #endif 396 397 398 /* 399 * Some macros to load and store a register window 400 */ 401 402 .macro SPILL storer,base,size,asi 403 404 .irpc n,01234567 405 \storer %l\n, [\base + (\n * \size)] \asi 406 .endr 407 .irpc n,01234567 408 \storer %i\n, [\base + ((8+\n) * \size)] \asi 409 .endr 410 411 .endm 412 413 414 .macro FILL loader, base, size, asi 415 416 .irpc n,01234567 417 \loader [\base + (\n * \size)] \asi, %l\n 418 .endr 419 420 .irpc n,01234567 421 \loader [\base + ((8+\n) * \size)] \asi, %i\n 422 .endr 423 424 .endm 425 426 /* 427 * Here are some oft repeated traps as macros. 428 */ 429 430 /* spill a 64-bit register window */ 431 #define SPILL64(label,as) \ 432 label: \ 433 wr %g0, as, %asi; \ 434 stxa %l0, [%sp+BIAS+0x00]%asi; \ 435 stxa %l1, [%sp+BIAS+0x08]%asi; \ 436 stxa %l2, [%sp+BIAS+0x10]%asi; \ 437 stxa %l3, [%sp+BIAS+0x18]%asi; \ 438 stxa %l4, [%sp+BIAS+0x20]%asi; \ 439 stxa %l5, [%sp+BIAS+0x28]%asi; \ 440 stxa %l6, [%sp+BIAS+0x30]%asi; \ 441 \ 442 stxa %l7, [%sp+BIAS+0x38]%asi; \ 443 stxa %i0, [%sp+BIAS+0x40]%asi; \ 444 stxa %i1, [%sp+BIAS+0x48]%asi; \ 445 stxa %i2, [%sp+BIAS+0x50]%asi; \ 446 stxa %i3, [%sp+BIAS+0x58]%asi; \ 447 stxa %i4, [%sp+BIAS+0x60]%asi; \ 448 stxa %i5, [%sp+BIAS+0x68]%asi; \ 449 stxa %i6, [%sp+BIAS+0x70]%asi; \ 450 \ 451 stxa %i7, [%sp+BIAS+0x78]%asi; \ 452 saved; \ 453 CLRTT; \ 454 retry; \ 455 NOTREACHED; \ 456 TA32 457 458 /* spill a 32-bit register window */ 459 #define SPILL32(label,as) \ 460 label: \ 461 wr %g0, as, %asi; \ 462 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 463 stwa %l0, [%sp+0x00]%asi; \ 464 stwa %l1, [%sp+0x04]%asi; \ 465 stwa %l2, [%sp+0x08]%asi; \ 466 stwa %l3, [%sp+0x0c]%asi; \ 467 stwa %l4, [%sp+0x10]%asi; \ 468 stwa %l5, [%sp+0x14]%asi; \ 469 \ 470 stwa %l6, [%sp+0x18]%asi; \ 471 stwa %l7, [%sp+0x1c]%asi; \ 472 stwa %i0, [%sp+0x20]%asi; \ 473 stwa %i1, [%sp+0x24]%asi; \ 474 stwa %i2, [%sp+0x28]%asi; \ 475 stwa %i3, [%sp+0x2c]%asi; \ 476 stwa %i4, [%sp+0x30]%asi; \ 477 stwa %i5, [%sp+0x34]%asi; \ 478 \ 479 stwa %i6, [%sp+0x38]%asi; \ 480 stwa %i7, [%sp+0x3c]%asi; \ 481 saved; \ 482 CLRTT; \ 483 retry; \ 484 NOTREACHED; \ 485 TA32 486 487 /* Spill either 32-bit or 64-bit register window. */ 488 #define SPILLBOTH(label64,label32,as) \ 489 andcc %sp, 1, %g0; \ 490 bnz,pt %xcc, label64+4; /* Is it a v9 or v8 stack? */ \ 491 wr %g0, as, %asi; \ 492 ba,pt %xcc, label32+8; \ 493 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 494 NOTREACHED; \ 495 TA32 496 497 /* fill a 64-bit register window */ 498 #define FILL64(label,as) \ 499 label: \ 500 wr %g0, as, %asi; \ 501 ldxa [%sp+BIAS+0x00]%asi, %l0; \ 502 ldxa [%sp+BIAS+0x08]%asi, %l1; \ 503 ldxa [%sp+BIAS+0x10]%asi, %l2; \ 504 ldxa [%sp+BIAS+0x18]%asi, %l3; \ 505 ldxa [%sp+BIAS+0x20]%asi, %l4; \ 506 ldxa [%sp+BIAS+0x28]%asi, %l5; \ 507 ldxa [%sp+BIAS+0x30]%asi, %l6; \ 508 \ 509 ldxa [%sp+BIAS+0x38]%asi, %l7; \ 510 ldxa [%sp+BIAS+0x40]%asi, %i0; \ 511 ldxa [%sp+BIAS+0x48]%asi, %i1; \ 512 ldxa [%sp+BIAS+0x50]%asi, %i2; \ 513 ldxa [%sp+BIAS+0x58]%asi, %i3; \ 514 ldxa [%sp+BIAS+0x60]%asi, %i4; \ 515 ldxa [%sp+BIAS+0x68]%asi, %i5; \ 516 ldxa [%sp+BIAS+0x70]%asi, %i6; \ 517 \ 518 ldxa [%sp+BIAS+0x78]%asi, %i7; \ 519 restored; \ 520 CLRTT; \ 521 retry; \ 522 NOTREACHED; \ 523 TA32 524 525 /* fill a 32-bit register window */ 526 #define FILL32(label,as) \ 527 label: \ 528 wr %g0, as, %asi; \ 529 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 530 lda [%sp+0x00]%asi, %l0; \ 531 lda [%sp+0x04]%asi, %l1; \ 532 lda [%sp+0x08]%asi, %l2; \ 533 lda [%sp+0x0c]%asi, %l3; \ 534 lda [%sp+0x10]%asi, %l4; \ 535 lda [%sp+0x14]%asi, %l5; \ 536 \ 537 lda [%sp+0x18]%asi, %l6; \ 538 lda [%sp+0x1c]%asi, %l7; \ 539 lda [%sp+0x20]%asi, %i0; \ 540 lda [%sp+0x24]%asi, %i1; \ 541 lda [%sp+0x28]%asi, %i2; \ 542 lda [%sp+0x2c]%asi, %i3; \ 543 lda [%sp+0x30]%asi, %i4; \ 544 lda [%sp+0x34]%asi, %i5; \ 545 \ 546 lda [%sp+0x38]%asi, %i6; \ 547 lda [%sp+0x3c]%asi, %i7; \ 548 restored; \ 549 CLRTT; \ 550 retry; \ 551 NOTREACHED; \ 552 TA32 553 554 /* fill either 32-bit or 64-bit register window. */ 555 #define FILLBOTH(label64,label32,as) \ 556 andcc %sp, 1, %i0; \ 557 bnz (label64)+4; /* See if it's a v9 stack or v8 */ \ 558 wr %g0, as, %asi; \ 559 ba (label32)+8; \ 560 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 561 NOTREACHED; \ 562 TA32 563 564 /* handle clean window trap when trap level = 0 */ 565 .macro CLEANWIN0 566 rdpr %cleanwin, %o7 567 inc %o7 ! This handler is in-lined and cannot fault 568 #ifdef DEBUG 569 set 0xbadcafe, %l0 ! DEBUG -- compiler should not rely on zero-ed registers. 570 #else 571 clr %l0 572 #endif 573 wrpr %g0, %o7, %cleanwin ! Nucleus (trap&IRQ) code does not need clean windows 574 575 mov %l0,%l1; mov %l0,%l2 ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done 576 mov %l0,%l3; mov %l0,%l4 577 #if 0 578 #ifdef DIAGNOSTIC 579 !! 580 !! Check the sp redzone 581 !! 582 !! Since we can't spill the current window, we'll just keep 583 !! track of the frame pointer. Problems occur when the routine 584 !! allocates and uses stack storage. 585 !! 586 ! rdpr %wstate, %l5 ! User stack? 587 ! cmp %l5, WSTATE_KERN 588 ! bne,pt %icc, 7f 589 sethi %hi(CPCB), %l5 590 LDPTR [%l5 + %lo(CPCB)], %l5 ! If pcb < fp < pcb+sizeof(pcb) 591 inc PCB_SIZE, %l5 ! then we have a stack overflow 592 btst %fp, 1 ! 64-bit stack? 593 sub %fp, %l5, %l7 594 bnz,a,pt %icc, 1f 595 inc BIAS, %l7 ! Remove BIAS 596 1: 597 cmp %l7, PCB_SIZE 598 blu %xcc, cleanwin_overflow 599 #endif 600 #endif 601 mov %l0, %l5 602 mov %l0, %l6; mov %l0, %l7; mov %l0, %o0; mov %l0, %o1 603 604 mov %l0, %o2; mov %l0, %o3; mov %l0, %o4; mov %l0, %o5; 605 mov %l0, %o6; mov %l0, %o7 606 CLRTT 607 retry; nop; NOTREACHED; TA32 608 .endm 609 610 /* handle clean window trap when trap level = 1 */ 611 .macro CLEANWIN1 612 clr %l0 613 #ifdef DEBUG 614 set 0xbadbeef, %l0 ! DEBUG 615 #endif 616 mov %l0, %l1; mov %l0, %l2 617 rdpr %cleanwin, %o7 ! This handler is in-lined and cannot fault 618 inc %o7; mov %l0, %l3 ! Nucleus (trap&IRQ) code does not need clean windows 619 wrpr %g0, %o7, %cleanwin ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done 620 #ifdef NOT_DEBUG 621 !! 622 !! Check the sp redzone 623 !! 624 rdpr %wstate, t1 625 cmp t1, WSTATE_KERN 626 bne,pt icc, 7f 627 sethi %hi(_C_LABEL(redzone)), t1 628 ldx [t1 + %lo(_C_LABEL(redzone))], t2 629 cmp %sp, t2 ! if sp >= t2, not in red zone 630 blu panic_red ! and can continue normally 631 7: 632 #endif 633 mov %l0, %l4; mov %l0, %l5; mov %l0, %l6; mov %l0, %l7 634 mov %l0, %o0; mov %l0, %o1; mov %l0, %o2; mov %l0, %o3 635 636 mov %l0, %o4; mov %l0, %o5; mov %l0, %o6; mov %l0, %o7 637 CLRTT 638 retry; nop; TA32 639 .endm 640 641 .globl start, _C_LABEL(kernel_text) 642 _C_LABEL(kernel_text) = kernel_start ! for kvm_mkdb(8) 643 kernel_start: 644 /* Traps from TL=0 -- traps from user mode */ 645 #ifdef __STDC__ 646 #define TABLE(name) user_ ## name 647 #else 648 #define TABLE(name) user_/**/name 649 #endif 650 .globl _C_LABEL(trapbase) 651 _C_LABEL(trapbase): 652 b dostart; nop; TA8 ! 000 = reserved -- Use it to boot 653 /* We should not get the next 5 traps */ 654 UTRAP(0x001) ! 001 = POR Reset -- ROM should get this 655 UTRAP(0x002) ! 002 = WDR -- ROM should get this 656 UTRAP(0x003) ! 003 = XIR -- ROM should get this 657 UTRAP(0x004) ! 004 = SIR -- ROM should get this 658 UTRAP(0x005) ! 005 = RED state exception 659 UTRAP(0x006); UTRAP(0x007) 660 VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except 661 VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss 662 VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err 663 UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) 664 TRAP(T_ILLINST) ! 010 = illegal instruction 665 TRAP(T_PRIVINST) ! 011 = privileged instruction 666 UTRAP(0x012) ! 012 = unimplemented LDD 667 UTRAP(0x013) ! 013 = unimplemented STD 668 UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) 669 UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) 670 UTRAP(0x01e); UTRAP(0x01f) 671 TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr 672 TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception 673 TRAP(T_FP_OTHER) ! 022 = other fp exception 674 TRAP(T_TAGOF) ! 023 = tag overflow 675 CLEANWIN0 ! 024-027 = clean window trap 676 TRAP(T_DIV0) ! 028 = divide by zero 677 UTRAP(0x029) ! 029 = internal processor error 678 UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) 679 VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault 680 UTRAP(0x031) ! 031 = data MMU miss -- no MMU 681 VTRAP(T_DATA_ERROR, winfault) ! 032 = data access error 682 VTRAP(T_DATA_PROT, winfault) ! 033 = data protection fault 683 TRAP(T_ALIGN) ! 034 = address alignment error -- we could fix it inline... 684 TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... 685 TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... 686 TRAP(T_PRIVACT) ! 037 = privileged action 687 UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); 688 UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); 689 VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault 690 SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt 691 HARDINT4U(2) ! 042 = level 2 interrupt 692 HARDINT4U(3) ! 043 = level 3 interrupt 693 SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt 694 HARDINT4U(5) ! 045 = level 5 interrupt 695 SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt 696 HARDINT4U(7) ! 047 = level 7 interrupt 697 HARDINT4U(8) ! 048 = level 8 interrupt 698 HARDINT4U(9) ! 049 = level 9 interrupt 699 HARDINT4U(10) ! 04a = level 10 interrupt 700 HARDINT4U(11) ! 04b = level 11 interrupt 701 ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt 702 HARDINT4U(13) ! 04d = level 13 interrupt 703 HARDINT4U(14) ! 04e = level 14 interrupt 704 HARDINT4U(15) ! 04f = nonmaskable interrupt 705 UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) 706 UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) 707 UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) 708 VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector 709 TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint 710 TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint 711 TRAP(T_ECCERR) ! 063 = corrected ECC error 712 ufast_IMMU_miss: ! 064 = fast instr access MMU miss 713 ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer 714 #ifdef NO_TSB 715 ba,a %icc, instr_miss 716 #endif 717 ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register 718 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 719 brgez,pn %g5, instr_miss ! Entry invalid? Punt 720 cmp %g1, %g4 ! Compare TLB tags 721 bne,pn %xcc, instr_miss ! Got right tag? 722 nop 723 CLRTT 724 stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 725 retry ! Try new mapping 726 1: 727 sir 728 TA32 729 ufast_DMMU_miss: ! 068 = fast data access MMU miss 730 ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer 731 #ifdef NO_TSB 732 ba,a %icc, data_miss 733 #endif 734 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 735 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 736 brgez,pn %g5, data_miss ! Entry invalid? Punt 737 cmp %g1, %g4 ! Compare TLB tags 738 bnz,pn %xcc, data_miss ! Got right tag? 739 nop 740 CLRTT 741 #ifdef TRAPSTATS 742 sethi %hi(_C_LABEL(udhit)), %g1 743 lduw [%g1+%lo(_C_LABEL(udhit))], %g2 744 inc %g2 745 stw %g2, [%g1+%lo(_C_LABEL(udhit))] 746 #endif 747 stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 748 retry ! Try new mapping 749 1: 750 sir 751 TA32 752 ufast_DMMU_protection: ! 06c = fast data access MMU protection 753 #ifdef TRAPSTATS 754 sethi %hi(_C_LABEL(udprot)), %g1 755 lduw [%g1+%lo(_C_LABEL(udprot))], %g2 756 inc %g2 757 stw %g2, [%g1+%lo(_C_LABEL(udprot))] 758 #endif 759 #ifdef HWREF 760 ba,a,pt %xcc, dmmu_write_fault 761 #else 762 ba,a,pt %xcc, winfault 763 #endif 764 nop 765 TA32 766 TRAP(0x070) ! 0x070 fast_ECC_error 767 ! Implementation dependent traps 768 UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) 769 UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) 770 UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) 771 TABLE(uspill): 772 SPILL64(uspill8,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode 773 SPILL32(uspill4,ASI_AIUS) ! 0x084 spill_1_normal 774 SPILLBOTH(uspill8,uspill4,ASI_AIUS) ! 0x088 spill_2_normal 775 UTRAP(0x08c); TA32 ! 0x08c spill_3_normal 776 TABLE(kspill): 777 SPILL64(kspill8,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 778 SPILL32(kspill4,ASI_N) ! 0x094 spill_5_normal 779 SPILLBOTH(kspill8,kspill4,ASI_N) ! 0x098 spill_6_normal 780 UTRAP(0x09c); TA32 ! 0x09c spill_7_normal 781 TABLE(uspillk): 782 SPILL64(uspillk8,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode 783 SPILL32(uspillk4,ASI_AIUS) ! 0x0a4 spill_1_other 784 SPILLBOTH(uspillk8,uspillk4,ASI_AIUS) ! 0x0a8 spill_2_other 785 UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other 786 UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other 787 UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other 788 UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other 789 UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other 790 TABLE(ufill): 791 FILL64(ufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode 792 FILL32(ufill4,ASI_AIUS) ! 0x0c4 fill_1_normal 793 FILLBOTH(ufill8,ufill4,ASI_AIUS) ! 0x0c8 fill_2_normal 794 UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal 795 TABLE(kfill): 796 FILL64(kfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode 797 FILL32(kfill4,ASI_N) ! 0x0d4 fill_5_normal 798 FILLBOTH(kfill8,kfill4,ASI_N) ! 0x0d8 fill_6_normal 799 UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal 800 TABLE(ufillk): 801 FILL64(ufillk8,ASI_AIUS) ! 0x0e0 fill_0_other 802 FILL32(ufillk4,ASI_AIUS) ! 0x0e4 fill_1_other 803 FILLBOTH(ufillk8,ufillk4,ASI_AIUS) ! 0x0e8 fill_2_other 804 UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other 805 UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other 806 UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other 807 UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other 808 UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other 809 TABLE(syscall): 810 SYSCALL ! 0x100 = sun syscall 811 BPT ! 0x101 = pseudo breakpoint instruction 812 STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) 813 SYSCALL ! 0x108 = svr4 syscall 814 SYSCALL ! 0x109 = bsd syscall 815 BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup 816 STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); 817 STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) 818 STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) 819 STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) 820 STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) 821 STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) 822 STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) 823 SYSCALL ! 0x140 SVID syscall (Solaris 2.7) 824 SYSCALL ! 0x141 SPARC International syscall 825 SYSCALL ! 0x142 OS Vendor syscall 826 SYSCALL ! 0x143 HW OEM syscall 827 STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) 828 STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) 829 STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) 830 STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) 831 STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) 832 STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) 833 STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) 834 STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) 835 ! Traps beyond 0x17f are reserved 836 UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) 837 UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) 838 UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) 839 UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) 840 UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) 841 UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) 842 UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) 843 UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) 844 UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) 845 UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) 846 UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) 847 UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) 848 UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) 849 UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) 850 UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) 851 UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) 852 853 /* Traps from TL>0 -- traps from supervisor mode */ 854 #undef TABLE 855 #ifdef __STDC__ 856 #define TABLE(name) nucleus_ ## name 857 #else 858 #define TABLE(name) nucleus_/**/name 859 #endif 860 trapbase_priv: 861 UTRAP(0x000) ! 000 = reserved -- Use it to boot 862 /* We should not get the next 5 traps */ 863 UTRAP(0x001) ! 001 = POR Reset -- ROM should get this 864 UTRAP(0x002) ! 002 = WDR Watchdog -- ROM should get this 865 UTRAP(0x003) ! 003 = XIR -- ROM should get this 866 UTRAP(0x004) ! 004 = SIR -- ROM should get this 867 UTRAP(0x005) ! 005 = RED state exception 868 UTRAP(0x006); UTRAP(0x007) 869 ktextfault: 870 VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except 871 VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss -- no MMU 872 VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err 873 UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) 874 TRAP(T_ILLINST) ! 010 = illegal instruction 875 TRAP(T_PRIVINST) ! 011 = privileged instruction 876 UTRAP(0x012) ! 012 = unimplemented LDD 877 UTRAP(0x013) ! 013 = unimplemented STD 878 UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) 879 UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) 880 UTRAP(0x01e); UTRAP(0x01f) 881 TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr 882 TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception 883 TRAP(T_FP_OTHER) ! 022 = other fp exception 884 TRAP(T_TAGOF) ! 023 = tag overflow 885 CLEANWIN1 ! 024-027 = clean window trap 886 TRAP(T_DIV0) ! 028 = divide by zero 887 UTRAP(0x029) ! 029 = internal processor error 888 UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) 889 kdatafault: 890 VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault 891 UTRAP(0x031) ! 031 = data MMU miss -- no MMU 892 VTRAP(T_DATA_ERROR, winfault) ! 032 = data fetch fault 893 VTRAP(T_DATA_PROT, winfault) ! 033 = data fetch fault 894 VTRAP(T_ALIGN, checkalign) ! 034 = address alignment error -- we could fix it inline... 895 TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... 896 TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... 897 TRAP(T_PRIVACT) ! 037 = privileged action 898 UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); 899 UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); 900 VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault 901 SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt 902 HARDINT4U(2) ! 042 = level 2 interrupt 903 HARDINT4U(3) ! 043 = level 3 interrupt 904 SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt 905 HARDINT4U(5) ! 045 = level 5 interrupt 906 SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt 907 HARDINT4U(7) ! 047 = level 7 interrupt 908 HARDINT4U(8) ! 048 = level 8 interrupt 909 HARDINT4U(9) ! 049 = level 9 interrupt 910 HARDINT4U(10) ! 04a = level 10 interrupt 911 HARDINT4U(11) ! 04b = level 11 interrupt 912 ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt 913 HARDINT4U(13) ! 04d = level 13 interrupt 914 HARDINT4U(14) ! 04e = level 14 interrupt 915 HARDINT4U(15) ! 04f = nonmaskable interrupt 916 UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) 917 UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) 918 UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) 919 VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector 920 TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint 921 TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint 922 TRAP(T_ECCERR) ! 063 = corrected ECC error 923 kfast_IMMU_miss: ! 064 = fast instr access MMU miss 924 ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer 925 #ifdef NO_TSB 926 ba,a %icc, instr_miss 927 #endif 928 ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register 929 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 930 brgez,pn %g5, instr_miss ! Entry invalid? Punt 931 cmp %g1, %g4 ! Compare TLB tags 932 bne,pn %xcc, instr_miss ! Got right tag? 933 nop 934 CLRTT 935 stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 936 retry ! Try new mapping 937 1: 938 sir 939 TA32 940 kfast_DMMU_miss: ! 068 = fast data access MMU miss 941 ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer 942 #ifdef NO_TSB 943 ba,a %icc, data_miss 944 #endif 945 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 946 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 947 brgez,pn %g5, data_miss ! Entry invalid? Punt 948 cmp %g1, %g4 ! Compare TLB tags 949 bnz,pn %xcc, data_miss ! Got right tag? 950 nop 951 CLRTT 952 #ifdef TRAPSTATS 953 sethi %hi(_C_LABEL(kdhit)), %g1 954 lduw [%g1+%lo(_C_LABEL(kdhit))], %g2 955 inc %g2 956 stw %g2, [%g1+%lo(_C_LABEL(kdhit))] 957 #endif 958 stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 959 retry ! Try new mapping 960 1: 961 sir 962 TA32 963 kfast_DMMU_protection: ! 06c = fast data access MMU protection 964 #ifdef TRAPSTATS 965 sethi %hi(_C_LABEL(kdprot)), %g1 966 lduw [%g1+%lo(_C_LABEL(kdprot))], %g2 967 inc %g2 968 stw %g2, [%g1+%lo(_C_LABEL(kdprot))] 969 #endif 970 #ifdef HWREF 971 ba,a,pt %xcc, dmmu_write_fault 972 #else 973 ba,a,pt %xcc, winfault 974 #endif 975 nop 976 TA32 977 TRAP(0x070) ! 0x070 fast_ECC_error 978 ! Implementation dependent traps 979 UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) 980 UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) 981 UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) 982 TABLE(uspill): 983 SPILL64(1,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows 984 SPILL32(2,ASI_AIUS) ! 0x084 spill_1_normal 985 SPILLBOTH(1b,2b,ASI_AIUS) ! 0x088 spill_2_normal 986 UTRAP(0x08c); TA32 ! 0x08c spill_3_normal 987 TABLE(kspill): 988 SPILL64(1,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 989 SPILL32(2,ASI_N) ! 0x094 spill_5_normal 990 SPILLBOTH(1b,2b,ASI_N) ! 0x098 spill_6_normal 991 UTRAP(0x09c); TA32 ! 0x09c spill_7_normal 992 TABLE(uspillk): 993 SPILL64(1,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in nucleus mode 994 SPILL32(2,ASI_AIUS) ! 0x0a4 spill_1_other 995 SPILLBOTH(1b,2b,ASI_AIUS) ! 0x0a8 spill_2_other 996 UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other 997 UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other 998 UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other 999 UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other 1000 UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other 1001 TABLE(ufill): 1002 FILL64(nufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running nucleus mode from user 1003 FILL32(nufill4,ASI_AIUS) ! 0x0c4 fill_1_normal 1004 FILLBOTH(nufill8,nufill4,ASI_AIUS) ! 0x0c8 fill_2_normal 1005 UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal 1006 TABLE(sfill): 1007 FILL64(sfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running nucleus mode from supervisor 1008 FILL32(sfill4,ASI_N) ! 0x0d4 fill_5_normal 1009 FILLBOTH(sfill8,sfill4,ASI_N) ! 0x0d8 fill_6_normal 1010 UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal 1011 TABLE(kfill): 1012 FILL64(nkfill8,ASI_AIUS) ! 0x0e0 fill_0_other -- used to fill user windows when running nucleus mode -- will we ever use this? 1013 FILL32(nkfill4,ASI_AIUS) ! 0x0e4 fill_1_other 1014 FILLBOTH(nkfill8,nkfill4,ASI_AIUS)! 0x0e8 fill_2_other 1015 UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other 1016 UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other 1017 UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other 1018 UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other 1019 UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other 1020 TABLE(syscall): 1021 SYSCALL ! 0x100 = sun syscall 1022 BPT ! 0x101 = pseudo breakpoint instruction 1023 STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) 1024 SYSCALL ! 0x108 = svr4 syscall 1025 SYSCALL ! 0x109 = bsd syscall 1026 BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup 1027 STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); 1028 STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) 1029 STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) 1030 STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) 1031 STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) 1032 STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) 1033 STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) 1034 STRAP(0x140); STRAP(0x141); STRAP(0x142); STRAP(0x143); STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) 1035 STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) 1036 STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) 1037 STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) 1038 STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) 1039 STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) 1040 STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) 1041 STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) 1042 ! Traps beyond 0x17f are reserved 1043 UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) 1044 UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) 1045 UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) 1046 UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) 1047 UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) 1048 UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) 1049 UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) 1050 UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) 1051 UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) 1052 UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) 1053 UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) 1054 UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) 1055 UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) 1056 UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) 1057 UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) 1058 UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) 1059 1060 #ifdef SUN4V 1061 1062 /* Macros for sun4v traps */ 1063 1064 .macro sun4v_trap_entry count 1065 .rept \count 1066 ba slowtrap 1067 nop 1068 .align 32 1069 .endr 1070 .endm 1071 1072 .macro sun4v_trap_entry_fail count 1073 .rept \count 1074 sir 1075 .align 32 1076 .endr 1077 .endm 1078 1079 .macro sun4v_trap_entry_spill_fill_fail count 1080 .rept \count 1081 sir 1082 .align 128 1083 .endr 1084 .endm 1085 1086 /* The actual trap base for sun4v */ 1087 .align 0x8000 1088 .globl _C_LABEL(trapbase_sun4v) 1089 _C_LABEL(trapbase_sun4v): 1090 ! 1091 ! trap level 0 1092 ! 1093 sun4v_trap_entry 8 ! 0x000-0x007 1094 VTRAP(T_INST_EXCEPT, sun4v_tl0_itsb_miss) ! 0x008 - inst except 1095 VTRAP(T_TEXTFAULT, sun4v_tl0_itsb_miss) ! 0x009 - inst MMU miss 1096 sun4v_trap_entry 26 ! 0x00a-0x023 1097 CLEANWIN0 ! 0x24-0x27 = clean window 1098 sun4v_trap_entry 8 ! 0x028-0x02f 1099 VTRAP(T_DATAFAULT, sun4v_datatrap) ! 0x030 = data access exception (UA 2005) 1100 VTRAP(T_DATA_MMU_MISS, sun4v_dtsb_miss) ! 0x031 = data MMU miss 1101 sun4v_trap_entry 2 ! 0x032-0x033 1102 TRAP(T_ALIGN) ! 0x034 = address alignment error 1103 sun4v_trap_entry 12 ! 0x035-0x040 1104 HARDINT4V(1) ! 0x041 = level 1 interrupt 1105 HARDINT4V(2) ! 0x042 = level 2 interrupt 1106 HARDINT4V(3) ! 0x043 = level 3 interrupt 1107 HARDINT4V(4) ! 0x044 = level 4 interrupt 1108 HARDINT4V(5) ! 0x045 = level 5 interrupt 1109 HARDINT4V(6) ! 0x046 = level 6 interrupt 1110 HARDINT4V(7) ! 0x047 = level 7 interrupt 1111 HARDINT4V(8) ! 0x048 = level 8 interrupt 1112 HARDINT4V(9) ! 0x049 = level 9 interrupt 1113 HARDINT4V(10) ! 0x04a = level 10 interrupt 1114 HARDINT4V(11) ! 0x04b = level 11 interrupt 1115 HARDINT4V(12) ! 0x04c = level 12 interrupt 1116 HARDINT4V(13) ! 0x04d = level 13 interrupt 1117 HARDINT4V(14) ! 0x04e = level 14 interrupt 1118 HARDINT4V(15) ! 0x04f = level 15 interrupt 1119 sun4v_trap_entry 28 ! 0x050-0x06b 1120 VTRAP(T_FDMMU_PROT, sun4v_tl0_dtsb_prot) ! 0x06c 1121 sun4v_trap_entry 15 ! 0x06d-0x07b 1122 VTRAP(T_CPU_MONDO, sun4v_cpu_mondo) ! 0x07c = cpu mondo 1123 VTRAP(T_DEV_MONDO, sun4v_dev_mondo) ! 0x07d = dev mondo 1124 sun4v_trap_entry 2 ! 0x07e-0x07f 1125 SPILL64(uspill8_sun4vt0,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode 1126 SPILL32(uspill4_sun4vt0,ASI_AIUS) ! 0x084 spill_1_normal 1127 SPILLBOTH(uspill8_sun4vt0,uspill4_sun4vt0,ASI_AIUS) ! 0x088 spill_2_normal 1128 sun4v_trap_entry_spill_fill_fail 1 ! 0x08c spill_3_normal 1129 SPILL64(kspill8_sun4vt0,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 1130 SPILL32(kspill4_sun4vt0,ASI_N) ! 0x094 spill_5_normal 1131 SPILLBOTH(kspill8_sun4vt0,kspill4_sun4vt0,ASI_N) ! 0x098 spill_6_normal 1132 sun4v_trap_entry_spill_fill_fail 1 ! 0x09c spill_7_normal 1133 SPILL64(uspillk8_sun4vt0,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode 1134 SPILL32(uspillk4_sun4vt0,ASI_AIUS) ! 0x0a4 spill_1_other 1135 SPILLBOTH(uspillk8_sun4vt0,uspillk4_sun4vt0,ASI_AIUS) ! 0x0a8 spill_2_other 1136 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ac spill_3_other 1137 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b0 spill_4_other 1138 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b4 spill_5_other 1139 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b8 spill_6_other 1140 sun4v_trap_entry_spill_fill_fail 1 ! 0x0bc spill_7_other 1141 FILL64(ufill8_sun4vt0,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode 1142 FILL32(ufill4_sun4vt0,ASI_AIUS) ! 0x0c4 fill_1_normal 1143 FILLBOTH(ufill8_sun4vt0,ufill4_sun4vt0,ASI_AIUS) ! 0x0c8 fill_2_normal 1144 sun4v_trap_entry_spill_fill_fail 1 ! 0x0cc fill_3_normal 1145 FILL64(kfill8_sun4vt0,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode 1146 FILL32(kfill4_sun4vt0,ASI_N) ! 0x0d4 fill_5_normal 1147 FILLBOTH(kfill8_sun4vt0,kfill4_sun4vt0,ASI_N) ! 0x0d8 fill_6_normal 1148 sun4v_trap_entry_spill_fill_fail 1 ! 0x0dc fill_7_normal 1149 FILL64(ufillk8_sun4vt0,ASI_AIUS) ! 0x0e0 fill_0_other 1150 FILL32(ufillk4_sun4vt0,ASI_AIUS) ! 0x0e4 fill_1_other 1151 FILLBOTH(ufillk8_sun4vt0,ufillk4_sun4vt0,ASI_AIUS) ! 0x0e8 fill_2_other 1152 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ec fill_3_other 1153 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f0 fill_4_other 1154 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f4 fill_5_other 1155 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f8 fill_6_other 1156 sun4v_trap_entry_spill_fill_fail 1 ! 0x0fc fill_7_other 1157 SYSCALL ! 0x100 = syscall 1158 BPT ! 0x101 = pseudo breakpoint instruction 1159 sun4v_trap_entry 254 ! 0x102-0x1ff 1160 ! 1161 ! trap level 1 1162 ! 1163 sun4v_trap_entry 36 ! 0x000-0x023 1164 CLEANWIN1 ! 0x24-0x27 = clean window 1165 sun4v_trap_entry 8 ! 0x028-0x02F 1166 VTRAP(T_DATAFAULT, sun4v_tl1_ptbl_miss) ! 0x030 = ??? 1167 VTRAP(T_DATA_MMU_MISS, sun4v_tl1_dtsb_miss) ! 0x031 = data MMU miss 1168 VTRAP(T_DATA_ERROR, sun4v_tl1_ptbl_miss) ! 0x032 = ??? 1169 VTRAP(T_DATA_PROT, sun4v_tl1_ptbl_miss) ! 0x033 = ??? 1170 sun4v_trap_entry 56 ! 0x034-0x06b 1171 VTRAP(T_FDMMU_PROT, sun4v_tl1_dtsb_prot) ! 0x06c 1172 sun4v_trap_entry 19 ! 0x06d-0x07f 1173 sun4v_tl1_uspill_normal ! 0x080 spill_0_normal -- save user windows 1174 sun4v_tl1_uspill_normal ! 0x084 spill_1_normal 1175 sun4v_tl1_uspill_normal ! 0x088 spill_2_normal 1176 sun4v_trap_entry_spill_fill_fail 1 ! 0x08c spill_3_normal 1177 SPILL64(kspill8_sun4vt1,ASI_N) ! 0x090 spill_4_normal -- save supervisor windows 1178 SPILL32(kspill4_sun4vt1,ASI_N) ! 0x094 spill_5_normal 1179 SPILLBOTH(kspill8_sun4vt1,kspill4_sun4vt1,ASI_N) ! 0x098 spill_6_normal 1180 sun4v_trap_entry_spill_fill_fail 1 ! 0x09c spill_7_normal 1181 sun4v_tl1_uspill_other ! 0x0a0 spill_0_other -- save user windows in nucleus mode 1182 sun4v_tl1_uspill_other ! 0x0a4 spill_1_other 1183 sun4v_tl1_uspill_other ! 0x0a8 spill_2_other 1184 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ac spill_3_other 1185 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b0 spill_4_other 1186 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b4 spill_5_other 1187 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b8 spill_6_other 1188 sun4v_trap_entry_spill_fill_fail 1 ! 0x0bc spill_7_other 1189 FILL64(ufill8_sun4vt1,ASI_AIUS) ! 0x0c0 fill_0_normal -- fill windows when running nucleus mode from user 1190 FILL32(ufill4_sun4vt1,ASI_AIUS) ! 0x0c4 fill_1_normal 1191 FILLBOTH(ufill8_sun4vt1,ufill4_sun4vt1,ASI_AIUS) ! 0x0c8 fill_2_normal 1192 sun4v_trap_entry_spill_fill_fail 1 ! 0x0cc fill_3_normal 1193 FILL64(kfill8_sun4vt1,ASI_N) ! 0x0d0 fill_4_normal -- fill windows when running nucleus mode from supervisor 1194 FILL32(kfill4_sun4vt1,ASI_N) ! 0x0d4 fill_5_normal 1195 FILLBOTH(kfill8_sun4vt1,kfill4_sun4vt1,ASI_N) ! 0x0d8 fill_6_normal 1196 sun4v_trap_entry_spill_fill_fail 1 ! 0x0dc fill_7_normal 1197 FILL64(ufillk8_sun4vt1,ASI_AIUS) ! 0x0e0 fill_0_other -- fill user windows when running nucleus mode -- will we ever use this? 1198 FILL32(ufillk4_sun4vt1,ASI_AIUS) ! 0x0e4 fill_1_other 1199 FILLBOTH(ufillk8_sun4vt1,ufillk4_sun4vt1,ASI_AIUS) ! 0x0e8 fill_2_other 1200 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ec fill_3_other 1201 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f0 fill_4_other 1202 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f4 fill_5_other 1203 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f8 fill_6_other 1204 sun4v_trap_entry_spill_fill_fail 1 ! 0x0fc fill_7_other 1205 sun4v_trap_entry_fail 256 ! 0x100-0x1ff 1206 1207 #endif 1208 1209 #if 0 1210 /* 1211 * If the cleanwin trap handler detects an overflow we come here. 1212 * We need to fix up the window registers, switch to the interrupt 1213 * stack, and then trap to the debugger. 1214 */ 1215 cleanwin_overflow: 1216 !! We've already incremented %cleanwin 1217 !! So restore %cwp 1218 rdpr %cwp, %l0 1219 dec %l0 1220 wrpr %l0, %g0, %cwp 1221 set EINTSTACK-STKB-CC64FSZ, %l0 1222 save %l0, 0, %sp 1223 1224 ta 1 ! Enter debugger 1225 sethi %hi(1f), %o0 1226 call _C_LABEL(panic) 1227 or %o0, %lo(1f), %o0 1228 restore 1229 retry 1230 .data 1231 1: 1232 .asciz "Kernel stack overflow!" 1233 _ALIGN 1234 .text 1235 #endif 1236 1237 #ifdef NOTDEF_DEBUG 1238 /* 1239 * A hardware red zone is impossible. We simulate one in software by 1240 * keeping a `red zone' pointer; if %sp becomes less than this, we panic. 1241 * This is expensive and is only enabled when debugging. 1242 */ 1243 #define REDSIZE (PCB_SIZE) /* Mark used portion of pcb structure out of bounds */ 1244 #define REDSTACK 2048 /* size of `panic: stack overflow' region */ 1245 .data 1246 _ALIGN 1247 redzone: 1248 .xword _C_LABEL(XXX) + REDSIZE 1249 redstack: 1250 .space REDSTACK 1251 eredstack: 1252 Lpanic_red: 1253 .asciz "kernel stack overflow" 1254 _ALIGN 1255 .text 1256 1257 /* set stack pointer redzone to base+minstack; alters base */ 1258 #define SET_SP_REDZONE(base, tmp) \ 1259 add base, REDSIZE, base; \ 1260 sethi %hi(_C_LABEL(redzone)), tmp; \ 1261 stx base, [tmp + %lo(_C_LABEL(redzone))] 1262 1263 /* variant with a constant */ 1264 #define SET_SP_REDZONE_CONST(const, tmp1, tmp2) \ 1265 set (const) + REDSIZE, tmp1; \ 1266 sethi %hi(_C_LABEL(redzone)), tmp2; \ 1267 stx tmp1, [tmp2 + %lo(_C_LABEL(redzone))] 1268 1269 /* check stack pointer against redzone (uses two temps) */ 1270 #define CHECK_SP_REDZONE(t1, t2) \ 1271 sethi KERNBASE, t1; \ 1272 cmp %sp, t1; \ 1273 blu,pt %xcc, 7f; \ 1274 sethi %hi(_C_LABEL(redzone)), t1; \ 1275 ldx [t1 + %lo(_C_LABEL(redzone))], t2; \ 1276 cmp %sp, t2; /* if sp >= t2, not in red zone */ \ 1277 blu panic_red; nop; /* and can continue normally */ \ 1278 7: 1279 1280 panic_red: 1281 /* move to panic stack */ 1282 stx %g0, [t1 + %lo(_C_LABEL(redzone))]; 1283 set eredstack - BIAS, %sp; 1284 /* prevent panic() from lowering ipl */ 1285 sethi %hi(_C_LABEL(panicstr)), t2; 1286 set Lpanic_red, t2; 1287 st t2, [t1 + %lo(_C_LABEL(panicstr))]; 1288 wrpr g0, 15, %pil /* t1 = splhigh() */ 1289 save %sp, -CCF64SZ, %sp; /* preserve current window */ 1290 sethi %hi(Lpanic_red), %o0; 1291 call _C_LABEL(panic); 1292 or %o0, %lo(Lpanic_red), %o0; 1293 1294 1295 #else 1296 1297 #define SET_SP_REDZONE(base, tmp) 1298 #define SET_SP_REDZONE_CONST(const, t1, t2) 1299 #define CHECK_SP_REDZONE(t1, t2) 1300 #endif 1301 1302 #define TRACESIZ 0x01000 1303 .globl _C_LABEL(trap_trace) 1304 .globl _C_LABEL(trap_trace_ptr) 1305 .globl _C_LABEL(trap_trace_end) 1306 .globl _C_LABEL(trap_trace_dis) 1307 .data 1308 _C_LABEL(trap_trace_dis): 1309 .word 1, 1 ! Starts disabled. DDB turns it on. 1310 _C_LABEL(trap_trace_ptr): 1311 .word 0, 0, 0, 0 1312 _C_LABEL(trap_trace): 1313 .space TRACESIZ 1314 _C_LABEL(trap_trace_end): 1315 .space 0x20 ! safety margin 1316 1317 1318 /* 1319 * v9 machines do not have a trap window. 1320 * 1321 * When we take a trap the trap state is pushed on to the stack of trap 1322 * registers, interrupts are disabled, then we switch to an alternate set 1323 * of global registers. 1324 * 1325 * The trap handling code needs to allocate a trap frame on the kernel, or 1326 * for interrupts, the interrupt stack, save the out registers to the trap 1327 * frame, then switch to the normal globals and save them to the trap frame 1328 * too. 1329 * 1330 * XXX it would be good to save the interrupt stack frame to the kernel 1331 * stack so we wouldn't have to copy it later if we needed to handle a AST. 1332 * 1333 * Since kernel stacks are all on one page and the interrupt stack is entirely 1334 * within the locked TLB, we can use physical addressing to save out our 1335 * trap frame so we don't trap during the TRAP_SETUP() operation. There 1336 * is unfortunately no supportable method for issuing a non-trapping save. 1337 * 1338 * However, if we use physical addresses to save our trapframe, we will need 1339 * to clear out the data cache before continuing much further. 1340 * 1341 * In short, what we need to do is: 1342 * 1343 * all preliminary processing is done using the alternate globals 1344 * 1345 * When we allocate our trap windows we must give up our globals because 1346 * their state may have changed during the save operation 1347 * 1348 * we need to save our normal globals as soon as we have a stack 1349 * 1350 * Finally, we may now call C code. 1351 * 1352 * This macro will destroy %g5-%g7. %g0-%g4 remain unchanged. 1353 * 1354 * In order to properly handle nested traps without lossage, alternate 1355 * global %g6 is used as a kernel stack pointer. It is set to the last 1356 * allocated stack pointer (trapframe) and the old value is stored in 1357 * tf_kstack. It is restored when returning from a trap. It is cleared 1358 * on entering user mode. 1359 */ 1360 1361 /* 1362 * Other misc. design criteria: 1363 * 1364 * When taking an address fault, fault info is in the sfsr, sfar, 1365 * TLB_TAG_ACCESS registers. If we take another address fault 1366 * while trying to handle the first fault then that information, 1367 * the only information that tells us what address we trapped on, 1368 * can potentially be lost. This trap can be caused when allocating 1369 * a register window with which to handle the trap because the save 1370 * may try to store or restore a register window that corresponds 1371 * to part of the stack that is not mapped. Preventing this trap, 1372 * while possible, is much too complicated to do in a trap handler, 1373 * and then we will need to do just as much work to restore the processor 1374 * window state. 1375 * 1376 * Possible solutions to the problem: 1377 * 1378 * Since we have separate AG, MG, and IG, we could have all traps 1379 * above level-1 preserve AG and use other registers. This causes 1380 * a problem for the return from trap code which is coded to use 1381 * alternate globals only. 1382 * 1383 * We could store the trapframe and trap address info to the stack 1384 * using physical addresses. Then we need to read it back using 1385 * physical addressing, or flush the D$. 1386 * 1387 * We could identify certain registers to hold address fault info. 1388 * this means that these registers need to be preserved across all 1389 * fault handling. But since we only have 7 useable globals, that 1390 * really puts a cramp in our style. 1391 * 1392 * Finally, there is the issue of returning from kernel mode to user 1393 * mode. If we need to issue a restore of a user window in kernel 1394 * mode, we need the window control registers in a user mode setup. 1395 * If the trap handlers notice the register windows are in user mode, 1396 * they will allocate a trapframe at the bottom of the kernel stack, 1397 * overwriting the frame we were trying to return to. This means that 1398 * we must complete the restoration of all registers *before* switching 1399 * to a user-mode window configuration. 1400 * 1401 * Essentially we need to be able to write re-entrant code w/no stack. 1402 */ 1403 .data 1404 trap_setup_msg: 1405 .asciz "TRAP_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" 1406 _ALIGN 1407 intr_setup_msg: 1408 .asciz "INTR_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" 1409 _ALIGN 1410 .text 1411 1412 #ifdef DEBUG 1413 /* Only save a snapshot of locals and ins in DEBUG kernels */ 1414 #define SAVE_LOCALS_INS \ 1415 /* Save local registers to trap frame */ \ 1416 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)]; \ 1417 stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)]; \ 1418 stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)]; \ 1419 stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)]; \ 1420 stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)]; \ 1421 stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)]; \ 1422 stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)]; \ 1423 stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)]; \ 1424 \ 1425 /* Save in registers to trap frame */ \ 1426 stx %i0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)]; \ 1427 stx %i1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)]; \ 1428 stx %i2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)]; \ 1429 stx %i3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)]; \ 1430 stx %i4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)]; \ 1431 stx %i5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)]; \ 1432 stx %i6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)]; \ 1433 stx %i7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)]; \ 1434 \ 1435 stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT]; 1436 #else 1437 #define SAVE_LOCALS_INS 1438 #endif 1439 1440 #ifdef _LP64 1441 #define FIXUP_TRAP_STACK \ 1442 btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ 1443 bnz,pt %icc, 1f; \ 1444 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1445 inc -BIAS, %g6; \ 1446 1: 1447 #else 1448 #define FIXUP_TRAP_STACK \ 1449 srl %g6, 0, %g6; /* truncate at 32-bits */ \ 1450 btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ 1451 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1452 add %g6, BIAS, %g5; \ 1453 movne %icc, %g5, %g6; 1454 #endif 1455 1456 #ifdef _LP64 1457 #define TRAP_SETUP(stackspace) \ 1458 sethi %hi(CPCB), %g6; \ 1459 sethi %hi((stackspace)), %g5; \ 1460 LDPTR [%g6 + %lo(CPCB)], %g6; \ 1461 sethi %hi(USPACE), %g7; /* Always multiple of page size */ \ 1462 or %g5, %lo((stackspace)), %g5; \ 1463 add %g6, %g7, %g6; \ 1464 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1465 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1466 \ 1467 sub %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ 1468 movrz %g7, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ 1469 FIXUP_TRAP_STACK \ 1470 SAVE_LOCALS_INS \ 1471 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1472 stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1473 stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ 1474 stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ 1475 stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ 1476 stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ 1477 stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ 1478 \ 1479 stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ 1480 brz,pt %g7, 1f; /* If we were in kernel mode start saving globals */ \ 1481 stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ 1482 mov CTX_PRIMARY, %g7; \ 1483 /* came from user mode -- switch to kernel mode stack */ \ 1484 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1485 wrpr %g0, 0, %canrestore; \ 1486 wrpr %g0, %g5, %otherwin; \ 1487 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1488 \ 1489 SET_MMU_CONTEXTID %g0, %g7,%g5; /* Switch MMU to kernel primary context */ \ 1490 sethi %hi(KERNBASE), %g5; \ 1491 flush %g5; /* Some convenient address that won't trap */ \ 1492 1: 1493 1494 /* 1495 * Interrupt setup is almost exactly like trap setup, but we need to 1496 * go to the interrupt stack if (a) we came from user mode or (b) we 1497 * came from kernel mode on the kernel stack. 1498 * 1499 * We don't guarantee any registers are preserved during this operation. 1500 * So we can be more efficient. 1501 */ 1502 #define INTR_SETUP(stackspace) \ 1503 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1504 \ 1505 sethi %hi(EINTSTACK-BIAS), %g6; \ 1506 sethi %hi(EINTSTACK-INTSTACK), %g4; \ 1507 \ 1508 or %g6, %lo(EINTSTACK-BIAS), %g6; /* Base of interrupt stack */ \ 1509 dec %g4; /* Make it into a mask */ \ 1510 \ 1511 sub %g6, %sp, %g1; /* Offset from interrupt stack */ \ 1512 sethi %hi((stackspace)), %g5; \ 1513 \ 1514 or %g5, %lo((stackspace)), %g5; \ 1515 \ 1516 andn %g1, %g4, %g4; /* Are we out of the interrupt stack range? */ \ 1517 xor %g7, WSTATE_KERN, %g3; /* Are we on the user stack ? */ \ 1518 \ 1519 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1520 orcc %g3, %g4, %g0; /* Definitely not off the interrupt stack */ \ 1521 \ 1522 sethi %hi(CPUINFO_VA + CI_EINTSTACK), %g4; \ 1523 bz,a,pt %xcc, 1f; \ 1524 mov %sp, %g6; \ 1525 \ 1526 ldx [%g4 + %lo(CPUINFO_VA + CI_EINTSTACK)], %g4; \ 1527 movrnz %g4, %g4, %g6; /* Use saved intr stack if exists */ \ 1528 \ 1529 1: add %g6, %g5, %g5; /* Allocate a stack frame */ \ 1530 btst 1, %g6; \ 1531 bnz,pt %icc, 1f; \ 1532 \ 1533 mov %g5, %g6; \ 1534 \ 1535 add %g5, -BIAS, %g6; \ 1536 \ 1537 1: SAVE_LOCALS_INS \ 1538 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1539 stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1540 stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ 1541 stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ 1542 stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ 1543 stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ 1544 \ 1545 stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ 1546 stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ 1547 stx %i6, [%sp + CC64FSZ + BIAS + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ 1548 brz,pt %g3, 1f; /* If we were in kernel mode start saving globals */ \ 1549 stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ 1550 /* came from user mode -- switch to kernel mode stack */ \ 1551 rdpr %otherwin, %g5; /* Has this already been done? */ \ 1552 \ 1553 brnz,pn %g5, 1f; /* Don't set this twice */ \ 1554 \ 1555 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1556 \ 1557 wrpr %g0, 0, %canrestore; \ 1558 \ 1559 wrpr %g0, %g5, %otherwin; \ 1560 \ 1561 mov CTX_PRIMARY, %g7; \ 1562 \ 1563 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1564 \ 1565 SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ 1566 \ 1567 sethi %hi(KERNBASE), %g5; \ 1568 flush %g5; /* Some convenient address that won't trap */ \ 1569 1: 1570 1571 #else /* _LP64 */ 1572 1573 #define TRAP_SETUP(stackspace) \ 1574 sethi %hi(CPCB), %g6; \ 1575 sethi %hi((stackspace)), %g5; \ 1576 LDPTR [%g6 + %lo(CPCB)], %g6; \ 1577 sethi %hi(USPACE), %g7; \ 1578 or %g5, %lo((stackspace)), %g5; \ 1579 add %g6, %g7, %g6; \ 1580 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1581 \ 1582 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1583 subcc %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ 1584 movz %icc, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ 1585 FIXUP_TRAP_STACK \ 1586 SAVE_LOCALS_INS \ 1587 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1588 stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1589 stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ 1590 stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ 1591 stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ 1592 stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ 1593 stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ 1594 \ 1595 stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ 1596 brz,pn %g7, 1f; /* If we were in kernel mode start saving globals */ \ 1597 stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ 1598 mov CTX_PRIMARY, %g7; \ 1599 /* came from user mode -- switch to kernel mode stack */ \ 1600 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1601 wrpr %g0, 0, %canrestore; \ 1602 wrpr %g0, %g5, %otherwin; \ 1603 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1604 \ 1605 SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ 1606 sethi %hi(KERNBASE), %g5; \ 1607 flush %g5; /* Some convenient address that won't trap */ \ 1608 1: 1609 1610 /* 1611 * Interrupt setup is almost exactly like trap setup, but we need to 1612 * go to the interrupt stack if (a) we came from user mode or (b) we 1613 * came from kernel mode on the kernel stack. 1614 * 1615 * We don't guarantee any registers are preserved during this operation. 1616 */ 1617 #define INTR_SETUP(stackspace) \ 1618 sethi %hi(EINTSTACK), %g1; \ 1619 sethi %hi((stackspace)), %g5; \ 1620 btst 1, %sp; \ 1621 add %sp, BIAS, %g6; \ 1622 movz %icc, %sp, %g6; \ 1623 or %g1, %lo(EINTSTACK), %g1; \ 1624 srl %g6, 0, %g6; /* truncate at 32-bits */ \ 1625 set (EINTSTACK-INTSTACK), %g7; \ 1626 or %g5, %lo((stackspace)), %g5; \ 1627 sub %g1, %g6, %g2; /* Determine if we need to switch to intr stack or not */ \ 1628 dec %g7; /* Make it into a mask */ \ 1629 sethi %hi(CPUINFO_VA + CI_EINTSTACK), %g3; \ 1630 andncc %g2, %g7, %g0; /* XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ 1631 LDPTR [%g3 + %lo(CPUINFO_VA + CI_EINTSTACK)], %g3; \ 1632 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1633 movrnz %g3, %g3, %g1; /* Use saved intr stack if exists */ \ 1634 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1635 movnz %xcc, %g1, %g6; /* Stay on interrupt stack? */ \ 1636 cmp %g7, WSTATE_KERN; /* User or kernel sp? */ \ 1637 movnz %icc, %g1, %g6; /* Stay on interrupt stack? */ \ 1638 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1639 \ 1640 SAVE_LOCALS_INS \ 1641 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1642 stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1643 stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ 1644 stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ 1645 stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ 1646 stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ 1647 stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ 1648 stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ 1649 stx %i6, [%sp + CC64FSZ + STKB + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ 1650 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1651 stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ 1652 cmp %g7, WSTATE_KERN; /* Compare & leave in register */ \ 1653 be,pn %icc, 1f; /* If we were in kernel mode start saving globals */ \ 1654 /* came from user mode -- switch to kernel mode stack */ \ 1655 rdpr %otherwin, %g5; /* Has this already been done? */ \ 1656 tst %g5; tnz %xcc, 1; nop; /* DEBUG -- this should _NEVER_ happen */ \ 1657 brnz,pn %g5, 1f; /* Don't set this twice */ \ 1658 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1659 wrpr %g0, 0, %canrestore; \ 1660 mov CTX_PRIMARY, %g7; \ 1661 wrpr %g0, %g5, %otherwin; \ 1662 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1663 SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ 1664 sethi %hi(KERNBASE), %g5; \ 1665 flush %g5; /* Some convenient address that won't trap */ \ 1666 1: 1667 #endif /* _LP64 */ 1668 1669 #ifdef DEBUG 1670 1671 /* Look up kpte to test algorithm */ 1672 .globl asmptechk 1673 asmptechk: 1674 mov %o0, %g4 ! pmap->pm_segs 1675 mov %o1, %g3 ! Addr to lookup -- mind the context 1676 1677 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1678 brz,pt %g5, 0f ! Should be zero or -1 1679 inc %g5 ! Make -1 -> 0 1680 brnz,pn %g5, 1f ! Error! 1681 0: 1682 srlx %g3, STSHIFT, %g5 1683 and %g5, STMASK, %g5 1684 sll %g5, 3, %g5 1685 add %g4, %g5, %g4 1686 DLFLUSH(%g4,%g5) 1687 ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED 1688 DLFLUSH2(%g5) 1689 brz,pn %g4, 1f ! NULL entry? check somewhere else 1690 1691 srlx %g3, PDSHIFT, %g5 1692 and %g5, PDMASK, %g5 1693 sll %g5, 3, %g5 1694 add %g4, %g5, %g4 1695 DLFLUSH(%g4,%g5) 1696 ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED 1697 DLFLUSH2(%g5) 1698 brz,pn %g4, 1f ! NULL entry? check somewhere else 1699 1700 srlx %g3, PTSHIFT, %g5 ! Convert to ptab offset 1701 and %g5, PTMASK, %g5 1702 sll %g5, 3, %g5 1703 add %g4, %g5, %g4 1704 DLFLUSH(%g4,%g5) 1705 ldxa [%g4] ASI_PHYS_CACHED, %g6 1706 DLFLUSH2(%g5) 1707 brgez,pn %g6, 1f ! Entry invalid? Punt 1708 srlx %g6, 32, %o0 1709 retl 1710 srl %g6, 0, %o1 1711 1: 1712 mov %g0, %o1 1713 retl 1714 mov %g0, %o0 1715 1716 .data 1717 2: 1718 .asciz "asmptechk: %x %x %x %x:%x\n" 1719 _ALIGN 1720 .text 1721 #endif 1722 1723 /* 1724 * This is the MMU protection handler. It's too big to fit 1725 * in the trap table so I moved it here. It's relatively simple. 1726 * It looks up the page mapping in the page table associated with 1727 * the trapping context. It checks to see if the S/W writable bit 1728 * is set. If so, it sets the H/W write bit, marks the tte modified, 1729 * and enters the mapping into the MMU. Otherwise it does a regular 1730 * data fault. 1731 */ 1732 ICACHE_ALIGN 1733 dmmu_write_fault: 1734 mov TLB_TAG_ACCESS, %g3 1735 sethi %hi(0x1fff), %g6 ! 8K context mask 1736 ldxa [%g3] ASI_DMMU, %g3 ! Get fault addr from Tag Target 1737 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 1738 or %g6, %lo(0x1fff), %g6 1739 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 1740 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1741 and %g3, %g6, %g6 ! Isolate context 1742 1743 inc %g5 ! (0 or -1) -> (1 or 0) 1744 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 1745 ldx [%g4+%g6], %g4 ! Load up our page table. 1746 srlx %g3, STSHIFT, %g6 1747 cmp %g5, 1 1748 bgu,pn %xcc, winfix ! Error! 1749 srlx %g3, PDSHIFT, %g5 1750 and %g6, STMASK, %g6 1751 sll %g6, 3, %g6 1752 1753 and %g5, PDMASK, %g5 1754 sll %g5, 3, %g5 1755 add %g6, %g4, %g4 1756 DLFLUSH(%g4,%g6) 1757 ldxa [%g4] ASI_PHYS_CACHED, %g4 1758 DLFLUSH2(%g6) 1759 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 1760 and %g6, PTMASK, %g6 1761 add %g5, %g4, %g5 1762 brz,pn %g4, winfix ! NULL entry? check somewhere else 1763 nop 1764 1765 ldxa [%g5] ASI_PHYS_CACHED, %g4 1766 sll %g6, 3, %g6 1767 brz,pn %g4, winfix ! NULL entry? check somewhere else 1768 add %g6, %g4, %g6 1769 1: 1770 ldxa [%g6] ASI_PHYS_CACHED, %g4 1771 brgez,pn %g4, winfix ! Entry invalid? Punt 1772 or %g4, SUN4U_TTE_MODIFY|SUN4U_TTE_ACCESS|SUN4U_TTE_W, %g7 ! Update the modified bit 1773 1774 btst SUN4U_TTE_REAL_W|SUN4U_TTE_W, %g4 ! Is it a ref fault? 1775 bz,pn %xcc, winfix ! No -- really fault 1776 #ifdef DEBUG 1777 /* Make sure we don't try to replace a kernel translation */ 1778 /* This should not be necessary */ 1779 sllx %g3, 64-13, %g2 ! Isolate context bits 1780 sethi %hi(KERNBASE), %g5 ! Don't need %lo 1781 brnz,pt %g2, 0f ! Ignore context != 0 1782 set 0x0800000, %g2 ! 8MB 1783 sub %g3, %g5, %g5 1784 cmp %g5, %g2 1785 tlu %xcc, 1; nop 1786 blu,pn %xcc, winfix ! Next insn in delay slot is unimportant 1787 0: 1788 #endif 1789 /* Need to check for and handle large pages. */ 1790 srlx %g4, 61, %g5 ! Isolate the size bits 1791 ldxa [%g0] ASI_DMMU_8KPTR, %g2 ! Load DMMU 8K TSB pointer 1792 andcc %g5, 0x3, %g5 ! 8K? 1793 bnz,pn %icc, winfix ! We punt to the pmap code since we can't handle policy 1794 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 1795 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 1796 membar #StoreLoad 1797 cmp %g4, %g7 1798 bne,pn %xcc, 1b 1799 or %g4, SUN4U_TTE_MODIFY|SUN4U_TTE_ACCESS|SUN4U_TTE_W, %g4 ! Update the modified bit 1800 stx %g1, [%g2] ! Update TSB entry tag 1801 mov SFSR, %g7 1802 stx %g4, [%g2+8] ! Update TSB entry data 1803 nop 1804 1805 #ifdef TRAPSTATS 1806 sethi %hi(_C_LABEL(protfix)), %g1 1807 lduw [%g1+%lo(_C_LABEL(protfix))], %g2 1808 inc %g2 1809 stw %g2, [%g1+%lo(_C_LABEL(protfix))] 1810 #endif 1811 mov DEMAP_PAGE_SECONDARY, %g1 ! Secondary flush 1812 mov DEMAP_PAGE_NUCLEUS, %g5 ! Nucleus flush 1813 stxa %g0, [%g7] ASI_DMMU ! clear out the fault 1814 sllx %g3, (64-13), %g7 ! Need to demap old entry first 1815 andn %g3, 0xfff, %g6 1816 movrz %g7, %g5, %g1 ! Pick one 1817 or %g6, %g1, %g6 1818 membar #Sync 1819 stxa %g6, [%g6] ASI_DMMU_DEMAP ! Do the demap 1820 membar #Sync 1821 1822 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1823 membar #Sync 1824 retry 1825 1826 /* 1827 * Each memory data access fault from a fast access miss handler comes here. 1828 * We will quickly check if this is an original prom mapping before going 1829 * to the generic fault handler 1830 * 1831 * We will assume that %pil is not lost so we won't bother to save it 1832 * unless we're in an interrupt handler. 1833 * 1834 * On entry: 1835 * We are on one of the alternate set of globals 1836 * %g1 = MMU tag target 1837 * %g2 = 8Kptr 1838 * %g3 = TLB TAG ACCESS 1839 * 1840 * On return: 1841 * 1842 */ 1843 ICACHE_ALIGN 1844 data_miss: 1845 #ifdef TRAPSTATS 1846 set _C_LABEL(kdmiss), %g3 1847 set _C_LABEL(udmiss), %g4 1848 rdpr %tl, %g6 1849 dec %g6 1850 movrz %g6, %g4, %g3 1851 lduw [%g3], %g4 1852 inc %g4 1853 stw %g4, [%g3] 1854 #endif 1855 mov TLB_TAG_ACCESS, %g3 ! Get real fault page 1856 sethi %hi(0x1fff), %g6 ! 8K context mask 1857 ldxa [%g3] ASI_DMMU, %g3 ! from tag access register 1858 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 1859 or %g6, %lo(0x1fff), %g6 1860 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 1861 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1862 and %g3, %g6, %g6 ! Isolate context 1863 1864 inc %g5 ! (0 or -1) -> (1 or 0) 1865 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 1866 ldx [%g4+%g6], %g4 ! Load up our page table. 1867 #ifdef DEBUG 1868 /* Make sure we don't try to replace a kernel translation */ 1869 /* This should not be necessary */ 1870 brnz,pt %g6, 1f ! If user context continue miss 1871 sethi %hi(KERNBASE), %g7 ! Don't need %lo 1872 set 0x0800000, %g6 ! 8MB 1873 sub %g3, %g7, %g7 1874 cmp %g7, %g6 1875 tlu %xcc, 1; nop 1876 1: 1877 #endif 1878 srlx %g3, STSHIFT, %g6 1879 cmp %g5, 1 1880 bgu,pn %xcc, winfix ! Error! 1881 srlx %g3, PDSHIFT, %g5 1882 and %g6, STMASK, %g6 1883 1884 sll %g6, 3, %g6 1885 and %g5, PDMASK, %g5 1886 sll %g5, 3, %g5 1887 add %g6, %g4, %g4 1888 ldxa [%g4] ASI_PHYS_CACHED, %g4 1889 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 1890 and %g6, PTMASK, %g6 1891 add %g5, %g4, %g5 1892 brz,pn %g4, data_nfo ! NULL entry? check somewhere else 1893 1894 nop 1895 ldxa [%g5] ASI_PHYS_CACHED, %g4 1896 sll %g6, 3, %g6 1897 brz,pn %g4, data_nfo ! NULL entry? check somewhere else 1898 add %g6, %g4, %g6 1899 1900 1: 1901 ldxa [%g6] ASI_PHYS_CACHED, %g4 1902 brgez,pn %g4, data_nfo ! Entry invalid? Punt 1903 or %g4, SUN4U_TTE_ACCESS, %g7 ! Update the access bit 1904 1905 btst SUN4U_TTE_ACCESS, %g4 ! Need to update access git? 1906 bne,pt %xcc, 1f 1907 nop 1908 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 1909 cmp %g4, %g7 1910 bne,pn %xcc, 1b 1911 or %g4, SUN4U_TTE_ACCESS, %g4 ! Update the access bit 1912 1913 1: 1914 stx %g1, [%g2] ! Update TSB entry tag 1915 stx %g4, [%g2+8] ! Update TSB entry data 1916 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1917 membar #Sync 1918 CLRTT 1919 retry 1920 NOTREACHED 1921 /* 1922 * We had a data miss but did not find a mapping. Insert 1923 * a NFO mapping to satisfy speculative loads and return. 1924 * If this had been a real load, it will re-execute and 1925 * result in a data fault or protection fault rather than 1926 * a TLB miss. We insert an 8K TTE with the valid and NFO 1927 * bits set. All others should zero. The TTE looks like this: 1928 * 1929 * 0x9000000000000000 1930 * 1931 */ 1932 data_nfo: 1933 sethi %hi(0x90000000), %g4 ! V(0x8)|NFO(0x1) 1934 sllx %g4, 32, %g4 1935 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1936 membar #Sync 1937 CLRTT 1938 retry 1939 1940 /* 1941 * Handler for making the trap window shiny clean. 1942 * 1943 * If the store that trapped was to a kernel address, panic. 1944 * 1945 * If the store that trapped was to a user address, stick it in the PCB. 1946 * Since we don't want to force user code to use the standard register 1947 * convention if we don't have to, we will not assume that %fp points to 1948 * anything valid. 1949 * 1950 * On entry: 1951 * We are on one of the alternate set of globals 1952 * %g1 = %tl - 1, tstate[tl-1], scratch - local 1953 * %g2 = %tl - local 1954 * %g3 = MMU tag access - in 1955 * %g4 = %cwp - local 1956 * %g5 = scratch - local 1957 * %g6 = cpcb - local 1958 * %g7 = scratch - local 1959 * 1960 * On return: 1961 * 1962 * NB: remove most of this from main codepath & cleanup I$ 1963 */ 1964 winfault: 1965 mov TLB_TAG_ACCESS, %g3 ! Get real fault page from tag access register 1966 ldxa [%g3] ASI_DMMU, %g3 ! And put it into the non-MMU alternate regs 1967 winfix: 1968 rdpr %tl, %g2 1969 subcc %g2, 1, %g1 1970 ble,pt %icc, datafault ! Don't go below trap level 1 1971 sethi %hi(CPCB), %g6 ! get current pcb 1972 1973 1974 wrpr %g1, 0, %tl ! Pop a trap level 1975 rdpr %tt, %g7 ! Read type of prev. trap 1976 rdpr %tstate, %g4 ! Try to restore prev %cwp if we were executing a restore 1977 andn %g7, 0x3f, %g5 ! window fill traps are all 0b 0000 11xx xxxx 1978 1979 #if 1 1980 cmp %g7, 0x30 ! If we took a datafault just before this trap 1981 bne,pt %icc, winfixfill ! our stack's probably bad so we need to switch somewhere else 1982 nop 1983 1984 !! 1985 !! Double data fault -- bad stack? 1986 !! 1987 wrpr %g2, %tl ! Restore trap level. 1988 sir ! Just issue a reset and don't try to recover. 1989 mov %fp, %l6 ! Save the frame pointer 1990 set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack 1991 add %fp, -CC64FSZ, %sp ! Create a stackframe 1992 wrpr %g0, 15, %pil ! Disable interrupts, too 1993 wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB 1994 wrpr %g0, 7, %cansave ! probably is too, so blow away 1995 ba slowtrap ! all our register windows. 1996 wrpr %g0, 0x101, %tt 1997 #endif 1998 1999 winfixfill: 2000 cmp %g5, 0x0c0 ! so we mask lower bits & compare to 0b 0000 1100 0000 2001 bne,pt %icc, winfixspill ! Dump our trap frame -- we will retry the fill when the page is loaded 2002 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx 2003 2004 !! 2005 !! This was a fill 2006 !! 2007 #ifdef TRAPSTATS 2008 set _C_LABEL(wfill), %g1 2009 lduw [%g1], %g5 2010 inc %g5 2011 stw %g5, [%g1] 2012 #endif 2013 btst TSTATE_PRIV, %g4 ! User mode? 2014 and %g4, CWP, %g5 ! %g4 = %cwp of trap 2015 wrpr %g7, 0, %tt 2016 bz,a,pt %icc, datafault ! We were in user mode -- normal fault 2017 wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consistent 2018 2019 /* 2020 * We're in a pickle here. We were trying to return to user mode 2021 * and the restore of the user window failed, so now we have one valid 2022 * kernel window and a user window state. If we do a TRAP_SETUP() now, 2023 * our kernel window will be considered a user window and cause a 2024 * fault when we try to save it later due to an invalid user address. 2025 * If we return to where we faulted, our window state will not be valid 2026 * and we will fault trying to enter user with our primary context of zero. 2027 * 2028 * What we'll do is arrange to have us return to return_from_trap so we will 2029 * start the whole business over again. But first, switch to a kernel window 2030 * setup. Let's see, canrestore and otherwin are zero. Set WSTATE_KERN and 2031 * make sure we're in kernel context and we're done. 2032 */ 2033 2034 #ifdef TRAPSTATS 2035 set _C_LABEL(kwfill), %g4 2036 lduw [%g4], %g7 2037 inc %g7 2038 stw %g7, [%g4] 2039 #endif 2040 #if 0 /* Need to switch over to new stuff to fix WDR bug */ 2041 wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consistent 2042 wrpr %g2, %g0, %tl ! Restore trap level -- we need to reuse it 2043 set return_from_trap, %g4 ! XXX - need to set %g1 to tstate 2044 set CTX_PRIMARY, %g7 2045 wrpr %g4, 0, %tpc 2046 stxa %g0, [%g7] ASI_DMMU 2047 inc 4, %g4 2048 membar #Sync 2049 flush %g4 ! Isn't this convenient? 2050 wrpr %g0, WSTATE_KERN, %wstate 2051 wrpr %g0, 0, %canrestore ! These should be zero but 2052 wrpr %g0, 0, %otherwin ! clear them just in case 2053 rdpr %ver, %g5 2054 and %g5, CWP, %g5 2055 wrpr %g0, 0, %cleanwin 2056 dec 1, %g5 ! NWINDOWS-1-1 2057 wrpr %g5, 0, %cansave ! Invalidate all windows 2058 ! flushw ! DEBUG 2059 ba,pt %icc, datafault 2060 wrpr %g4, 0, %tnpc 2061 #else 2062 wrpr %g2, %g0, %tl ! Restore trap level 2063 cmp %g2, 3 2064 tne %icc, 1 2065 rdpr %tt, %g5 2066 wrpr %g0, 1, %tl ! Revert to TL==1 XXX what if this wasn't in rft_user? Oh well. 2067 wrpr %g5, %g0, %tt ! Set trap type correctly 2068 /* 2069 * Here we need to implement the beginning of datafault. 2070 * TRAP_SETUP expects to come from either kernel mode or 2071 * user mode with at least one valid register window. It 2072 * will allocate a trap frame, save the out registers, and 2073 * fix the window registers to think we have one user 2074 * register window. 2075 * 2076 * However, under these circumstances we don't have any 2077 * valid register windows, so we need to clean up the window 2078 * registers to prevent garbage from being saved to either 2079 * the user stack or the PCB before calling the datafault 2080 * handler. 2081 * 2082 * We could simply jump to datafault if we could somehow 2083 * make the handler issue a `saved' instruction immediately 2084 * after creating the trapframe. 2085 * 2086 * The following is duplicated from datafault: 2087 */ 2088 #ifdef TRAPS_USE_IG 2089 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals 2090 #else 2091 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals 2092 #endif 2093 wr %g0, ASI_DMMU, %asi ! We need to re-load trap info 2094 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2095 ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first 2096 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2097 stxa %g0, [SFSR] %asi ! Clear out fault now 2098 2099 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2100 saved ! Blow away that one register window we didn't ever use. 2101 ba,a,pt %icc, Ldatafault_internal ! Now we should return directly to user mode 2102 nop 2103 #endif 2104 winfixspill: 2105 bne,a,pt %xcc, datafault ! Was not a spill -- handle it normally 2106 wrpr %g2, 0, %tl ! Restore trap level for now XXXX 2107 2108 !! 2109 !! This was a spill 2110 !! 2111 #if 1 2112 btst TSTATE_PRIV, %g4 ! From user mode? 2113 wrpr %g2, 0, %tl ! We need to load the fault type so we can 2114 rdpr %tt, %g5 ! overwrite the lower trap and get it to the fault handler 2115 wrpr %g1, 0, %tl 2116 wrpr %g5, 0, %tt ! Copy over trap type for the fault handler 2117 and %g4, CWP, %g5 ! find %cwp from trap 2118 be,a,pt %xcc, datafault ! Let's do a regular datafault. When we try a save in datafault we'll 2119 wrpr %g5, 0, %cwp ! return here and write out all dirty windows. 2120 #endif 2121 wrpr %g2, 0, %tl ! Restore trap level for now XXXX 2122 LDPTR [%g6 + %lo(CPCB)], %g6 ! This is in the locked TLB and should not fault 2123 #ifdef TRAPSTATS 2124 set _C_LABEL(wspill), %g7 2125 lduw [%g7], %g5 2126 inc %g5 2127 stw %g5, [%g7] 2128 #endif 2129 2130 /* 2131 * Traverse kernel map to find paddr of cpcb and only us ASI_PHYS_CACHED to 2132 * prevent any faults while saving the windows. BTW if it isn't mapped, we 2133 * will trap and hopefully panic. 2134 */ 2135 2136 ! ba 0f ! DEBUG -- don't use phys addresses 2137 wr %g0, ASI_NUCLEUS, %asi ! In case of problems finding PA 2138 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g1 2139 LDPTR [%g1 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g1 ! Load start of ctxbusy 2140 #ifdef DEBUG 2141 srax %g6, HOLESHIFT, %g7 ! Check for valid address 2142 brz,pt %g7, 1f ! Should be zero or -1 2143 addcc %g7, 1, %g7 ! Make -1 -> 0 2144 tnz %xcc, 1 ! Invalid address??? How did this happen? 2145 1: 2146 #endif 2147 srlx %g6, STSHIFT, %g7 2148 ldx [%g1], %g1 ! Load pointer to kernel_pmap 2149 and %g7, STMASK, %g7 2150 sll %g7, 3, %g7 2151 add %g7, %g1, %g1 2152 DLFLUSH(%g1,%g7) 2153 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 2154 DLFLUSH2(%g7) 2155 2156 srlx %g6, PDSHIFT, %g7 ! Do page directory 2157 and %g7, PDMASK, %g7 2158 sll %g7, 3, %g7 2159 brz,pn %g1, 0f 2160 add %g7, %g1, %g1 2161 DLFLUSH(%g1,%g7) 2162 ldxa [%g1] ASI_PHYS_CACHED, %g1 2163 DLFLUSH2(%g7) 2164 2165 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 2166 and %g7, PTMASK, %g7 2167 brz %g1, 0f 2168 sll %g7, 3, %g7 2169 add %g1, %g7, %g7 2170 DLFLUSH(%g7,%g1) 2171 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 2172 DLFLUSH2(%g1) 2173 brgez %g7, 0f 2174 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 2175 sll %g6, 32-PGSHIFT, %g6 ! And offset 2176 sllx %g7, PGSHIFT+23, %g7 ! There are 23 bits to the left of the PA in the TTE 2177 srl %g6, 32-PGSHIFT, %g6 2178 srax %g7, 23, %g7 2179 or %g7, %g6, %g6 ! Then combine them to form PA 2180 2181 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 2182 0: 2183 /* 2184 * Now save all user windows to cpcb. 2185 */ 2186 #ifdef NOTDEF_DEBUG 2187 add %g6, PCB_NSAVED, %g7 2188 DLFLUSH(%g7,%g5) 2189 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! make sure that pcb_nsaved 2190 DLFLUSH2(%g5) 2191 brz,pt %g7, 1f ! is zero, else 2192 nop 2193 wrpr %g0, 4, %tl 2194 sir ! Force a watchdog 2195 1: 2196 #endif 2197 rdpr %otherwin, %g7 2198 brnz,pt %g7, 1f 2199 rdpr %canrestore, %g5 2200 rdpr %cansave, %g1 2201 add %g5, 1, %g7 ! add the %cwp window to the list to save 2202 ! movrnz %g1, %g5, %g7 ! If we're issuing a save 2203 ! mov %g5, %g7 ! DEBUG 2204 wrpr %g0, 0, %canrestore 2205 wrpr %g7, 0, %otherwin ! Still in user mode -- need to switch to kernel mode 2206 1: 2207 mov %g7, %g1 2208 add %g6, PCB_NSAVED, %g7 2209 DLFLUSH(%g7,%g5) 2210 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Start incrementing pcb_nsaved 2211 DLFLUSH2(%g5) 2212 2213 #ifdef DEBUG 2214 wrpr %g0, 5, %tl 2215 #endif 2216 mov %g6, %g5 2217 brz,pt %g7, winfixsave ! If it's in use, panic 2218 saved ! frob window registers 2219 2220 /* PANIC */ 2221 ! sir ! Force a watchdog 2222 #ifdef DEBUG 2223 wrpr %g2, 0, %tl 2224 #endif 2225 mov %g7, %o2 2226 rdpr %ver, %o1 2227 sethi %hi(2f), %o0 2228 and %o1, CWP, %o1 2229 wrpr %g0, %o1, %cleanwin 2230 dec 1, %o1 2231 wrpr %g0, %o1, %cansave ! kludge away any more window problems 2232 wrpr %g0, 0, %canrestore 2233 wrpr %g0, 0, %otherwin 2234 or %lo(2f), %o0, %o0 2235 wrpr %g0, WSTATE_KERN, %wstate 2236 set PANICSTACK-CC64FSZ-STKB, %sp 2237 ta 1; nop ! This helps out traptrace. 2238 call _C_LABEL(panic) ! This needs to be fixed properly but we should panic here 2239 mov %g1, %o1 2240 NOTREACHED 2241 .data 2242 2: 2243 .asciz "winfault: double invalid window at %p, nsaved=%d" 2244 _ALIGN 2245 .text 2246 3: 2247 saved 2248 save 2249 winfixsave: 2250 stxa %l0, [%g5 + PCB_RW + ( 0*8)] %asi ! Save the window in the pcb, we can schedule other stuff in here 2251 stxa %l1, [%g5 + PCB_RW + ( 1*8)] %asi 2252 stxa %l2, [%g5 + PCB_RW + ( 2*8)] %asi 2253 stxa %l3, [%g5 + PCB_RW + ( 3*8)] %asi 2254 stxa %l4, [%g5 + PCB_RW + ( 4*8)] %asi 2255 stxa %l5, [%g5 + PCB_RW + ( 5*8)] %asi 2256 stxa %l6, [%g5 + PCB_RW + ( 6*8)] %asi 2257 stxa %l7, [%g5 + PCB_RW + ( 7*8)] %asi 2258 2259 stxa %i0, [%g5 + PCB_RW + ( 8*8)] %asi 2260 stxa %i1, [%g5 + PCB_RW + ( 9*8)] %asi 2261 stxa %i2, [%g5 + PCB_RW + (10*8)] %asi 2262 stxa %i3, [%g5 + PCB_RW + (11*8)] %asi 2263 stxa %i4, [%g5 + PCB_RW + (12*8)] %asi 2264 stxa %i5, [%g5 + PCB_RW + (13*8)] %asi 2265 stxa %i6, [%g5 + PCB_RW + (14*8)] %asi 2266 stxa %i7, [%g5 + PCB_RW + (15*8)] %asi 2267 2268 ! rdpr %otherwin, %g1 ! Check to see if we's done 2269 dec %g1 2270 wrpr %g0, 7, %cleanwin ! BUGBUG -- we should not hardcode this, but I have no spare globals 2271 inc 16*8, %g5 ! Move to next window 2272 inc %g7 ! inc pcb_nsaved 2273 brnz,pt %g1, 3b 2274 stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out 2275 2276 /* fix up pcb fields */ 2277 stba %g7, [%g6 + PCB_NSAVED] %asi ! cpcb->pcb_nsaved = n 2278 #if 0 2279 mov %g7, %g5 ! fixup window registers 2280 5: 2281 dec %g5 2282 brgz,a,pt %g5, 5b 2283 restore 2284 #ifdef NOT_DEBUG 2285 rdpr %wstate, %g5 ! DEBUG 2286 wrpr %g0, WSTATE_KERN, %wstate ! DEBUG 2287 wrpr %g0, 4, %tl 2288 rdpr %cansave, %g7 2289 rdpr %canrestore, %g6 2290 flushw ! DEBUG 2291 wrpr %g2, 0, %tl 2292 wrpr %g5, 0, %wstate ! DEBUG 2293 #endif 2294 #else 2295 /* 2296 * We just issued a bunch of saves, so %cansave is now 0, 2297 * probably (if we were doing a flushw then we may have 2298 * come in with only partially full register windows and 2299 * it may not be 0). 2300 * 2301 * %g7 contains the count of the windows we just finished 2302 * saving. 2303 * 2304 * What we need to do now is move some of the windows from 2305 * %canrestore to %cansave. What we should do is take 2306 * min(%canrestore, %g7) and move that over to %cansave. 2307 * 2308 * %g7 is the number of windows we flushed, so we should 2309 * use that as a base. Clear out %otherwin, set %cansave 2310 * to min(%g7, NWINDOWS - 2), set %cleanwin to %canrestore 2311 * + %cansave and the rest follows: 2312 * 2313 * %otherwin = 0 2314 * %cansave = NWINDOWS - 2 - %canrestore 2315 */ 2316 wrpr %g0, 0, %otherwin 2317 rdpr %canrestore, %g1 2318 sub %g1, %g7, %g1 ! Calculate %canrestore - %g7 2319 movrlz %g1, %g0, %g1 ! Clamp at zero 2320 wrpr %g1, 0, %canrestore ! This is the new canrestore 2321 rdpr %ver, %g5 2322 and %g5, CWP, %g5 ! NWINDOWS-1 2323 dec %g5 ! NWINDOWS-2 2324 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel 2325 sub %g5, %g1, %g5 ! NWINDOWS-2-%canrestore 2326 wrpr %g5, 0, %cansave 2327 #ifdef NOT_DEBUG 2328 rdpr %wstate, %g5 ! DEBUG 2329 wrpr %g0, WSTATE_KERN, %wstate ! DEBUG 2330 wrpr %g0, 4, %tl 2331 flushw ! DEBUG 2332 wrpr %g2, 0, %tl 2333 wrpr %g5, 0, %wstate ! DEBUG 2334 #endif 2335 #endif 2336 2337 #ifdef NOTDEF_DEBUG 2338 set panicstack-CC64FSZ, %g1 2339 save %g1, 0, %sp 2340 GLOBTOLOC 2341 rdpr %wstate, %l0 2342 wrpr %g0, WSTATE_KERN, %wstate 2343 set 8f, %o0 2344 mov %g7, %o1 2345 call printf 2346 mov %g5, %o2 2347 wrpr %l0, 0, %wstate 2348 LOCTOGLOB 2349 restore 2350 .data 2351 8: 2352 .asciz "winfix: spill fixup\n" 2353 _ALIGN 2354 .text 2355 #endif 2356 ! rdpr %tl, %g2 ! DEBUG DEBUG -- did we trap somewhere? 2357 sub %g2, 1, %g1 2358 rdpr %tt, %g2 2359 wrpr %g1, 0, %tl ! We will not attempt to re-execute the spill, so dump our trap frame permanently 2360 wrpr %g2, 0, %tt ! Move trap type from fault frame here, overwriting spill 2361 2362 /* Did we save a user or kernel window ? */ 2363 ! srax %g3, 48, %g5 ! User or kernel store? (TAG TARGET) 2364 sllx %g3, (64-13), %g5 ! User or kernel store? (TAG ACCESS) 2365 sethi %hi(dcache_size), %g7 2366 ld [%g7 + %lo(dcache_size)], %g7 2367 sethi %hi(dcache_line_size), %g6 2368 ld [%g6 + %lo(dcache_line_size)], %g6 2369 brnz,pt %g5, 1f ! User fault -- save windows to pcb 2370 sub %g7, %g6, %g7 2371 2372 and %g4, CWP, %g4 ! %g4 = %cwp of trap 2373 wrpr %g4, 0, %cwp ! Kernel fault -- restore %cwp and force and trap to debugger 2374 !! 2375 !! Here we managed to fault trying to access a kernel window 2376 !! This is a bug. Switch to the interrupt stack if we aren't 2377 !! there already and then trap into the debugger or panic. 2378 !! 2379 sethi %hi(EINTSTACK-BIAS), %g6 2380 btst 1, %sp 2381 bnz,pt %icc, 0f 2382 mov %sp, %g1 2383 add %sp, -BIAS, %g1 2384 0: 2385 or %g6, %lo(EINTSTACK-BIAS), %g6 2386 set (EINTSTACK-INTSTACK), %g7 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses 2387 sub %g6, %g1, %g2 ! Determine if we need to switch to intr stack or not 2388 dec %g7 ! Make it into a mask 2389 andncc %g2, %g7, %g0 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ 2390 movz %xcc, %g1, %g6 ! Stay on interrupt stack? 2391 add %g6, -CCFSZ, %g6 ! Allocate a stack frame 2392 mov %sp, %l6 ! XXXXX Save old stack pointer 2393 mov %g6, %sp 2394 ta 1; nop ! Enter debugger 2395 NOTREACHED 2396 1: 2397 #if 1 2398 /* Now we need to blast away the D$ to make sure we're in sync */ 2399 stxa %g0, [%g7] ASI_DCACHE_TAG 2400 brnz,pt %g7, 1b 2401 sub %g7, %g6, %g7 2402 #endif 2403 2404 #ifdef NOTDEF_DEBUG 2405 set panicstack-CC64FSZ, %g5 2406 save %g5, 0, %sp 2407 GLOBTOLOC 2408 rdpr %wstate, %l0 2409 wrpr %g0, WSTATE_KERN, %wstate 2410 set 8f, %o0 2411 call printf 2412 mov %fp, %o1 2413 wrpr %l0, 0, %wstate 2414 LOCTOGLOB 2415 restore 2416 .data 2417 8: 2418 .asciz "winfix: kernel spill retry\n" 2419 _ALIGN 2420 .text 2421 #endif 2422 #ifdef TRAPSTATS 2423 set _C_LABEL(wspillskip), %g4 2424 lduw [%g4], %g5 2425 inc %g5 2426 stw %g5, [%g4] 2427 #endif 2428 /* 2429 * If we had WSTATE_KERN then we had at least one valid kernel window. 2430 * We should re-execute the trapping save. 2431 */ 2432 rdpr %wstate, %g3 2433 mov %g3, %g3 2434 cmp %g3, WSTATE_KERN 2435 bne,pt %icc, 1f 2436 nop 2437 retry ! Now we can complete the save 2438 1: 2439 /* 2440 * Since we had a WSTATE_USER, we had no valid kernel windows. This should 2441 * only happen inside TRAP_SETUP or INTR_SETUP. Emulate 2442 * the instruction, clean up the register windows, then done. 2443 */ 2444 rdpr %cwp, %g1 2445 inc %g1 2446 rdpr %tstate, %g2 2447 wrpr %g1, %cwp 2448 andn %g2, CWP, %g2 2449 wrpr %g1, %g2, %tstate 2450 #ifdef TRAPS_USE_IG 2451 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 2452 #else 2453 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate 2454 #endif 2455 mov %g6, %sp 2456 done 2457 2458 /* 2459 * Each memory data access fault, from user or kernel mode, 2460 * comes here. 2461 * 2462 * We will assume that %pil is not lost so we won't bother to save it 2463 * unless we're in an interrupt handler. 2464 * 2465 * On entry: 2466 * We are on one of the alternate set of globals 2467 * %g1 = MMU tag target 2468 * %g2 = %tl 2469 * 2470 * On return: 2471 * 2472 */ 2473 datafault: 2474 #ifdef TRAPS_USE_IG 2475 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals 2476 #else 2477 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals 2478 #endif 2479 wr %g0, ASI_DMMU, %asi ! We need to re-load trap info 2480 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2481 ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first 2482 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2483 stxa %g0, [SFSR] %asi ! Clear out fault now 2484 2485 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2486 Ldatafault_internal: 2487 INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) 2488 ! ldx [%sp + CC64FSZ + STKB + TF_FAULT], %g1 ! DEBUG make sure this has not changed 2489 mov %g1, %o0 ! Move these to the out regs so we can save the globals 2490 mov %g2, %o4 2491 mov %g3, %o5 2492 2493 ldxa [%g0] ASI_AFAR, %o2 ! get async fault address 2494 ldxa [%g0] ASI_AFSR, %o3 ! get async fault status 2495 mov -1, %g7 2496 stxa %g7, [%g0] ASI_AFSR ! And clear this out, too 2497 2498 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 2499 2500 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 2501 rdpr %tt, %o1 ! find out what trap brought us here 2502 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 2503 rdpr %tstate, %g1 2504 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) 2505 rdpr %tpc, %g2 2506 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 2507 rdpr %tnpc, %g3 2508 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 2509 mov %g2, %o7 ! Make the fault address look like the return address 2510 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 2511 rd %y, %g5 ! save y 2512 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 2513 2514 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] 2515 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc 2516 stx %g2, [%sp + CC64FSZ + STKB + TF_PC] ! set tf.tf_npc 2517 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] 2518 2519 rdpr %pil, %g4 2520 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] 2521 stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2522 2523 #if 1 2524 rdpr %tl, %g7 2525 dec %g7 2526 movrlz %g7, %g0, %g7 2527 wrpr %g0, %g7, %tl ! Revert to kernel mode 2528 #else 2529 wrpr %g0, 0, %tl ! Revert to kernel mode 2530 #endif 2531 /* Finish stackframe, call C trap handler */ 2532 flushw ! Get this clean so we won't take any more user faults 2533 #ifdef NOTDEF_DEBUG 2534 set CPCB, %o7 2535 LDPTR [%o7], %o7 2536 ldub [%o7 + PCB_NSAVED], %o7 2537 brz,pt %o7, 2f 2538 nop 2539 save %sp, -CC64FSZ, %sp 2540 set 1f, %o0 2541 call printf 2542 mov %i7, %o1 2543 ta 1; nop 2544 restore 2545 .data 2546 1: .asciz "datafault: nsaved = %d\n" 2547 _ALIGN 2548 .text 2549 2: 2550 #endif 2551 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2552 !! In our case we need to clear it before calling any C-code 2553 clr %g4 2554 2555 /* 2556 * Right now the registers have the following values: 2557 * 2558 * %o0 -- MMU_TAG_ACCESS 2559 * %o1 -- TT 2560 * %o2 -- afar 2561 * %o3 -- afsr 2562 * %o4 -- sfar 2563 * %o5 -- sfsr 2564 */ 2565 2566 cmp %o1, T_DATA_ERROR 2567 st %g5, [%sp + CC64FSZ + STKB + TF_Y] 2568 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2569 be,pn %icc, data_error 2570 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2571 2572 mov %o0, %o3 ! (argument: trap address) 2573 mov %g2, %o2 ! (argument: trap pc) 2574 call _C_LABEL(data_access_fault) ! data_access_fault(&tf, type, 2575 ! pc, addr, sfva, sfsr) 2576 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2577 2578 data_recover: 2579 #ifdef TRAPSTATS 2580 set _C_LABEL(uintrcnt), %g1 2581 stw %g0, [%g1] 2582 set _C_LABEL(iveccnt), %g1 2583 stw %g0, [%g1] 2584 #endif 2585 wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts 2586 b return_from_trap ! go return 2587 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 2588 NOTREACHED 2589 2590 data_error: 2591 call _C_LABEL(data_access_error) ! data_access_error(&tf, type, 2592 ! afva, afsr, sfva, sfsr) 2593 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2594 ba data_recover 2595 nop 2596 NOTREACHED 2597 2598 /* 2599 * Each memory instruction access fault from a fast access handler comes here. 2600 * We will quickly check if this is an original prom mapping before going 2601 * to the generic fault handler 2602 * 2603 * We will assume that %pil is not lost so we won't bother to save it 2604 * unless we're in an interrupt handler. 2605 * 2606 * On entry: 2607 * We are on one of the alternate set of globals 2608 * %g1 = MMU tag target 2609 * %g2 = TSB entry ptr 2610 * %g3 = TLB Tag Access 2611 * 2612 * On return: 2613 * 2614 */ 2615 2616 ICACHE_ALIGN 2617 instr_miss: 2618 #ifdef TRAPSTATS 2619 set _C_LABEL(ktmiss), %g3 2620 set _C_LABEL(utmiss), %g4 2621 rdpr %tl, %g6 2622 dec %g6 2623 movrz %g6, %g4, %g3 2624 lduw [%g3], %g4 2625 inc %g4 2626 stw %g4, [%g3] 2627 #endif 2628 mov TLB_TAG_ACCESS, %g3 ! Get real fault page 2629 sethi %hi(0x1fff), %g7 ! 8K context mask 2630 ldxa [%g3] ASI_IMMU, %g3 ! from tag access register 2631 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 2632 or %g7, %lo(0x1fff), %g7 2633 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 2634 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2635 and %g3, %g7, %g6 ! Isolate context 2636 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2637 inc %g5 ! (0 or -1) -> (1 or 0) 2638 ldx [%g4+%g6], %g4 ! Load up our page table. 2639 #ifdef DEBUG 2640 /* Make sure we don't try to replace a kernel translation */ 2641 /* This should not be necessary */ 2642 brnz,pt %g6, 1f ! If user context continue miss 2643 sethi %hi(KERNBASE), %g7 ! Don't need %lo 2644 set 0x0800000, %g6 ! 8MB 2645 sub %g3, %g7, %g7 2646 cmp %g7, %g6 2647 tlu %xcc, 1; nop 2648 1: 2649 #endif 2650 srlx %g3, STSHIFT, %g6 2651 cmp %g5, 1 2652 bgu,pn %xcc, textfault ! Error! 2653 srlx %g3, PDSHIFT, %g5 2654 and %g6, STMASK, %g6 2655 sll %g6, 3, %g6 2656 and %g5, PDMASK, %g5 2657 nop 2658 2659 sll %g5, 3, %g5 2660 add %g6, %g4, %g4 2661 ldxa [%g4] ASI_PHYS_CACHED, %g4 2662 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2663 and %g6, PTMASK, %g6 2664 add %g5, %g4, %g5 2665 brz,pn %g4, textfault ! NULL entry? check somewhere else 2666 nop 2667 2668 ldxa [%g5] ASI_PHYS_CACHED, %g4 2669 sll %g6, 3, %g6 2670 brz,pn %g4, textfault ! NULL entry? check somewhere else 2671 add %g6, %g4, %g6 2672 1: 2673 ldxa [%g6] ASI_PHYS_CACHED, %g4 2674 brgez,pn %g4, textfault 2675 nop 2676 2677 /* Check if it's an executable mapping. */ 2678 andcc %g4, SUN4U_TTE_EXEC, %g0 2679 bz,pn %xcc, textfault 2680 nop 2681 2682 or %g4, SUN4U_TTE_ACCESS, %g7 ! Update accessed bit 2683 btst SUN4U_TTE_ACCESS, %g4 ! Need to update access git? 2684 bne,pt %xcc, 1f 2685 nop 2686 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and store it 2687 cmp %g4, %g7 2688 bne,pn %xcc, 1b 2689 or %g4, SUN4U_TTE_ACCESS, %g4 ! Update accessed bit 2690 1: 2691 stx %g1, [%g2] ! Update TSB entry tag 2692 stx %g4, [%g2+8] ! Update TSB entry data 2693 stxa %g4, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 2694 membar #Sync 2695 CLRTT 2696 retry 2697 NOTREACHED 2698 !! 2699 !! Check our prom mappings -- temporary 2700 !! 2701 2702 /* 2703 * Each memory text access fault, from user or kernel mode, 2704 * comes here. 2705 * 2706 * We will assume that %pil is not lost so we won't bother to save it 2707 * unless we're in an interrupt handler. 2708 * 2709 * On entry: 2710 * We are on one of the alternate set of globals 2711 * %g1 = MMU tag target 2712 * %g2 = %tl 2713 * %g3 = %tl - 1 2714 * 2715 * On return: 2716 * 2717 */ 2718 2719 textfault: 2720 #ifdef TRAPS_USE_IG 2721 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals 2722 #else 2723 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals 2724 #endif 2725 wr %g0, ASI_IMMU, %asi 2726 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2727 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2728 membar #LoadStore 2729 stxa %g0, [SFSR] %asi ! Clear out old info 2730 2731 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2732 INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) 2733 2734 mov %g3, %o3 2735 2736 wrpr %g0, PSTATE_KERN, %pstate ! Switch to normal globals 2737 ldxa [%g0] ASI_AFSR, %o4 ! get async fault status 2738 ldxa [%g0] ASI_AFAR, %o5 ! get async fault address 2739 mov -1, %o0 2740 stxa %o0, [%g0] ASI_AFSR ! Clear this out 2741 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 2742 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 2743 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) 2744 rdpr %tt, %o1 ! Find out what caused this trap 2745 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 2746 rdpr %tstate, %g1 2747 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 2748 rdpr %tpc, %o2 ! sync virt addr; must be read first 2749 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 2750 rdpr %tnpc, %g3 2751 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 2752 rd %y, %g5 ! save y 2753 2754 /* Finish stackframe, call C trap handler */ 2755 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc 2756 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] ! debug 2757 2758 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 2759 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] ! set tf.tf_npc 2760 2761 rdpr %pil, %g4 2762 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] 2763 stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2764 2765 rdpr %tl, %g7 2766 dec %g7 2767 movrlz %g7, %g0, %g7 2768 wrpr %g0, %g7, %tl ! Revert to kernel mode 2769 2770 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2771 flushw ! Get rid of any user windows so we don't deadlock 2772 2773 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2774 !! In our case we need to clear it before calling any C-code 2775 clr %g4 2776 2777 /* Use trap type to see what handler to call */ 2778 cmp %o1, T_INST_ERROR 2779 be,pn %xcc, text_error 2780 st %g5, [%sp + CC64FSZ + STKB + TF_Y] ! set tf.tf_y 2781 2782 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2783 call _C_LABEL(text_access_fault) ! mem_access_fault(&tf, type, pc, sfsr) 2784 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2785 text_recover: 2786 wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts 2787 b return_from_trap ! go return 2788 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 2789 NOTREACHED 2790 2791 text_error: 2792 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2793 call _C_LABEL(text_access_error) ! mem_access_fault(&tfm type, sfva [pc], sfsr, 2794 ! afva, afsr); 2795 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2796 ba text_recover 2797 nop 2798 NOTREACHED 2799 2800 #ifdef SUN4V 2801 2802 /* 2803 * Traps for sun4v. 2804 */ 2805 2806 sun4v_dtsb_miss: 2807 GET_MMFSA %g1 ! MMU Fault status area 2808 add %g1, 0x48, %g3 2809 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 2810 add %g1, 0x50, %g6 2811 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2812 2813 GET_CTXBUSY %g4 2814 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2815 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 2816 2817 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2818 brz,pt %g5, 0f ! Should be zero or -1 2819 inc %g5 ! Make -1 -> 0 2820 brnz,pn %g5, sun4v_datatrap ! Error! In hole! 2821 0: 2822 srlx %g3, STSHIFT, %g6 2823 and %g6, STMASK, %g6 ! Index into pm_segs 2824 sll %g6, 3, %g6 2825 add %g4, %g6, %g4 2826 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 2827 srlx %g3, PDSHIFT, %g6 2828 and %g6, PDMASK, %g6 2829 sll %g6, 3, %g6 2830 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 2831 add %g4, %g6, %g4 2832 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 2833 2834 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2835 and %g6, PTMASK, %g6 2836 sll %g6, 3, %g6 2837 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 2838 add %g4, %g6, %g6 2839 1: 2840 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 2841 brgez,pn %g4, sun4v_datatrap ! Entry invalid? Punt 2842 or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit 2843 2844 btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? 2845 bne,pt %xcc, 2f 2846 nop 2847 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 2848 cmp %g4, %g7 2849 bne,pn %xcc, 1b 2850 or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the access bit 2851 2: 2852 GET_TSB_DMMU %g2 2853 2854 /* Construct TSB tag word. */ 2855 add %g1, 0x50, %g6 2856 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2857 mov %g3, %g1 ! Data fault address 2858 srlx %g1, 22, %g1 ! 63..22 of virt addr 2859 sllx %g6, 48, %g6 ! context_id in 63..48 2860 or %g1, %g6, %g1 ! construct TTE tag 2861 srlx %g3, PTSHIFT, %g3 2862 sethi %hi(_C_LABEL(tsbsize)), %g5 2863 mov 512, %g6 2864 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 2865 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 2866 sub %g5, 1, %g5 ! TSBENTS -> offset 2867 and %g3, %g5, %g3 ! mask out TTE index 2868 sllx %g3, 4, %g3 ! TTE size is 16 bytes 2869 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 2870 2871 membar #StoreStore 2872 2873 STPTR %g4, [%g2 + 8] ! store TTE data 2874 STPTR %g1, [%g2] ! store TTE tag 2875 2876 retry 2877 NOTREACHED 2878 2879 sun4v_tl1_dtsb_miss: 2880 GET_MMFSA %g1 ! MMU Fault status area 2881 add %g1, 0x48, %g3 2882 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 2883 add %g1, 0x50, %g6 2884 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2885 2886 GET_CTXBUSY %g4 2887 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2888 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 2889 2890 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2891 brz,pt %g5, 0f ! Should be zero or -1 2892 inc %g5 ! Make -1 -> 0 2893 brnz,pn %g5, sun4v_tl1_ptbl_miss ! Error! In hole! 2894 0: 2895 srlx %g3, STSHIFT, %g6 2896 and %g6, STMASK, %g6 ! Index into pm_segs 2897 sll %g6, 3, %g6 2898 add %g4, %g6, %g4 2899 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 2900 srlx %g3, PDSHIFT, %g6 2901 and %g6, PDMASK, %g6 2902 sll %g6, 3, %g6 2903 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 2904 add %g4, %g6, %g4 2905 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 2906 2907 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2908 and %g6, PTMASK, %g6 2909 sll %g6, 3, %g6 2910 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 2911 add %g4, %g6, %g6 2912 1: 2913 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 2914 brgez,pn %g4, sun4v_tl1_ptbl_miss ! Entry invalid? Punt 2915 or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit 2916 2917 btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? 2918 bne,pt %xcc, 2f 2919 nop 2920 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 2921 cmp %g4, %g7 2922 bne,pn %xcc, 1b 2923 or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the access bit 2924 2: 2925 GET_TSB_DMMU %g2 2926 2927 /* Construct TSB tag word. */ 2928 add %g1, 0x50, %g6 2929 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2930 mov %g3, %g1 ! Data fault address 2931 srlx %g1, 22, %g1 ! 63..22 of virt addr 2932 sllx %g6, 48, %g6 ! context_id in 63..48 2933 or %g1, %g6, %g1 ! construct TTE tag 2934 srlx %g3, PTSHIFT, %g3 2935 sethi %hi(_C_LABEL(tsbsize)), %g5 2936 mov 512, %g6 2937 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 2938 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 2939 sub %g5, 1, %g5 ! TSBENTS -> offset 2940 and %g3, %g5, %g3 ! mask out TTE index 2941 sllx %g3, 4, %g3 ! TTE size is 16 bytes 2942 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 2943 2944 membar #StoreStore 2945 2946 STPTR %g4, [%g2 + 8] ! store TTE data 2947 STPTR %g1, [%g2] ! store TTE tag 2948 2949 retry 2950 NOTREACHED 2951 2952 sun4v_datatrap: 2953 GET_MMFSA %g3 ! MMU Fault status area 2954 add %g3, 0x48, %g1 2955 LDPTRA [%g1] ASI_PHYS_CACHED, %g1 ! Data fault address 2956 add %g3, 0x50, %g2 2957 LDPTRA [%g2] ASI_PHYS_CACHED, %g2 ! Data fault context 2958 2959 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2960 or %g1, %g2, %o3 2961 mov %g1, %o4 2962 2963 rdpr %tt, %g4 2964 rdpr %tstate, %g1 2965 rdpr %tpc, %g2 2966 rdpr %tnpc, %g3 2967 2968 stx %g1, [%sp + CC64FSZ + BIAS + TF_TSTATE] 2969 mov %g4, %o1 ! (type) 2970 stx %g2, [%sp + CC64FSZ + BIAS + TF_PC] 2971 rd %y, %g5 2972 stx %g3, [%sp + CC64FSZ + BIAS + TF_NPC] 2973 st %g5, [%sp + CC64FSZ + BIAS + TF_Y] 2974 mov %g2, %o2 ! (pc) 2975 sth %o1, [%sp + CC64FSZ + BIAS + TF_TT]! debug 2976 2977 cmp %o1, T_FDMMU_PROT 2978 bne,pn %icc, 1f 2979 mov SFSR_FV, %o5 2980 or %o5, SFSR_W, %o5 2981 2982 1: 2983 NORMAL_GLOBALS_SUN4V 2984 2985 stx %g1, [%sp + CC64FSZ + BIAS + TF_G + (1*8)] 2986 stx %g2, [%sp + CC64FSZ + BIAS + TF_G + (2*8)] 2987 add %sp, CC64FSZ + BIAS, %o0 ! (&tf) 2988 stx %g3, [%sp + CC64FSZ + BIAS + TF_G + (3*8)] 2989 stx %g4, [%sp + CC64FSZ + BIAS + TF_G + (4*8)] 2990 stx %g5, [%sp + CC64FSZ + BIAS + TF_G + (5*8)] 2991 rdpr %pil, %g5 2992 stx %g6, [%sp + CC64FSZ + BIAS + TF_G + (6*8)] 2993 stx %g7, [%sp + CC64FSZ + BIAS + TF_G + (7*8)] 2994 stb %g5, [%sp + CC64FSZ + BIAS + TF_PIL] 2995 stb %g5, [%sp + CC64FSZ + BIAS + TF_OLDPIL] 2996 2997 /* 2998 * Phew, ready to enable traps and call C code. 2999 */ 3000 wrpr %g0, 0, %tl 3001 3002 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 3003 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 3004 call _C_LABEL(data_access_fault) ! data_acces_fault(tf, type, ...) 3005 nop 3006 3007 ba,a,pt %icc, return_from_trap 3008 nop 3009 NOTREACHED 3010 3011 sun4v_tl0_dtsb_prot: 3012 GET_MMFSA %g1 ! MMU Fault status area 3013 add %g1, 0x48, %g3 3014 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 3015 add %g1, 0x50, %g6 3016 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3017 3018 GET_CTXBUSY %g4 3019 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 3020 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 3021 3022 srax %g3, HOLESHIFT, %g5 ! Check for valid address 3023 brz,pt %g5, 0f ! Should be zero or -1 3024 inc %g5 ! Make -1 -> 0 3025 brnz,pn %g5, sun4v_datatrap ! Error! In hole! 3026 0: 3027 srlx %g3, STSHIFT, %g6 3028 and %g6, STMASK, %g6 ! Index into pm_segs 3029 sll %g6, 3, %g6 3030 add %g4, %g6, %g4 3031 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 3032 3033 srlx %g3, PDSHIFT, %g6 3034 and %g6, PDMASK, %g6 3035 sll %g6, 3, %g6 3036 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 3037 add %g4, %g6, %g4 3038 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 3039 3040 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 3041 and %g6, PTMASK, %g6 3042 sll %g6, 3, %g6 3043 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 3044 add %g4, %g6, %g6 3045 1: 3046 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 3047 brgez,pn %g4, sun4v_datatrap ! Entry invalid? Punt 3048 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g7 ! Update the modified bit 3049 3050 # btst SUN4V_TLB_REAL_W|SUN4V_TLB_W, %g4 ! Is it a ref fault? 3051 mov 1, %g2 3052 sllx %g2, 61, %g2 ! %g2 is now SUN4V_TLB_REAL_W 3053 or %g2, SUN4V_TLB_W, %g2 3054 btst %g2, %g4 3055 bz,pn %xcc, sun4v_datatrap ! No -- really fault 3056 nop 3057 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 3058 cmp %g4, %g7 3059 bne,pn %xcc, 1b 3060 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g4 ! Update the modified bit 3061 2: 3062 GET_TSB_DMMU %g2 3063 3064 mov %g1, %g7 ! save MMFSA 3065 3066 /* Construct TSB tag word. */ 3067 add %g1, 0x50, %g6 3068 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3069 mov %g3, %g1 ! Data fault address 3070 srlx %g1, 22, %g1 ! 63..22 of virt addr 3071 sllx %g6, 48, %g6 ! context_id in 63..48 3072 or %g1, %g6, %g1 ! construct TTE tag 3073 3074 srlx %g3, PTSHIFT, %g3 3075 sethi %hi(_C_LABEL(tsbsize)), %g5 3076 mov 512, %g6 3077 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 3078 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 3079 sub %g5, 1, %g5 ! TSBENTS -> offset 3080 and %g3, %g5, %g3 ! mask out TTE index 3081 sllx %g3, 4, %g3 ! TTE size is 16 bytes 3082 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 3083 3084 membar #StoreStore 3085 3086 STPTR %g4, [%g2 + 8] ! store TTE data 3087 STPTR %g1, [%g2] ! store TTE tag 3088 3089 mov %o0, %g1 3090 mov %o1, %g2 3091 mov %o2, %g3 3092 3093 add %g7, 0x48, %o0 3094 ldxa [%o0] ASI_PHYS_CACHED, %o0 ! Data fault address 3095 add %g7, 0x50, %o1 3096 ldxa [%o1] ASI_PHYS_CACHED, %o1 ! Data fault context 3097 mov MAP_DTLB, %o2 3098 ta ST_MMU_UNMAP_ADDR 3099 3100 mov %g1, %o0 3101 mov %g2, %o1 3102 mov %g3, %o2 3103 3104 retry 3105 NOTREACHED 3106 3107 sun4v_tl0_itsb_miss: 3108 GET_MMFSA %g1 ! MMU Fault status area 3109 add %g1, 0x8, %g3 3110 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Instruction fault address 3111 add %g1, 0x10, %g6 3112 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3113 3114 GET_CTXBUSY %g4 3115 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 3116 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 3117 3118 srax %g3, HOLESHIFT, %g5 ! Check for valid address 3119 brz,pt %g5, 0f ! Should be zero or -1 3120 inc %g5 ! Make -1 -> 0 3121 brnz,pn %g5, sun4v_texttrap ! Error! In hole! 3122 0: 3123 srlx %g3, STSHIFT, %g6 3124 and %g6, STMASK, %g6 ! Index into pm_segs 3125 sll %g6, 3, %g6 3126 add %g4, %g6, %g4 3127 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 3128 3129 srlx %g3, PDSHIFT, %g6 3130 and %g6, PDMASK, %g6 3131 sll %g6, 3, %g6 3132 brz,pn %g4, sun4v_texttrap ! NULL entry? check somewhere else 3133 add %g4, %g6, %g4 3134 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 3135 3136 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 3137 and %g6, PTMASK, %g6 3138 sll %g6, 3, %g6 3139 brz,pn %g4, sun4v_texttrap ! NULL entry? check somewhere else 3140 add %g4, %g6, %g6 3141 1: 3142 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 3143 brgez,pn %g4, sun4v_texttrap ! Entry invalid? Punt 3144 or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit 3145 3146 btst SUN4V_TLB_EXEC, %g4 ! Need to update exec bit? 3147 bz,pn %xcc, sun4v_texttrap 3148 nop 3149 btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? 3150 bne,pt %xcc, 2f 3151 nop 3152 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 3153 cmp %g4, %g7 3154 bne,pn %xcc, 1b 3155 or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the modified bit 3156 2: 3157 GET_TSB_DMMU %g2 3158 3159 mov %g1, %g7 3160 /* Construct TSB tag word. */ 3161 add %g1, 0x10, %g6 3162 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Instruction fault context 3163 mov %g3, %g1 ! Instruction fault address 3164 srlx %g1, 22, %g1 ! 63..22 of virt addr 3165 sllx %g6, 48, %g6 ! context_id in 63..48 3166 or %g1, %g6, %g1 ! construct TTE tag 3167 3168 srlx %g3, PTSHIFT, %g3 3169 sethi %hi(_C_LABEL(tsbsize)), %g5 3170 mov 512, %g6 3171 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 3172 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 3173 sub %g5, 1, %g5 ! TSBENTS -> offset 3174 and %g3, %g5, %g3 ! mask out TTE index 3175 sllx %g3, 4, %g3 ! TTE size is 16 bytes 3176 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu (FIXME ci_tsb_immu?) 3177 3178 membar #StoreStore 3179 STPTR %g4, [%g2 + 8] ! store TTE data 3180 stx %g1, [%g2] ! store TTE tag 3181 3182 retry 3183 NOTREACHED 3184 3185 sun4v_texttrap: 3186 GET_MMFSA %g3 ! MMU Fault status area 3187 add %g3, 0x08, %g1 3188 LDPTRA [%g1] ASI_PHYS_CACHED, %g1 ! Instruction fault address 3189 add %g3, 0x10, %g2 3190 LDPTRA [%g2] ASI_PHYS_CACHED, %g2 ! Instruction fault context 3191 3192 TRAP_SETUP(-CC64FSZ-TF_SIZE) 3193 3194 or %g1, %g2, %o2 3195 clr %o3 3196 3197 rdpr %tt, %g4 3198 rdpr %tstate, %g1 3199 rdpr %tpc, %g2 3200 rdpr %tnpc, %g3 3201 3202 stx %g1, [%sp + CC64FSZ + BIAS + TF_TSTATE] 3203 mov %g4, %o1 ! (type) 3204 stx %g2, [%sp + CC64FSZ + BIAS + TF_PC] 3205 rd %y, %g5 3206 stx %g3, [%sp + CC64FSZ + BIAS + TF_NPC] 3207 st %g5, [%sp + CC64FSZ + BIAS + TF_Y] 3208 sth %o1, [%sp + CC64FSZ + BIAS + TF_TT]! debug 3209 3210 ! Get back to normal globals 3211 wrpr %g0, PSTATE_KERN, %pstate 3212 NORMAL_GLOBALS_SUN4V 3213 3214 stx %g1, [%sp + CC64FSZ + BIAS + TF_G + (1*8)] 3215 stx %g2, [%sp + CC64FSZ + BIAS + TF_G + (2*8)] 3216 add %sp, CC64FSZ + BIAS, %o0 ! (&tf) 3217 stx %g3, [%sp + CC64FSZ + BIAS + TF_G + (3*8)] 3218 stx %g4, [%sp + CC64FSZ + BIAS + TF_G + (4*8)] 3219 stx %g5, [%sp + CC64FSZ + BIAS + TF_G + (5*8)] 3220 rdpr %pil, %g5 3221 stx %g6, [%sp + CC64FSZ + BIAS + TF_G + (6*8)] 3222 stx %g7, [%sp + CC64FSZ + BIAS + TF_G + (7*8)] 3223 stb %g5, [%sp + CC64FSZ + BIAS + TF_PIL] 3224 stb %g5, [%sp + CC64FSZ + BIAS + TF_OLDPIL] 3225 3226 /* 3227 * Phew, ready to enable traps and call C code. 3228 */ 3229 wrpr %g0, 0, %tl 3230 3231 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 3232 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 3233 call _C_LABEL(text_access_fault) ! text_access_fault(tf, type, ...) 3234 nop 3235 3236 ba,a,pt %icc, return_from_trap 3237 nop 3238 NOTREACHED 3239 3240 sun4v_tl1_dtsb_prot: 3241 GET_MMFSA %g1 ! MMU Fault status area 3242 add %g1, 0x48, %g3 3243 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 3244 add %g1, 0x50, %g6 3245 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3246 3247 GET_CTXBUSY %g4 3248 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 3249 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 3250 3251 srax %g3, HOLESHIFT, %g5 ! Check for valid address 3252 brz,pt %g5, 0f ! Should be zero or -1 3253 inc %g5 ! Make -1 -> 0 3254 brnz,pn %g5, sun4v_tl1_ptbl_miss ! Error! In hole! 3255 0: 3256 srlx %g3, STSHIFT, %g6 3257 and %g6, STMASK, %g6 ! Index into pm_segs 3258 sll %g6, 3, %g6 3259 add %g4, %g6, %g4 3260 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 3261 3262 srlx %g3, PDSHIFT, %g6 3263 and %g6, PDMASK, %g6 3264 sll %g6, 3, %g6 3265 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 3266 add %g4, %g6, %g4 3267 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 3268 3269 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 3270 and %g6, PTMASK, %g6 3271 sll %g6, 3, %g6 3272 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 3273 add %g4, %g6, %g6 3274 1: 3275 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 3276 brgez,pn %g4, sun4v_tl1_ptbl_miss ! Entry invalid? Punt 3277 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g7 ! Update the modified bit 3278 3279 # btst SUN4V_TLB_REAL_W|SUN4V_TLB_W, %g4 ! Is it a ref fault? 3280 mov 1, %g2 3281 sllx %g2, 61, %g2 ! %g2 is now SUN4V_TLB_REAL_W 3282 or %g2, SUN4V_TLB_W, %g2 3283 btst %g2, %g4 3284 bz,pn %xcc, sun4v_tl1_ptbl_miss ! No -- really fault 3285 nop 3286 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 3287 cmp %g4, %g7 3288 bne,pn %xcc, 1b 3289 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g4 ! Update the modified bit 3290 2: 3291 GET_TSB_DMMU %g2 3292 3293 mov %g1, %g7 ! save MMFSA 3294 3295 /* Construct TSB tag word. */ 3296 add %g1, 0x50, %g6 3297 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3298 mov %g3, %g1 ! Data fault address 3299 srlx %g1, 22, %g1 ! 63..22 of virt addr 3300 sllx %g6, 48, %g6 ! context_id in 63..48 3301 or %g1, %g6, %g1 ! construct TTE tag 3302 3303 srlx %g3, PTSHIFT, %g3 3304 sethi %hi(_C_LABEL(tsbsize)), %g5 3305 mov 512, %g6 3306 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 3307 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 3308 sub %g5, 1, %g5 ! TSBENTS -> offset 3309 and %g3, %g5, %g3 ! mask out TTE index 3310 sllx %g3, 4, %g3 ! TTE size is 16 bytes 3311 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 3312 3313 membar #StoreStore 3314 3315 STPTR %g4, [%g2 + 8] ! store TTE data 3316 STPTR %g1, [%g2] ! store TTE tag 3317 3318 mov %o0, %g1 3319 mov %o1, %g2 3320 mov %o2, %g3 3321 3322 add %g7, 0x48, %o0 3323 ldxa [%o0] ASI_PHYS_CACHED, %o0 ! Data fault address 3324 add %g7, 0x50, %o1 3325 ldxa [%o1] ASI_PHYS_CACHED, %o1 ! Data fault context 3326 mov MAP_DTLB, %o2 3327 ta ST_MMU_UNMAP_ADDR 3328 3329 mov %g1, %o0 3330 mov %g2, %o1 3331 mov %g3, %o2 3332 3333 retry 3334 NOTREACHED 3335 3336 sun4v_tl1_ptbl_miss: 3337 rdpr %tpc, %g1 3338 3339 set rft_user_fault_start, %g2 3340 cmp %g1, %g2 3341 blu,pt %xcc, 1f 3342 set rft_user_fault_end, %g2 3343 cmp %g1, %g2 3344 bgeu,pt %xcc, 1f 3345 nop 3346 3347 /* We had a miss inside rtf_user_fault_start/rtf_user_fault_end block (FILL) */ 3348 3349 /* Fixup %cwp. */ 3350 rdpr %cwp, %g1 3351 inc %g1 3352 wrpr %g1, %cwp 3353 3354 rdpr %tt, %g1 3355 wrpr 1, %tl 3356 wrpr %g1, %tt 3357 rdpr %cwp, %g1 3358 set TSTATE_KERN, %g2 3359 wrpr %g1, %g2, %tstate 3360 set return_from_trap, %g1 3361 wrpr %g1, %tpc 3362 add %g1, 4, %g1 3363 wrpr %g1, %tnpc 3364 wrpr %g0, 1, %gl 3365 3366 ba,pt %xcc, sun4v_datatrap 3367 wrpr WSTATE_KERN, %wstate 3368 3369 1: 3370 rdpr %tstate, %g3 3371 rdpr %tt, %g4 3372 3373 rdpr %tl, %g1 3374 dec %g1 3375 wrpr %g1, %tl 3376 rdpr %tt, %g2 3377 inc %g1 3378 wrpr %g1, %tl 3379 3380 wrpr %g0, %g3, %tstate 3381 wrpr %g0, %g4, %tt 3382 3383 andn %g2, 0x00f, %g3 3384 cmp %g3, 0x080 3385 be,pn %icc, flush_normals 3386 nop 3387 cmp %g3, 0x0a0 3388 be,pn %icc, flush_others 3389 nop 3390 cmp %g3, 0x0c0 3391 be,pn %icc, ufill_trap 3392 nop 3393 3394 Debugger() 3395 NOTREACHED 3396 3397 flush_others: 3398 set pcbspill_others, %g1 3399 wrpr %g1, %tnpc 3400 done 3401 NOTREACHED 3402 3403 flush_normals: 3404 ufill_trap: 3405 3406 /* 3407 * Rearrange our trap state such that it appears as if we got 3408 * this trap directly from user mode. Then process it at TL = 1. 3409 * We'll take the spill/fill trap again once we return to user mode. 3410 */ 3411 rdpr %tt, %g1 3412 rdpr %tstate, %g3 3413 wrpr %g0, 1, %tl 3414 wrpr %g0, %g1, %tt 3415 rdpr %tstate, %g2 3416 wrpr %g0, 2, %tl 3417 and %g2, TSTATE_CWP, %g2 3418 andn %g3, TSTATE_CWP, %g3 3419 wrpr %g2, %g3, %tstate 3420 set sun4v_datatrap, %g4 3421 wrpr %g0, %g4, %tnpc 3422 done 3423 3424 /* 3425 * Spill user windows into the PCB. 3426 */ 3427 pcbspill_normals: 3428 ba,pt %xcc, pcbspill 3429 wrpr 0x80, %tt 3430 3431 pcbspill_others: 3432 wrpr 0xa0, %tt 3433 3434 pcbspill: 3435 set CPUINFO_VA, %g6 3436 ldx [%g6 + CI_CPCB], %g6 3437 3438 GET_CTXBUSY %g1 3439 3440 ldx [%g1], %g1 ! kernel pmap is ctx 0 3441 3442 srlx %g6, STSHIFT, %g7 3443 and %g7, STMASK, %g7 3444 sll %g7, 3, %g7 ! byte offset into ctxbusy 3445 add %g7, %g1, %g1 3446 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 3447 3448 srlx %g6, PDSHIFT, %g7 ! Do page directory 3449 and %g7, PDMASK, %g7 3450 sll %g7, 3, %g7 3451 brz,pn %g1, pcbspill_fail 3452 add %g7, %g1, %g1 3453 ldxa [%g1] ASI_PHYS_CACHED, %g1 3454 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 3455 and %g7, PTMASK, %g7 3456 brz %g1, pcbspill_fail 3457 sll %g7, 3, %g7 3458 add %g1, %g7, %g7 3459 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 3460 brgez %g7, pcbspill_fail 3461 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 3462 sll %g6, 32-PGSHIFT, %g6 ! And offset 3463 sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE 3464 srl %g6, 32-PGSHIFT, %g6 3465 srax %g7, 8, %g7 3466 or %g7, %g6, %g6 ! Then combine them to form PA 3467 3468 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 3469 3470 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Fetch current nsaved from the pcb 3471 sllx %g7, 7, %g5 ! 8+8 registers each 8 bytes = 128 bytes (2^7) 3472 add %g6, %g5, %g5 ! Offset into pcb_rw 3473 1: 3474 SPILL stxa, %g5 + PCB_RW, 8, %asi ! Store the locals and ins 3475 3476 add %g5, 16*8, %g5 ! Next location for saved register windows 3477 3478 stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out 3479 3480 saved ! Increments %cansave and decrements %canrestore 3481 ! or %otherwin 3482 3483 rdpr %cwp, %g1 ! shift register window forward 3484 inc %g1 3485 wrpr %g1, %cwp 3486 inc %g7 ! increment number of saved register windows 3487 3488 rdpr %otherwin, %g1 ! Check to see if done spill'ing otherwin 3489 brnz,pt %g1, 1b 3490 nop 3491 3492 stba %g7, [%g6 + PCB_NSAVED] %asi 3493 3494 retry 3495 NOTREACHED 3496 3497 pcbspill_fail: 3498 Debugger() 3499 NOTREACHED 3500 3501 3502 pcbspill_other: 3503 set CPUINFO_VA, %g6 3504 ldx [%g6 + CI_CPCB], %g6 3505 3506 GET_CTXBUSY %g1 3507 3508 ldx [%g1], %g1 ! kernel pmap is ctx 0 3509 3510 srlx %g6, STSHIFT, %g7 3511 and %g7, STMASK, %g7 3512 sll %g7, 3, %g7 ! byte offset into ctxbusy 3513 add %g7, %g1, %g1 3514 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 3515 3516 srlx %g6, PDSHIFT, %g7 ! Do page directory 3517 and %g7, PDMASK, %g7 3518 sll %g7, 3, %g7 3519 brz,pn %g1, pcbspill_other_fail 3520 add %g7, %g1, %g1 3521 ldxa [%g1] ASI_PHYS_CACHED, %g1 3522 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 3523 and %g7, PTMASK, %g7 3524 brz %g1, pcbspill_other_fail 3525 sll %g7, 3, %g7 3526 add %g1, %g7, %g7 3527 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 3528 brgez %g7, pcbspill_other_fail 3529 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 3530 sll %g6, 32-PGSHIFT, %g6 ! And offset 3531 sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE 3532 srl %g6, 32-PGSHIFT, %g6 3533 srax %g7, 8, %g7 3534 or %g7, %g6, %g6 ! Then combine them to form PA 3535 3536 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 3537 3538 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Fetch current nsaved from the pcb 3539 sllx %g7, 7, %g5 ! 8+8 registers each 8 bytes = 128 bytes (2^7) 3540 add %g6, %g5, %g5 ! Offset into pcb_rw 3541 1: 3542 SPILL stxa, %g5 + PCB_RW, 8, %asi ! Store the locals and ins 3543 3544 add %g5, 16*8, %g5 ! Next location for saved register windows 3545 3546 stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out 3547 3548 saved ! Increments %cansave and decrements %canrestore 3549 ! or %otherwin 3550 3551 rdpr %cwp, %g1 ! shift register window forward 3552 inc %g1 3553 wrpr %g1, %cwp 3554 3555 3556 inc %g7 ! increment number of saved register windows 3557 3558 rdpr %otherwin, %g1 ! Check to see if done spill'ing otherwin 3559 brnz,pt %g1, 1b 3560 nop 3561 3562 stba %g7, [%g6 + PCB_NSAVED] %asi 3563 3564 retry 3565 NOTREACHED 3566 3567 pcbspill_other_fail: 3568 Debugger() 3569 NOTREACHED 3570 3571 3572 spill_normal_to_user_stack: 3573 mov %sp, %g6 ! calculate virtual address of destination stack 3574 add %g6, BIAS, %g6 3575 3576 mov CTX_SECONDARY, %g2 ! Is this context ok or should it be CTX_PRIMARY? XXX 3577 GET_MMU_CONTEXTID %g3, %g2, %g1 3578 sllx %g3, 3, %g3 ! Make it into an offset into ctxbusy (see below) 3579 3580 GET_CTXBUSY %g1 3581 ldx [%g1 + %g3], %g1 ! Fetch pmap for current context id 3582 3583 ! Start of code to extract PA 3584 srlx %g6, STSHIFT, %g7 3585 and %g7, STMASK, %g7 3586 sll %g7, 3, %g7 ! byte offset into ctxbusy 3587 add %g7, %g1, %g1 3588 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 3589 srlx %g6, PDSHIFT, %g7 ! Do page directory 3590 and %g7, PDMASK, %g7 3591 sll %g7, 3, %g7 3592 brz,pn %g1, spill_normal_to_user_stack_fail 3593 add %g7, %g1, %g1 3594 3595 ldxa [%g1] ASI_PHYS_CACHED, %g1 3596 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 3597 and %g7, PTMASK, %g7 3598 brz %g1, spill_normal_to_user_stack_fail 3599 sll %g7, 3, %g7 3600 3601 add %g1, %g7, %g7 3602 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 3603 brgez %g7, spill_normal_to_user_stack_fail 3604 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 3605 3606 sll %g6, 32-PGSHIFT, %g6 ! And offset 3607 sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE 3608 srl %g6, 32-PGSHIFT, %g6 3609 srax %g7, 8, %g7 3610 or %g7, %g6, %g6 ! Then combine them to form PA 3611 ! End of code to extract PA 3612 3613 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 3614 SPILL stxa, %g6, 8, %asi ! Store the locals and ins 3615 saved 3616 3617 retry 3618 NOTREACHED 3619 3620 spill_normal_to_user_stack_fail: 3621 sir 3622 nop 3623 3624 /* 3625 * End of traps for sun4v. 3626 */ 3627 3628 #endif 3629 3630 /* 3631 * We're here because we took an alignment fault in NUCLEUS context. 3632 * This could be a kernel bug or it could be due to saving a user 3633 * window to an invalid stack pointer. 3634 * 3635 * If the latter is the case, we could try to emulate unaligned accesses, 3636 * but we really don't know where to store the registers since we can't 3637 * determine if there's a stack bias. Or we could store all the regs 3638 * into the PCB and punt, until the user program uses up all the CPU's 3639 * register windows and we run out of places to store them. So for 3640 * simplicity we'll just blow them away and enter the trap code which 3641 * will generate a bus error. Debugging the problem will be a bit 3642 * complicated since lots of register windows will be lost, but what 3643 * can we do? 3644 */ 3645 checkalign: 3646 rdpr %tl, %g2 3647 subcc %g2, 1, %g1 3648 bneg,pn %icc, slowtrap ! Huh? 3649 sethi %hi(CPCB), %g6 ! get current pcb 3650 3651 wrpr %g1, 0, %tl 3652 rdpr %tt, %g7 3653 rdpr %tstate, %g4 3654 andn %g7, 0x3f, %g5 3655 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx 3656 bne,a,pn %icc, slowtrap 3657 wrpr %g1, 0, %tl ! Revert TL XXX wrpr in a delay slot... 3658 3659 #ifdef DEBUG 3660 cmp %g7, 0x34 ! If we took a datafault just before this trap 3661 bne,pt %icc, checkalignspill ! our stack's probably bad so we need to switch somewhere else 3662 nop 3663 3664 !! 3665 !! Double data fault -- bad stack? 3666 !! 3667 wrpr %g2, %tl ! Restore trap level. 3668 sir ! Just issue a reset and don't try to recover. 3669 mov %fp, %l6 ! Save the frame pointer 3670 set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack 3671 add %fp, -CC64FSZ, %sp ! Create a stackframe 3672 wrpr %g0, 15, %pil ! Disable interrupts, too 3673 wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB 3674 wrpr %g0, 7, %cansave ! probably is too, so blow away 3675 ba slowtrap ! all our register windows. 3676 wrpr %g0, 0x101, %tt 3677 #endif 3678 checkalignspill: 3679 /* 3680 * %g1 -- current tl 3681 * %g2 -- original tl 3682 * %g4 -- tstate 3683 * %g7 -- tt 3684 */ 3685 3686 and %g4, CWP, %g5 3687 wrpr %g5, %cwp ! Go back to the original register win 3688 3689 /* 3690 * Remember: 3691 * 3692 * %otherwin = 0 3693 * %cansave = NWINDOWS - 2 - %canrestore 3694 */ 3695 3696 rdpr %otherwin, %g6 3697 rdpr %canrestore, %g3 3698 rdpr %ver, %g5 3699 sub %g3, %g6, %g3 ! Calculate %canrestore - %g7 3700 and %g5, CWP, %g5 ! NWINDOWS-1 3701 movrlz %g3, %g0, %g3 ! Clamp at zero 3702 wrpr %g0, 0, %otherwin 3703 wrpr %g3, 0, %canrestore ! This is the new canrestore 3704 dec %g5 ! NWINDOWS-2 3705 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel 3706 sub %g5, %g3, %g5 ! NWINDOWS-2-%canrestore 3707 wrpr %g5, 0, %cansave 3708 3709 wrpr %g0, T_ALIGN, %tt ! This was an alignment fault 3710 /* 3711 * Now we need to determine if this was a userland store or not. 3712 * Userland stores occur in anything other than the kernel spill 3713 * handlers (trap type 09x). 3714 */ 3715 and %g7, 0xff0, %g5 3716 cmp %g5, 0x90 3717 bz,pn %icc, slowtrap 3718 nop 3719 bclr TSTATE_PRIV, %g4 3720 wrpr %g4, 0, %tstate 3721 ba,a,pt %icc, slowtrap 3722 nop 3723 3724 /* 3725 * slowtrap() builds a trap frame and calls trap(). 3726 * This is called `slowtrap' because it *is*.... 3727 * We have to build a full frame for ptrace(), for instance. 3728 * 3729 * Registers: 3730 * 3731 */ 3732 slowtrap: 3733 #ifdef TRAPS_USE_IG 3734 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 3735 #endif 3736 #ifdef DIAGNOSTIC 3737 /* Make sure kernel stack is aligned */ 3738 btst 0x03, %sp ! 32-bit stack OK? 3739 and %sp, 0x07, %g4 ! 64-bit stack OK? 3740 bz,pt %icc, 1f 3741 cmp %g4, 0x1 ! Must end in 0b001 3742 be,pt %icc, 1f 3743 rdpr %wstate, %g7 3744 cmp %g7, WSTATE_KERN 3745 bnz,pt %icc, 1f ! User stack -- we'll blow it away 3746 nop 3747 set PANICSTACK-CC64FSZ-STKB, %sp 3748 1: 3749 #endif 3750 rdpr %tt, %g4 3751 rdpr %tstate, %g1 3752 rdpr %tpc, %g2 3753 rdpr %tnpc, %g3 3754 3755 TRAP_SETUP(-CC64FSZ-TF_SIZE) 3756 Lslowtrap_reenter: 3757 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 3758 mov %g4, %o1 ! (type) 3759 stx %g2, [%sp + CC64FSZ + STKB + TF_PC] 3760 rd %y, %g5 3761 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] 3762 mov %g1, %o3 ! (pstate) 3763 st %g5, [%sp + CC64FSZ + STKB + TF_Y] 3764 mov %g2, %o2 ! (pc) 3765 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 3766 3767 ! Get back to normal globals 3768 #ifdef SUN4V 3769 sethi %hi(cputyp), %g5 3770 ld [%g5 + %lo(cputyp)], %g5 3771 cmp %g5, CPU_SUN4V 3772 bne,pt %icc, 1f 3773 nop 3774 NORMAL_GLOBALS_SUN4V 3775 ba 2f 3776 nop 3777 1: 3778 #endif 3779 NORMAL_GLOBALS_SUN4U 3780 2: 3781 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] 3782 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] 3783 add %sp, CC64FSZ + STKB, %o0 ! (&tf) 3784 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] 3785 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] 3786 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] 3787 rdpr %pil, %g5 3788 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] 3789 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] 3790 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 3791 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 3792 /* 3793 * Phew, ready to enable traps and call C code. 3794 */ 3795 rdpr %tl, %g1 3796 dec %g1 3797 movrlz %g1, %g0, %g1 3798 wrpr %g0, %g1, %tl ! Revert to kernel mode 3799 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 3800 !! In our case we need to clear it before calling any C-code 3801 clr %g4 3802 3803 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 3804 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 3805 call _C_LABEL(trap) ! trap(tf, type, pc, pstate) 3806 nop 3807 3808 b return_from_trap 3809 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 3810 NOTREACHED 3811 #if 1 3812 /* 3813 * This code is no longer needed. 3814 */ 3815 /* 3816 * Do a `software' trap by re-entering the trap code, possibly first 3817 * switching from interrupt stack to kernel stack. This is used for 3818 * scheduling and signal ASTs (which generally occur from softclock or 3819 * tty or net interrupts). 3820 * 3821 * We enter with the trap type in %g1. All we have to do is jump to 3822 * Lslowtrap_reenter above, but maybe after switching stacks.... 3823 * 3824 * We should be running alternate globals. The normal globals and 3825 * out registers were just loaded from the old trap frame. 3826 * 3827 * Input Params: 3828 * %g1 = tstate 3829 * %g2 = tpc 3830 * %g3 = tnpc 3831 * %g4 = tt == T_AST 3832 */ 3833 softtrap: 3834 sethi %hi(EINTSTACK-STKB), %g5 3835 sethi %hi(EINTSTACK-INTSTACK), %g7 3836 or %g5, %lo(EINTSTACK-STKB), %g5 3837 dec %g7 3838 sub %g5, %sp, %g5 3839 sethi %hi(CPCB), %g6 3840 andncc %g5, %g7, %g0 3841 bnz,pt %xcc, Lslowtrap_reenter 3842 LDPTR [%g6 + %lo(CPCB)], %g7 3843 set USPACE-CC64FSZ-TF_SIZE-STKB, %g5 3844 add %g7, %g5, %g6 3845 SET_SP_REDZONE(%g7, %g5) 3846 #ifdef DEBUG 3847 stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT] ! Generate a new trapframe 3848 #endif 3849 stx %i0, [%g6 + CC64FSZ + STKB + TF_O + (0*8)] ! but don't bother with 3850 stx %i1, [%g6 + CC64FSZ + STKB + TF_O + (1*8)] ! locals and ins 3851 stx %i2, [%g6 + CC64FSZ + STKB + TF_O + (2*8)] 3852 stx %i3, [%g6 + CC64FSZ + STKB + TF_O + (3*8)] 3853 stx %i4, [%g6 + CC64FSZ + STKB + TF_O + (4*8)] 3854 stx %i5, [%g6 + CC64FSZ + STKB + TF_O + (5*8)] 3855 stx %i6, [%g6 + CC64FSZ + STKB + TF_O + (6*8)] 3856 stx %i7, [%g6 + CC64FSZ + STKB + TF_O + (7*8)] 3857 #ifdef DEBUG 3858 ldx [%sp + CC64FSZ + STKB + TF_I + (0*8)], %l0 ! Copy over the rest of the regs 3859 ldx [%sp + CC64FSZ + STKB + TF_I + (1*8)], %l1 ! But just dirty the locals 3860 ldx [%sp + CC64FSZ + STKB + TF_I + (2*8)], %l2 3861 ldx [%sp + CC64FSZ + STKB + TF_I + (3*8)], %l3 3862 ldx [%sp + CC64FSZ + STKB + TF_I + (4*8)], %l4 3863 ldx [%sp + CC64FSZ + STKB + TF_I + (5*8)], %l5 3864 ldx [%sp + CC64FSZ + STKB + TF_I + (6*8)], %l6 3865 ldx [%sp + CC64FSZ + STKB + TF_I + (7*8)], %l7 3866 stx %l0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)] 3867 stx %l1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)] 3868 stx %l2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)] 3869 stx %l3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)] 3870 stx %l4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)] 3871 stx %l5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)] 3872 stx %l6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)] 3873 stx %l7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)] 3874 ldx [%sp + CC64FSZ + STKB + TF_L + (0*8)], %l0 3875 ldx [%sp + CC64FSZ + STKB + TF_L + (1*8)], %l1 3876 ldx [%sp + CC64FSZ + STKB + TF_L + (2*8)], %l2 3877 ldx [%sp + CC64FSZ + STKB + TF_L + (3*8)], %l3 3878 ldx [%sp + CC64FSZ + STKB + TF_L + (4*8)], %l4 3879 ldx [%sp + CC64FSZ + STKB + TF_L + (5*8)], %l5 3880 ldx [%sp + CC64FSZ + STKB + TF_L + (6*8)], %l6 3881 ldx [%sp + CC64FSZ + STKB + TF_L + (7*8)], %l7 3882 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)] 3883 stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)] 3884 stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)] 3885 stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)] 3886 stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)] 3887 stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)] 3888 stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)] 3889 stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)] 3890 #endif 3891 ba,pt %xcc, Lslowtrap_reenter 3892 mov %g6, %sp 3893 #endif 3894 3895 #if 0 3896 /* 3897 * breakpoint: capture as much info as possible and then call DDB 3898 * or trap, as the case may be. 3899 * 3900 * First, we switch to interrupt globals, and blow away %g7. Then 3901 * switch down one stackframe -- just fiddle w/cwp, don't save or 3902 * we'll trap. Then slowly save all the globals into our static 3903 * register buffer. etc. etc. 3904 */ 3905 3906 breakpoint: 3907 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! Get IG to use 3908 rdpr %cwp, %g7 3909 inc 1, %g7 ! Equivalent of save 3910 wrpr %g7, 0, %cwp ! Now we have some unused locals to fiddle with 3911 XXX ddb_regs is now ddb-regp and is a pointer not a symbol. 3912 set _C_LABEL(ddb_regs), %l0 3913 stx %g1, [%l0+DBR_IG+(1*8)] ! Save IGs 3914 stx %g2, [%l0+DBR_IG+(2*8)] 3915 stx %g3, [%l0+DBR_IG+(3*8)] 3916 stx %g4, [%l0+DBR_IG+(4*8)] 3917 stx %g5, [%l0+DBR_IG+(5*8)] 3918 stx %g6, [%l0+DBR_IG+(6*8)] 3919 stx %g7, [%l0+DBR_IG+(7*8)] 3920 wrpr %g0, PSTATE_KERN|PSTATE_MG, %pstate ! Get MG to use 3921 stx %g1, [%l0+DBR_MG+(1*8)] ! Save MGs 3922 stx %g2, [%l0+DBR_MG+(2*8)] 3923 stx %g3, [%l0+DBR_MG+(3*8)] 3924 stx %g4, [%l0+DBR_MG+(4*8)] 3925 stx %g5, [%l0+DBR_MG+(5*8)] 3926 stx %g6, [%l0+DBR_MG+(6*8)] 3927 stx %g7, [%l0+DBR_MG+(7*8)] 3928 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Get AG to use 3929 stx %g1, [%l0+DBR_AG+(1*8)] ! Save AGs 3930 stx %g2, [%l0+DBR_AG+(2*8)] 3931 stx %g3, [%l0+DBR_AG+(3*8)] 3932 stx %g4, [%l0+DBR_AG+(4*8)] 3933 stx %g5, [%l0+DBR_AG+(5*8)] 3934 stx %g6, [%l0+DBR_AG+(6*8)] 3935 stx %g7, [%l0+DBR_AG+(7*8)] 3936 wrpr %g0, PSTATE_KERN, %pstate ! Get G to use 3937 stx %g1, [%l0+DBR_G+(1*8)] ! Save Gs 3938 stx %g2, [%l0+DBR_G+(2*8)] 3939 stx %g3, [%l0+DBR_G+(3*8)] 3940 stx %g4, [%l0+DBR_G+(4*8)] 3941 stx %g5, [%l0+DBR_G+(5*8)] 3942 stx %g6, [%l0+DBR_G+(6*8)] 3943 stx %g7, [%l0+DBR_G+(7*8)] 3944 rdpr %canrestore, %l1 3945 stb %l1, [%l0+DBR_CANRESTORE] 3946 rdpr %cansave, %l2 3947 stb %l2, [%l0+DBR_CANSAVE] 3948 rdpr %cleanwin, %l3 3949 stb %l3, [%l0+DBR_CLEANWIN] 3950 rdpr %wstate, %l4 3951 stb %l4, [%l0+DBR_WSTATE] 3952 rd %y, %l5 3953 stw %l5, [%l0+DBR_Y] 3954 rdpr %tl, %l6 3955 stb %l6, [%l0+DBR_TL] 3956 dec 1, %g7 3957 #endif 3958 3959 /* 3960 * I will not touch any of the DDB or KGDB stuff until I know what's going 3961 * on with the symbol table. This is all still v7/v8 code and needs to be fixed. 3962 */ 3963 #ifdef KGDB 3964 /* 3965 * bpt is entered on all breakpoint traps. 3966 * If this is a kernel breakpoint, we do not want to call trap(). 3967 * Among other reasons, this way we can set breakpoints in trap(). 3968 */ 3969 bpt: 3970 set TSTATE_PRIV, %l4 3971 andcc %l4, %l0, %g0 ! breakpoint from kernel? 3972 bz slowtrap ! no, go do regular trap 3973 nop 3974 3975 /* 3976 * Build a trap frame for kgdb_trap_glue to copy. 3977 * Enable traps but set ipl high so that we will not 3978 * see interrupts from within breakpoints. 3979 */ 3980 save %sp, -CCFSZ-TF_SIZE, %sp ! allocate a trap frame 3981 TRAP_SETUP(-CCFSZ-TF_SIZE) 3982 or %l0, PSR_PIL, %l4 ! splhigh() 3983 wr %l4, 0, %psr ! the manual claims that this 3984 wr %l4, PSR_ET, %psr ! song and dance is necessary 3985 std %l0, [%sp + CCFSZ + 0] ! tf.tf_psr, tf.tf_pc 3986 mov %l3, %o0 ! trap type arg for kgdb_trap_glue 3987 rd %y, %l3 3988 std %l2, [%sp + CCFSZ + 8] ! tf.tf_npc, tf.tf_y 3989 rd %wim, %l3 3990 st %l3, [%sp + CCFSZ + 16] ! tf.tf_wim (a kgdb-only r/o field) 3991 st %g1, [%sp + CCFSZ + 20] ! tf.tf_global[1] 3992 std %g2, [%sp + CCFSZ + 24] ! etc 3993 std %g4, [%sp + CCFSZ + 32] 3994 std %g6, [%sp + CCFSZ + 40] 3995 std %i0, [%sp + CCFSZ + 48] ! tf.tf_in[0..1] 3996 std %i2, [%sp + CCFSZ + 56] ! etc 3997 std %i4, [%sp + CCFSZ + 64] 3998 std %i6, [%sp + CCFSZ + 72] 3999 4000 /* 4001 * Now call kgdb_trap_glue(); if it returns, call trap(). 4002 */ 4003 mov %o0, %l3 ! gotta save trap type 4004 call _C_LABEL(kgdb_trap_glue) ! kgdb_trap_glue(type, &trapframe) 4005 add %sp, CCFSZ, %o1 ! (&trapframe) 4006 4007 /* 4008 * Use slowtrap to call trap---but first erase our tracks 4009 * (put the registers back the way they were). 4010 */ 4011 mov %l3, %o0 ! slowtrap will need trap type 4012 ld [%sp + CCFSZ + 12], %l3 4013 wr %l3, 0, %y 4014 ld [%sp + CCFSZ + 20], %g1 4015 ldd [%sp + CCFSZ + 24], %g2 4016 ldd [%sp + CCFSZ + 32], %g4 4017 b Lslowtrap_reenter 4018 ldd [%sp + CCFSZ + 40], %g6 4019 4020 /* 4021 * Enter kernel breakpoint. Write all the windows (not including the 4022 * current window) into the stack, so that backtrace works. Copy the 4023 * supplied trap frame to the kgdb stack and switch stacks. 4024 * 4025 * kgdb_trap_glue(type, tf0) 4026 * int type; 4027 * struct trapframe *tf0; 4028 */ 4029 ENTRY_NOPROFILE(kgdb_trap_glue) 4030 save %sp, -CCFSZ, %sp 4031 4032 flushw ! flush all windows 4033 mov %sp, %l4 ! %l4 = current %sp 4034 4035 /* copy trapframe to top of kgdb stack */ 4036 set _C_LABEL(kgdb_stack) + KGDB_STACK_SIZE - 80, %l0 4037 ! %l0 = tfcopy -> end_of_kgdb_stack 4038 mov 80, %l1 4039 1: ldd [%i1], %l2 4040 inc 8, %i1 4041 deccc 8, %l1 4042 std %l2, [%l0] 4043 bg 1b 4044 inc 8, %l0 4045 4046 #ifdef NOTDEF_DEBUG 4047 /* save old red zone and then turn it off */ 4048 sethi %hi(_C_LABEL(redzone)), %l7 4049 ld [%l7 + %lo(_C_LABEL(redzone))], %l6 4050 st %g0, [%l7 + %lo(_C_LABEL(redzone))] 4051 #endif 4052 /* switch to kgdb stack */ 4053 add %l0, -CCFSZ-TF_SIZE, %sp 4054 4055 /* if (kgdb_trap(type, tfcopy)) kgdb_rett(tfcopy); */ 4056 mov %i0, %o0 4057 call _C_LABEL(kgdb_trap) 4058 add %l0, -80, %o1 4059 tst %o0 4060 bnz,a kgdb_rett 4061 add %l0, -80, %g1 4062 4063 /* 4064 * kgdb_trap() did not handle the trap at all so the stack is 4065 * still intact. A simple `restore' will put everything back, 4066 * after we reset the stack pointer. 4067 */ 4068 mov %l4, %sp 4069 #ifdef NOTDEF_DEBUG 4070 st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! restore red zone 4071 #endif 4072 ret 4073 restore 4074 4075 /* 4076 * Return from kgdb trap. This is sort of special. 4077 * 4078 * We know that kgdb_trap_glue wrote the window above it, so that we will 4079 * be able to (and are sure to have to) load it up. We also know that we 4080 * came from kernel land and can assume that the %fp (%i6) we load here 4081 * is proper. We must also be sure not to lower ipl (it is at splhigh()) 4082 * until we have traps disabled, due to the SPARC taking traps at the 4083 * new ipl before noticing that PSR_ET has been turned off. We are on 4084 * the kgdb stack, so this could be disastrous. 4085 * 4086 * Note that the trapframe argument in %g1 points into the current stack 4087 * frame (current window). We abandon this window when we move %g1->tf_psr 4088 * into %psr, but we will not have loaded the new %sp yet, so again traps 4089 * must be disabled. 4090 */ 4091 kgdb_rett: 4092 rd %psr, %g4 ! turn off traps 4093 wr %g4, PSR_ET, %psr 4094 /* use the three-instruction delay to do something useful */ 4095 ld [%g1], %g2 ! pick up new %psr 4096 ld [%g1 + 12], %g3 ! set %y 4097 wr %g3, 0, %y 4098 #ifdef NOTDEF_DEBUG 4099 st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! and restore red zone 4100 #endif 4101 wr %g0, 0, %wim ! enable window changes 4102 nop; nop; nop 4103 /* now safe to set the new psr (changes CWP, leaves traps disabled) */ 4104 wr %g2, 0, %psr ! set rett psr (including cond codes) 4105 /* 3 instruction delay before we can use the new window */ 4106 /*1*/ ldd [%g1 + 24], %g2 ! set new %g2, %g3 4107 /*2*/ ldd [%g1 + 32], %g4 ! set new %g4, %g5 4108 /*3*/ ldd [%g1 + 40], %g6 ! set new %g6, %g7 4109 4110 /* now we can use the new window */ 4111 mov %g1, %l4 4112 ld [%l4 + 4], %l1 ! get new pc 4113 ld [%l4 + 8], %l2 ! get new npc 4114 ld [%l4 + 20], %g1 ! set new %g1 4115 4116 /* set up returnee's out registers, including its %sp */ 4117 ldd [%l4 + 48], %i0 4118 ldd [%l4 + 56], %i2 4119 ldd [%l4 + 64], %i4 4120 ldd [%l4 + 72], %i6 4121 4122 /* load returnee's window, making the window above it be invalid */ 4123 restore 4124 restore %g0, 1, %l1 ! move to inval window and set %l1 = 1 4125 rd %psr, %l0 4126 srl %l1, %l0, %l1 4127 wr %l1, 0, %wim ! %wim = 1 << (%psr & 31) 4128 sethi %hi(CPCB), %l1 4129 LDPTR [%l1 + %lo(CPCB)], %l1 4130 and %l0, 31, %l0 ! CWP = %psr & 31; 4131 ! st %l0, [%l1 + PCB_WIM] ! cpcb->pcb_wim = CWP; 4132 save %g0, %g0, %g0 ! back to window to reload 4133 ! LOADWIN(%sp) 4134 save %g0, %g0, %g0 ! back to trap window 4135 /* note, we have not altered condition codes; safe to just rett */ 4136 RETT 4137 #endif 4138 4139 /* 4140 * syscall_setup() builds a trap frame and calls syscall(). 4141 * sun_syscall is same but delivers sun system call number 4142 * XXX should not have to save&reload ALL the registers just for 4143 * ptrace... 4144 */ 4145 syscall_setup: 4146 #ifdef TRAPS_USE_IG 4147 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 4148 #endif 4149 TRAP_SETUP(-CC64FSZ-TF_SIZE) 4150 4151 #ifdef DEBUG 4152 rdpr %tt, %o1 ! debug 4153 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 4154 #endif 4155 4156 ! Get back to normal globals 4157 #ifdef SUN4V 4158 sethi %hi(cputyp), %g5 4159 ld [%g5 + %lo(cputyp)], %g5 4160 cmp %g5, CPU_SUN4V 4161 bne,pt %icc, 1f 4162 nop 4163 NORMAL_GLOBALS_SUN4V 4164 ba 2f 4165 nop 4166 1: 4167 #endif 4168 NORMAL_GLOBALS_SUN4U 4169 2: 4170 4171 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 4172 mov %g1, %o1 ! code 4173 rdpr %tpc, %o2 ! (pc) 4174 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 4175 rdpr %tstate, %g1 4176 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 4177 rdpr %tnpc, %o3 4178 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 4179 rd %y, %o4 4180 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 4181 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 4182 wrpr %g0, 0, %tl ! return to tl=0 4183 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 4184 add %sp, CC64FSZ + STKB, %o0 ! (&tf) 4185 4186 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 4187 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 4188 stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] 4189 st %o4, [%sp + CC64FSZ + STKB + TF_Y] 4190 4191 rdpr %pil, %g5 4192 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 4193 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 4194 4195 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 4196 !! In our case we need to clear it before calling any C-code 4197 clr %g4 4198 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 4199 4200 sethi %hi(CURLWP), %l1 4201 LDPTR [%l1 + %lo(CURLWP)], %l1 4202 LDPTR [%l1 + L_PROC], %l1 ! now %l1 points to p 4203 LDPTR [%l1 + P_MD_SYSCALL], %l1 4204 call %l1 4205 wrpr %g0, PSTATE_INTR, %pstate ! turn on interrupts 4206 4207 /* see `lwp_trampoline' for the reason for this label */ 4208 return_from_syscall: 4209 wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts 4210 wrpr %g0, 0, %tl ! Return to tl==0 4211 b return_from_trap 4212 nop 4213 NOTREACHED 4214 4215 /* 4216 * interrupt_vector: 4217 * 4218 * Spitfire chips never get level interrupts directly from H/W. 4219 * Instead, all interrupts come in as interrupt_vector traps. 4220 * The interrupt number or handler address is an 11 bit number 4221 * encoded in the first interrupt data word. Additional words 4222 * are application specific and used primarily for cross-calls. 4223 * 4224 * The interrupt vector handler then needs to identify the 4225 * interrupt source from the interrupt number and arrange to 4226 * invoke the interrupt handler. This can either be done directly 4227 * from here, or a softint at a particular level can be issued. 4228 * 4229 * To call an interrupt directly and not overflow the trap stack, 4230 * the trap registers should be saved on the stack, registers 4231 * cleaned, trap-level decremented, the handler called, and then 4232 * the process must be reversed. 4233 * 4234 * To simplify life all we do here is issue an appropriate softint. 4235 * 4236 * Note: It is impossible to identify or change a device's 4237 * interrupt number until it is probed. That's the 4238 * purpose for all the funny interrupt acknowledge 4239 * code. 4240 * 4241 */ 4242 4243 /* 4244 * Vectored interrupts: 4245 * 4246 * When an interrupt comes in, interrupt_vector uses the interrupt 4247 * vector number to lookup the appropriate intrhand from the intrlev 4248 * array. It then looks up the interrupt level from the intrhand 4249 * structure. It uses the level to index the intrpending array, 4250 * which is 8 slots for each possible interrupt level (so we can 4251 * shift instead of multiply for address calculation). It hunts for 4252 * any available slot at that level. Available slots are NULL. 4253 * 4254 * Then interrupt_vector uses the interrupt level in the intrhand 4255 * to issue a softint of the appropriate level. The softint handler 4256 * figures out what level interrupt it's handling and pulls the first 4257 * intrhand pointer out of the intrpending array for that interrupt 4258 * level, puts a NULL in its place, clears the interrupt generator, 4259 * and invokes the interrupt handler. 4260 */ 4261 4262 /* intrpending array is now in per-CPU structure. */ 4263 4264 #ifdef DEBUG 4265 #define INTRDEBUG_VECTOR 0x1 4266 #define INTRDEBUG_LEVEL 0x2 4267 #define INTRDEBUG_FUNC 0x4 4268 #define INTRDEBUG_SPUR 0x8 4269 .data 4270 .globl _C_LABEL(intrdebug) 4271 _C_LABEL(intrdebug): .word 0x0 4272 /* 4273 * Note: we use the local label `97' to branch forward to, to skip 4274 * actual debugging code following a `intrdebug' bit test. 4275 */ 4276 #endif 4277 .text 4278 interrupt_vector: 4279 #ifdef TRAPSTATS 4280 set _C_LABEL(kiveccnt), %g1 4281 set _C_LABEL(iveccnt), %g2 4282 rdpr %tl, %g3 4283 dec %g3 4284 movrz %g3, %g2, %g1 4285 lduw [%g1], %g2 4286 inc %g2 4287 stw %g2, [%g1] 4288 #endif 4289 ldxa [%g0] ASI_IRSR, %g1 4290 mov IRDR_0H, %g7 4291 ldxa [%g7] ASI_IRDR, %g7 ! Get interrupt number 4292 membar #Sync 4293 4294 btst IRSR_BUSY, %g1 4295 bz,pn %icc, 3f ! spurious interrupt 4296 #ifdef MULTIPROCESSOR 4297 sethi %hi(KERNBASE), %g1 4298 4299 cmp %g7, %g1 4300 bl,a,pt %xcc, Lsoftint_regular ! >= KERNBASE is a fast cross-call 4301 and %g7, (MAXINTNUM-1), %g7 ! XXX make sun4us work 4302 4303 mov IRDR_1H, %g2 4304 ldxa [%g2] ASI_IRDR, %g2 ! Get IPI handler argument 1 4305 mov IRDR_2H, %g3 4306 ldxa [%g3] ASI_IRDR, %g3 ! Get IPI handler argument 2 4307 4308 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ 4309 membar #Sync ! Should not be needed due to retry 4310 4311 jmpl %g7, %g0 4312 nop 4313 #else 4314 and %g7, (MAXINTNUM-1), %g7 ! XXX make sun4us work 4315 #endif 4316 4317 Lsoftint_regular: 4318 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ 4319 membar #Sync ! Should not be needed due to retry 4320 sethi %hi(_C_LABEL(intrlev)), %g3 4321 sllx %g7, PTRSHFT, %g5 ! Calculate entry number 4322 or %g3, %lo(_C_LABEL(intrlev)), %g3 4323 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler 4324 brz,pn %g5, 3f ! NULL means it isn't registered yet. Skip it. 4325 nop 4326 4327 ! increment per-ivec counter 4328 ldx [%g5 + IH_CNT], %g1 4329 inc %g1 4330 stx %g1, [%g5 + IH_CNT] 4331 4332 setup_sparcintr: 4333 LDPTR [%g5+IH_PEND], %g6 ! Read pending flag 4334 brnz,pn %g6, ret_from_intr_vector ! Skip it if it's running 4335 ldub [%g5+IH_PIL], %g6 ! Read interrupt level 4336 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %g1 4337 sll %g6, PTRSHFT, %g3 ! Find start of table for this IPL 4338 or %g1, %lo(CPUINFO_VA+CI_INTRPENDING), %g1 4339 add %g1, %g3, %g1 4340 1: 4341 LDPTR [%g1], %g3 ! Load list head 4342 STPTR %g3, [%g5+IH_PEND] ! Link our intrhand node in 4343 mov %g5, %g7 4344 CASPTRA [%g1] ASI_N, %g3, %g7 4345 cmp %g7, %g3 ! Did it work? 4346 bne,pn CCCR, 1b ! No, try again 4347 .empty 4348 2: 4349 #ifdef NOT_DEBUG 4350 set _C_LABEL(intrdebug), %g7 4351 ld [%g7], %g7 4352 btst INTRDEBUG_VECTOR, %g7 4353 bz,pt %icc, 97f 4354 nop 4355 4356 cmp %g6, 0xa ! ignore clock interrupts? 4357 bz,pt %icc, 97f 4358 nop 4359 4360 STACKFRAME(-CC64FSZ) ! Get a clean register window 4361 LOAD_ASCIZ(%o0,\ 4362 "interrupt_vector: number %lx softint mask %lx pil %lu slot %p\n") 4363 mov %g2, %o1 4364 rdpr %pil, %o3 4365 mov %g1, %o4 4366 GLOBTOLOC 4367 clr %g4 4368 call prom_printf 4369 mov %g6, %o2 4370 LOCTOGLOB 4371 restore 4372 97: 4373 #endif 4374 mov 1, %g7 4375 sll %g7, %g6, %g6 4376 wr %g6, 0, SET_SOFTINT ! Invoke a softint 4377 4378 .global ret_from_intr_vector 4379 ret_from_intr_vector: 4380 retry 4381 NOTREACHED 4382 4383 3: 4384 #ifdef NOT_DEBUG /* always do this */ 4385 set _C_LABEL(intrdebug), %g6 4386 ld [%g6], %g6 4387 btst INTRDEBUG_SPUR, %g6 4388 bz,pt %icc, 97f 4389 nop 4390 #endif 4391 #if 1 4392 set PANICSTACK-STKB, %g1 ! Use panic stack temporarily 4393 save %g1, -CC64FSZ, %sp ! Get a clean register window 4394 LOAD_ASCIZ(%o0, "interrupt_vector: spurious vector %lx at pil %d\n") 4395 mov %g7, %o1 4396 GLOBTOLOC 4397 clr %g4 4398 call prom_printf 4399 rdpr %pil, %o2 4400 LOCTOGLOB 4401 restore 4402 97: 4403 #endif 4404 ba,a ret_from_intr_vector 4405 nop ! XXX spitfire bug? 4406 4407 sun4v_cpu_mondo: 4408 ! XXX Rework this when a UP kernel works - crash for now 4409 sir 4410 mov 0x3c0, %g1 ! CPU Mondo Queue Head 4411 ldxa [%g1] ASI_QUEUE, %g2 ! fetch index value for head 4412 set CPUINFO_VA, %g3 4413 ldx [%g3 + CI_PADDR], %g3 4414 add %g3, CI_CPUMQ, %g3 4415 ldxa [%g3] ASI_PHYS_CACHED, %g3 ! fetch head element 4416 ldxa [%g3 + %g2] ASI_PHYS_CACHED, %g4 ! fetch func 4417 add %g2, 8, %g5 4418 ldxa [%g3 + %g5] ASI_PHYS_CACHED, %g5 ! fetch arg1 4419 add %g2, 16, %g6 4420 ldxa [%g3 + %g6] ASI_PHYS_CACHED, %g6 ! fetch arg2 4421 add %g2, 64, %g2 ! point to next element in queue 4422 and %g2, 0x7ff, %g2 ! modulo queue size 2048 (32*64) 4423 stxa %g2, [%g1] ASI_QUEUE ! update head index 4424 membar #Sync 4425 4426 mov %g4, %g2 4427 mov %g5, %g3 4428 mov %g6, %g5 4429 jmpl %g2, %g0 4430 nop ! No store here! 4431 retry 4432 NOTREACHED 4433 4434 sun4v_dev_mondo: 4435 mov 0x3d0, %g1 ! Dev Mondo Queue Head 4436 ldxa [%g1] ASI_QUEUE, %g2 ! fetch index value 4437 mov 0x3d8, %g1 ! Dev Mondo Queue Tail 4438 ldxa [%g1] ASI_QUEUE, %g4 ! fetch index value 4439 cmp %g2, %g4 ! head = queue? 4440 bne,pt %xcc, 2f ! unusually not the case 4441 nop 4442 retry ! unlikely, ignore interrupt 4443 2: 4444 set CPUINFO_VA, %g3 ! fetch cpuinfo pa 4445 ldx [%g3 + CI_PADDR], %g3 ! fetch intstack pa 4446 set CPUINFO_VA-INTSTACK, %g4 ! offset to cpuinfo 4447 add %g4, %g3, %g3 ! %g3 is now cpuifo 4448 add %g3, CI_DEVMQ, %g3 ! calc offset to devmq 4449 ldxa [%g3] ASI_PHYS_CACHED, %g3 ! fetch address of devmq 4450 ldxa [%g3 + %g2] ASI_PHYS_CACHED, %g5 ! 4451 add %g2, 64, %g2 ! each element is 64 bytes 4452 and %g2, 0x7ff, %g2 ! assume 32 elements 4453 mov 0x3d0, %g1 ! Dev Mondo Queue Head 4454 stxa %g2, [%g1] ASI_QUEUE ! adjust head index value 4455 membar #Sync 4456 4457 cmp %g5, MAXINTNUM ! Handle both sun4v legacy (sysino) and cookies. 4458 bgeu,pn %xcc, 1f ! See UltraSPARC Virtual Machine Specification 4459 nop ! version 3 chapter 6 (Interrupt model) 4460 4461 sethi %hi(_C_LABEL(intrlev)), %g3 4462 sllx %g5, PTRSHFT, %g5 ! Calculate entry number 4463 or %g3, %lo(_C_LABEL(intrlev)), %g3 4464 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler 4465 1: 4466 brnz,pt %g5, setup_sparcintr ! branch if valid handle 4467 nop 4468 4469 ba,a 3b ! log if invalid handle 4470 nop 4471 4472 /* 4473 * Ultra1 and Ultra2 CPUs use soft interrupts for everything. What we do 4474 * on a soft interrupt, is we should check which bits in SOFTINT(%asr22) 4475 * are set, handle those interrupts, then clear them by setting the 4476 * appropriate bits in CLEAR_SOFTINT(%asr21). 4477 * 4478 * We have an array of 8 interrupt vector slots for each of 15 interrupt 4479 * levels. If a vectored interrupt can be dispatched, the dispatch 4480 * routine will place a pointer to an intrhand structure in one of 4481 * the slots. The interrupt handler will go through the list to look 4482 * for an interrupt to dispatch. If it finds one it will pull it off 4483 * the list, free the entry, and call the handler. The code is like 4484 * this: 4485 * 4486 * for (i=0; i<8; i++) 4487 * if (ih = intrpending[intlev][i]) { 4488 * intrpending[intlev][i] = NULL; 4489 * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) 4490 * return; 4491 * strayintr(&frame); 4492 * return; 4493 * } 4494 * 4495 * Otherwise we go back to the old style of polled interrupts. 4496 * 4497 * After preliminary setup work, the interrupt is passed to each 4498 * registered handler in turn. These are expected to return nonzero if 4499 * they took care of the interrupt. If a handler claims the interrupt, 4500 * we exit (hardware interrupts are latched in the requestor so we'll 4501 * just take another interrupt in the unlikely event of simultaneous 4502 * interrupts from two different devices at the same level). If we go 4503 * through all the registered handlers and no one claims it, we report a 4504 * stray interrupt. This is more or less done as: 4505 * 4506 * for (ih = intrhand[intlev]; ih; ih = ih->ih_next) 4507 * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) 4508 * return; 4509 * strayintr(&frame); 4510 * 4511 * Inputs: 4512 * %l0 = %tstate 4513 * %l1 = return pc 4514 * %l2 = return npc 4515 * %l3 = interrupt level 4516 * (software interrupt only) %l4 = bits to clear in interrupt register 4517 * 4518 * Internal: 4519 * %l4, %l5: local variables 4520 * %l6 = %y 4521 * %l7 = %g1 4522 * %g2..%g7 go to stack 4523 * 4524 * An interrupt frame is built in the space for a full trapframe; 4525 * this contains the psr, pc, npc, and interrupt level. 4526 * 4527 * The level of this interrupt is determined by: 4528 * 4529 * IRQ# = %tt - 0x40 4530 */ 4531 4532 ENTRY_NOPROFILE(sparc_interrupt) 4533 #ifdef TRAPS_USE_IG 4534 ! This is for interrupt debugging 4535 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 4536 #endif 4537 /* 4538 * If this is a %tick or %stick softint, clear it then call 4539 * interrupt_vector. Only one of them should be enabled at any given 4540 * time. 4541 */ 4542 rd SOFTINT, %g1 4543 set TICK_INT|STICK_INT, %g5 4544 andcc %g5, %g1, %g5 4545 bz,pt %icc, 0f 4546 sethi %hi(CPUINFO_VA+CI_TICK_IH), %g3 4547 wr %g0, %g5, CLEAR_SOFTINT 4548 ba,pt %icc, setup_sparcintr 4549 LDPTR [%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5 4550 0: 4551 4552 #ifdef TRAPSTATS 4553 sethi %hi(_C_LABEL(kintrcnt)), %g1 4554 sethi %hi(_C_LABEL(uintrcnt)), %g2 4555 or %g1, %lo(_C_LABEL(kintrcnt)), %g1 4556 or %g1, %lo(_C_LABEL(uintrcnt)), %g2 4557 rdpr %tl, %g3 4558 dec %g3 4559 movrz %g3, %g2, %g1 4560 lduw [%g1], %g2 4561 inc %g2 4562 stw %g2, [%g1] 4563 /* See if we're on the interrupt stack already. */ 4564 set EINTSTACK, %g2 4565 set (EINTSTACK-INTSTACK), %g1 4566 btst 1, %sp 4567 add %sp, BIAS, %g3 4568 movz %icc, %sp, %g3 4569 srl %g3, 0, %g3 4570 sub %g2, %g3, %g3 4571 cmp %g3, %g1 4572 bgu 1f 4573 set _C_LABEL(intristk), %g1 4574 lduw [%g1], %g2 4575 inc %g2 4576 stw %g2, [%g1] 4577 1: 4578 #endif 4579 INTR_SETUP(-CC64FSZ-TF_SIZE) 4580 4581 ! Switch to normal globals so we can save them 4582 #ifdef SUN4V 4583 sethi %hi(cputyp), %g5 4584 ld [%g5 + %lo(cputyp)], %g5 4585 cmp %g5, CPU_SUN4V 4586 bne,pt %icc, 1f 4587 nop 4588 NORMAL_GLOBALS_SUN4V 4589 ! Save the normal globals 4590 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 4591 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 4592 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 4593 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 4594 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 4595 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 4596 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 4597 4598 /* 4599 * In the EMBEDANY memory model %g4 points to the start of the 4600 * data segment. In our case we need to clear it before calling 4601 * any C-code. 4602 */ 4603 clr %g4 4604 4605 ba 2f 4606 nop 4607 1: 4608 #endif 4609 NORMAL_GLOBALS_SUN4U 4610 ! Save the normal globals 4611 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 4612 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 4613 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 4614 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 4615 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 4616 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 4617 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 4618 4619 /* 4620 * In the EMBEDANY memory model %g4 points to the start of the 4621 * data segment. In our case we need to clear it before calling 4622 * any C-code. 4623 */ 4624 clr %g4 4625 4626 flushw ! Do not remove this insn -- causes interrupt loss 4627 4628 2: 4629 rd %y, %l6 4630 INCR64(CPUINFO_VA+CI_NINTR) ! cnt.v_ints++ (clobbers %o0,%o1) 4631 rdpr %tt, %l5 ! Find out our current IPL 4632 rdpr %tstate, %l0 4633 rdpr %tpc, %l1 4634 rdpr %tnpc, %l2 4635 rdpr %tl, %l3 ! Dump our trap frame now we have taken the IRQ 4636 stw %l6, [%sp + CC64FSZ + STKB + TF_Y] ! Silly, but we need to save this for rft 4637 dec %l3 4638 wrpr %g0, %l3, %tl 4639 sth %l5, [%sp + CC64FSZ + STKB + TF_TT]! debug 4640 stx %l0, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set up intrframe/clockframe 4641 stx %l1, [%sp + CC64FSZ + STKB + TF_PC] 4642 btst TSTATE_PRIV, %l0 ! User mode? 4643 stx %l2, [%sp + CC64FSZ + STKB + TF_NPC] 4644 4645 sub %l5, 0x40, %l6 ! Convert to interrupt level 4646 sethi %hi(_C_LABEL(intr_evcnts)), %l4 4647 stb %l6, [%sp + CC64FSZ + STKB + TF_PIL] ! set up intrframe/clockframe 4648 rdpr %pil, %o1 4649 mulx %l6, EVC_SIZE, %l3 4650 or %l4, %lo(_C_LABEL(intr_evcnts)), %l4 ! intrcnt[intlev]++; 4651 stb %o1, [%sp + CC64FSZ + STKB + TF_OLDPIL] ! old %pil 4652 ldx [%l4 + %l3], %o0 4653 add %l4, %l3, %l4 4654 clr %l5 ! Zero handled count 4655 #ifdef MULTIPROCESSOR 4656 mov 1, %l3 ! Ack softint 4657 1: add %o0, 1, %l7 4658 casxa [%l4] ASI_N, %o0, %l7 4659 cmp %o0, %l7 4660 bne,a,pn %xcc, 1b ! retry if changed 4661 mov %l7, %o0 4662 #else 4663 inc %o0 4664 mov 1, %l3 ! Ack softint 4665 stx %o0, [%l4] 4666 #endif 4667 sll %l3, %l6, %l3 ! Generate IRQ mask 4668 4669 wrpr %l6, %pil 4670 4671 #define SOFTINT_INT \ 4672 (1<<IPL_SOFTCLOCK|1<<IPL_SOFTBIO|1<<IPL_SOFTNET|1<<IPL_SOFTSERIAL) 4673 4674 ! Increment the per-cpu interrupt depth in case of hardintrs 4675 btst SOFTINT_INT, %l3 4676 bnz,pn %icc, sparc_intr_retry 4677 sethi %hi(CPUINFO_VA+CI_IDEPTH), %l1 4678 ld [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)], %l2 4679 inc %l2 4680 st %l2, [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)] 4681 4682 sparc_intr_retry: 4683 wr %l3, 0, CLEAR_SOFTINT ! (don't clear possible %tick IRQ) 4684 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %l4 4685 sll %l6, PTRSHFT, %l2 4686 or %l4, %lo(CPUINFO_VA+CI_INTRPENDING), %l4 4687 add %l2, %l4, %l4 4688 4689 1: 4690 membar #StoreLoad ! Make sure any failed casxa insns complete 4691 LDPTR [%l4], %l2 ! Check a slot 4692 cmp %l2, -1 4693 beq,pn CCCR, intrcmplt ! Empty list? 4694 mov -1, %l7 4695 membar #LoadStore 4696 CASPTRA [%l4] ASI_N, %l2, %l7 ! Grab the entire list 4697 cmp %l7, %l2 4698 bne,pn CCCR, 1b 4699 add %sp, CC64FSZ+STKB, %o2 ! tf = %sp + CC64FSZ + STKB 4700 LDPTR [%l2 + IH_PEND], %l7 4701 cmp %l7, -1 ! Last slot? 4702 be,pt CCCR, 3f 4703 membar #LoadStore 4704 4705 /* 4706 * Reverse a pending list since setup_sparcintr/send_softint 4707 * makes it in a LIFO order. 4708 */ 4709 mov -1, %o0 ! prev = -1 4710 1: STPTR %o0, [%l2 + IH_PEND] ! ih->ih_pending = prev 4711 mov %l2, %o0 ! prev = ih 4712 mov %l7, %l2 ! ih = ih->ih_pending 4713 LDPTR [%l2 + IH_PEND], %l7 4714 cmp %l7, -1 ! Last slot? 4715 bne,pn CCCR, 1b 4716 membar #LoadStore 4717 ba,pt CCCR, 3f 4718 mov %o0, %l7 ! save ih->ih_pending 4719 4720 2: 4721 add %sp, CC64FSZ+STKB, %o2 ! tf = %sp + CC64FSZ + STKB 4722 LDPTR [%l2 + IH_PEND], %l7 ! save ih->ih_pending 4723 membar #LoadStore 4724 3: 4725 STPTR %g0, [%l2 + IH_PEND] ! Clear pending flag 4726 membar #Sync 4727 LDPTR [%l2 + IH_FUN], %o4 ! ih->ih_fun 4728 LDPTR [%l2 + IH_ARG], %o0 ! ih->ih_arg 4729 4730 #ifdef NOT_DEBUG 4731 set _C_LABEL(intrdebug), %o3 4732 ld [%o2], %o3 4733 btst INTRDEBUG_FUNC, %o3 4734 bz,a,pt %icc, 97f 4735 nop 4736 4737 cmp %l6, 0xa ! ignore clock interrupts? 4738 bz,pt %icc, 97f 4739 nop 4740 4741 STACKFRAME(-CC64FSZ) ! Get a clean register window 4742 LOAD_ASCIZ(%o0, "sparc_interrupt: func %p arg %p\n") 4743 mov %i0, %o2 ! arg 4744 GLOBTOLOC 4745 call prom_printf 4746 mov %i4, %o1 ! func 4747 LOCTOGLOB 4748 restore 4749 97: 4750 mov %l4, %o1 4751 #endif 4752 4753 wrpr %g0, PSTATE_INTR, %pstate ! Reenable interrupts 4754 jmpl %o4, %o7 ! handled = (*ih->ih_fun)(...) 4755 movrz %o0, %o2, %o0 ! arg = (arg == 0) ? arg : tf 4756 wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts 4757 LDPTR [%l2 + IH_CLR], %l1 4758 membar #Sync 4759 4760 brz,pn %l1, 0f 4761 add %l5, %o0, %l5 4762 stx %g0, [%l1] ! Clear intr source 4763 membar #Sync ! Should not be needed 4764 0: 4765 LDPTR [%l2 + IH_ACK], %l1 ! ih->ih_ack 4766 brz,pn %l1, 1f 4767 nop 4768 jmpl %l1, %o7 ! (*ih->ih_ack)(ih) 4769 mov %l2, %o0 4770 1: 4771 cmp %l7, -1 4772 bne,pn CCCR, 2b ! 'Nother? 4773 mov %l7, %l2 4774 4775 intrcmplt: 4776 /* 4777 * Re-read SOFTINT to see if any new pending interrupts 4778 * at this level. 4779 */ 4780 mov 1, %l3 ! Ack softint 4781 rd SOFTINT, %l7 ! %l5 contains #intr handled. 4782 sll %l3, %l6, %l3 ! Generate IRQ mask 4783 btst %l3, %l7 ! leave mask in %l3 for retry code 4784 bnz,pn %icc, sparc_intr_retry 4785 mov 1, %l5 ! initialize intr count for next run 4786 4787 ! Decrement this cpu's interrupt depth in case of hardintrs 4788 btst SOFTINT_INT, %l3 4789 bnz,pn %icc, 1f 4790 sethi %hi(CPUINFO_VA+CI_IDEPTH), %l4 4791 ld [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5 4792 dec %l5 4793 st %l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)] 4794 1: 4795 4796 #ifdef NOT_DEBUG 4797 set _C_LABEL(intrdebug), %o2 4798 ld [%o2], %o2 4799 btst INTRDEBUG_FUNC, %o2 4800 bz,a,pt %icc, 97f 4801 nop 4802 4803 cmp %l6, 0xa ! ignore clock interrupts? 4804 bz,pt %icc, 97f 4805 nop 4806 4807 STACKFRAME(-CC64FSZ) ! Get a clean register window 4808 LOAD_ASCIZ(%o0, "sparc_interrupt: done\n") 4809 GLOBTOLOC 4810 call prom_printf 4811 nop 4812 LOCTOGLOB 4813 restore 4814 97: 4815 #endif 4816 4817 ldub [%sp + CC64FSZ + STKB + TF_OLDPIL], %l3 ! restore old %pil 4818 wrpr %l3, 0, %pil 4819 4820 b return_from_trap 4821 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 4822 4823 #ifdef notyet 4824 /* 4825 * Level 12 (ZS serial) interrupt. Handle it quickly, schedule a 4826 * software interrupt, and get out. Do the software interrupt directly 4827 * if we would just take it on the way out. 4828 * 4829 * Input: 4830 * %l0 = %psr 4831 * %l1 = return pc 4832 * %l2 = return npc 4833 * Internal: 4834 * %l3 = zs device 4835 * %l4, %l5 = temporary 4836 * %l6 = rr3 (or temporary data) + 0x100 => need soft int 4837 * %l7 = zs soft status 4838 */ 4839 zshard: 4840 #endif /* notyet */ 4841 4842 .globl return_from_trap, rft_kernel, rft_user 4843 .globl softtrap, slowtrap 4844 4845 /* 4846 * Various return-from-trap routines (see return_from_trap). 4847 */ 4848 4849 /* 4850 * Return from trap. 4851 * registers are: 4852 * 4853 * [%sp + CC64FSZ + STKB] => trap frame 4854 * %g1 => tstate from trap frame 4855 * 4856 * We must load all global, out, and trap registers from the trap frame. 4857 * 4858 * If returning to kernel, we should be at the proper trap level because 4859 * we don't touch %tl. 4860 * 4861 * When returning to user mode, the trap level does not matter, as it 4862 * will be set explicitly. 4863 * 4864 * If we are returning to user code, we must: 4865 * 1. Check for register windows in the pcb that belong on the stack. 4866 * If there are any, reload them 4867 */ 4868 return_from_trap: 4869 #ifdef DEBUG 4870 !! Make sure we don't have pc == npc == 0 or we suck. 4871 ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 4872 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 4873 orcc %g2, %g3, %g0 4874 tz %icc, 1 4875 #endif 4876 4877 !! 4878 !! We'll make sure we flush our pcb here, rather than later. 4879 !! 4880 ! ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! already passed in, no need to reload 4881 btst TSTATE_PRIV, %g1 ! returning to userland? 4882 4883 !! 4884 !! Let all pending interrupts drain before returning to userland 4885 !! 4886 bnz,pn %icc, 1f ! Returning to userland? 4887 nop 4888 ENABLE_INTERRUPTS %g5 4889 wrpr %g0, %g0, %pil ! Lower IPL 4890 1: 4891 !! Make sure we have no IRQs 4892 DISABLE_INTERRUPTS %g5 4893 4894 #ifdef SUN4V 4895 sethi %hi(cputyp), %g5 4896 ld [%g5 + %lo(cputyp)], %g5 4897 cmp %g5, CPU_SUN4V 4898 bne,pt %icc, 1f 4899 nop 4900 !! Make sure we have normal globals 4901 NORMAL_GLOBALS_SUN4V 4902 /* Restore normal globals */ 4903 ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 4904 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 4905 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 4906 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 4907 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 4908 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 4909 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 4910 /* Switch to alternate globals */ 4911 ALTERNATE_GLOBALS_SUN4V 4912 ba 2f 4913 nop 4914 1: 4915 #endif 4916 !! Make sure we have normal globals 4917 NORMAL_GLOBALS_SUN4U 4918 /* Restore normal globals */ 4919 ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 4920 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 4921 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 4922 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 4923 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 4924 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 4925 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 4926 /* Switch to alternate globals */ 4927 #ifdef TRAPS_USE_IG 4928 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 4929 #else 4930 ALTERNATE_GLOBALS_SUN4U 4931 #endif 4932 2: 4933 4934 /* Load outs */ 4935 ldx [%sp + CC64FSZ + STKB + TF_O + (0*8)], %i0 4936 ldx [%sp + CC64FSZ + STKB + TF_O + (1*8)], %i1 4937 ldx [%sp + CC64FSZ + STKB + TF_O + (2*8)], %i2 4938 ldx [%sp + CC64FSZ + STKB + TF_O + (3*8)], %i3 4939 ldx [%sp + CC64FSZ + STKB + TF_O + (4*8)], %i4 4940 ldx [%sp + CC64FSZ + STKB + TF_O + (5*8)], %i5 4941 ldx [%sp + CC64FSZ + STKB + TF_O + (6*8)], %i6 4942 ldx [%sp + CC64FSZ + STKB + TF_O + (7*8)], %i7 4943 /* Now load trap registers into alternate globals */ 4944 ld [%sp + CC64FSZ + STKB + TF_Y], %g4 4945 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! load new values 4946 wr %g4, 0, %y 4947 ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 4948 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 4949 4950 #ifdef NOTDEF_DEBUG 4951 ldub [%sp + CC64FSZ + STKB + TF_PIL], %g5 ! restore %pil 4952 wrpr %g5, %pil ! DEBUG 4953 #endif 4954 4955 /* Returning to user mode or kernel mode? */ 4956 btst TSTATE_PRIV, %g1 ! returning to userland? 4957 bz,pt %icc, rft_user 4958 sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! first instr of rft_user 4959 4960 /* 4961 * Return from trap, to kernel. 4962 * 4963 * We will assume, for the moment, that all kernel traps are properly stacked 4964 * in the trap registers, so all we have to do is insert the (possibly modified) 4965 * register values into the trap registers then do a retry. 4966 * 4967 */ 4968 rft_kernel: 4969 rdpr %tl, %g4 ! Grab a set of trap registers 4970 inc %g4 4971 wrpr %g4, %g0, %tl 4972 wrpr %g3, 0, %tnpc 4973 wrpr %g2, 0, %tpc 4974 wrpr %g1, 0, %tstate 4975 4976 rdpr %canrestore, %g2 4977 brnz %g2, 1f 4978 nop 4979 4980 wr %g0, ASI_NUCLEUS, %asi 4981 rdpr %cwp, %g1 4982 dec %g1 4983 wrpr %g1, %cwp 4984 #ifdef _LP64 4985 FILL ldxa, %sp+BIAS, 8, %asi 4986 #else 4987 FILL lda, %sp, 4, %asi 4988 #endif 4989 restored 4990 inc %g1 4991 wrpr %g1, %cwp 4992 1: 4993 restore 4994 rdpr %tstate, %g1 ! Since we may have trapped our regs may be toast 4995 rdpr %cwp, %g2 4996 andn %g1, CWP, %g1 4997 wrpr %g1, %g2, %tstate ! Put %cwp in %tstate 4998 CLRTT 4999 #ifdef TRAPSTATS 5000 rdpr %tl, %g2 5001 set _C_LABEL(rftkcnt), %g1 5002 sllx %g2, 2, %g2 5003 add %g1, %g2, %g1 5004 lduw [%g1], %g2 5005 inc %g2 5006 stw %g2, [%g1] 5007 #endif 5008 #if 0 5009 wrpr %g0, 0, %cleanwin ! DEBUG 5010 #endif 5011 #if defined(DDB) && defined(MULTIPROCESSOR) 5012 set sparc64_ipi_pause_trap_point, %g1 5013 rdpr %tpc, %g2 5014 cmp %g1, %g2 5015 bne,pt %icc, 0f 5016 nop 5017 done 5018 0: 5019 #endif 5020 retry 5021 NOTREACHED 5022 /* 5023 * Return from trap, to user. Checks for scheduling trap (`ast') first; 5024 * will re-enter trap() if set. Note that we may have to switch from 5025 * the interrupt stack to the kernel stack in this case. 5026 * %g1 = %tstate 5027 * %g2 = return %pc 5028 * %g3 = return %npc 5029 * If returning to a valid window, just set psr and return. 5030 */ 5031 .data 5032 rft_wcnt: .word 0 5033 .text 5034 5035 rft_user: 5036 ! sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! (done above) 5037 lduw [%g7 + %lo(CPUINFO_VA+CI_WANT_AST)], %g7! want AST trap? 5038 brnz,pn %g7, softtrap ! yes, re-enter trap with type T_AST 5039 mov T_AST, %g4 5040 5041 #ifdef NOTDEF_DEBUG 5042 sethi %hi(CPCB), %g4 5043 LDPTR [%g4 + %lo(CPCB)], %g4 5044 ldub [%g4 + PCB_NSAVED], %g4 ! nsaved 5045 brz,pt %g4, 2f ! Only print if nsaved <> 0 5046 nop 5047 5048 set 1f, %o0 5049 mov %g4, %o1 5050 mov %g2, %o2 ! pc 5051 wr %g0, ASI_DMMU, %asi ! restore the user context 5052 ldxa [CTX_SECONDARY] %asi, %o3 ! ctx 5053 GLOBTOLOC 5054 mov %g3, %o5 5055 call printf 5056 mov %i6, %o4 ! sp 5057 ! wrpr %g0, PSTATE_INTR, %pstate ! Allow IRQ service 5058 ! wrpr %g0, PSTATE_KERN, %pstate ! DenyIRQ service 5059 LOCTOGLOB 5060 1: 5061 .data 5062 .asciz "rft_user: nsaved=%x pc=%d ctx=%x sp=%x npc=%p\n" 5063 _ALIGN 5064 .text 5065 #endif 5066 5067 /* 5068 * NB: only need to do this after a cache miss 5069 */ 5070 #ifdef TRAPSTATS 5071 set _C_LABEL(rftucnt), %g6 5072 lduw [%g6], %g7 5073 inc %g7 5074 stw %g7, [%g6] 5075 #endif 5076 /* 5077 * Now check to see if any regs are saved in the pcb and restore them. 5078 * 5079 * Here we need to undo the damage caused by switching to a kernel 5080 * stack. 5081 * 5082 * We will use alternate globals %g4..%g7 because %g1..%g3 are used 5083 * by the data fault trap handlers and we don't want possible conflict. 5084 */ 5085 5086 sethi %hi(CPCB), %g6 5087 rdpr %otherwin, %g7 ! restore register window controls 5088 #ifdef DEBUG 5089 rdpr %canrestore, %g5 ! DEBUG 5090 tst %g5 ! DEBUG 5091 tnz %icc, 1; nop ! DEBUG 5092 ! mov %g0, %g5 ! There should be *NO* %canrestore 5093 add %g7, %g5, %g7 ! DEBUG 5094 #endif 5095 wrpr %g0, %g7, %canrestore 5096 LDPTR [%g6 + %lo(CPCB)], %g6 5097 wrpr %g0, 0, %otherwin 5098 5099 ldub [%g6 + PCB_NSAVED], %g7 ! Any saved reg windows? 5100 wrpr %g0, WSTATE_USER, %wstate ! Need to know where our sp points 5101 5102 #ifdef DEBUG 5103 set rft_wcnt, %g4 ! Keep track of all the windows we restored 5104 stw %g7, [%g4] 5105 #endif 5106 5107 brz,pt %g7, 5f ! No saved reg wins 5108 nop 5109 dec %g7 ! We can do this now or later. Move to last entry 5110 5111 #ifdef DEBUG 5112 rdpr %canrestore, %g4 ! DEBUG Make sure we've restored everything 5113 brnz,a,pn %g4, 0f ! DEBUG 5114 sir ! DEBUG we should NOT have any usable windows here 5115 0: ! DEBUG 5116 wrpr %g0, 5, %tl 5117 #endif 5118 rdpr %otherwin, %g4 5119 sll %g7, 7, %g5 ! calculate ptr into rw64 array 8*16 == 128 or 7 bits 5120 brz,pt %g4, 6f ! We should not have any user windows left 5121 add %g5, %g6, %g5 5122 5123 set 1f, %o0 5124 mov %g7, %o1 5125 mov %g4, %o2 5126 call printf 5127 wrpr %g0, PSTATE_KERN, %pstate 5128 set 2f, %o0 5129 call panic 5130 nop 5131 NOTREACHED 5132 .data 5133 1: .asciz "pcb_nsaved=%x and otherwin=%x\n" 5134 2: .asciz "rft_user\n" 5135 _ALIGN 5136 .text 5137 6: 5138 3: 5139 restored ! Load in the window 5140 restore ! This should not trap! 5141 ldx [%g5 + PCB_RW + ( 0*8)], %l0 ! Load the window from the pcb 5142 ldx [%g5 + PCB_RW + ( 1*8)], %l1 5143 ldx [%g5 + PCB_RW + ( 2*8)], %l2 5144 ldx [%g5 + PCB_RW + ( 3*8)], %l3 5145 ldx [%g5 + PCB_RW + ( 4*8)], %l4 5146 ldx [%g5 + PCB_RW + ( 5*8)], %l5 5147 ldx [%g5 + PCB_RW + ( 6*8)], %l6 5148 ldx [%g5 + PCB_RW + ( 7*8)], %l7 5149 5150 ldx [%g5 + PCB_RW + ( 8*8)], %i0 5151 ldx [%g5 + PCB_RW + ( 9*8)], %i1 5152 ldx [%g5 + PCB_RW + (10*8)], %i2 5153 ldx [%g5 + PCB_RW + (11*8)], %i3 5154 ldx [%g5 + PCB_RW + (12*8)], %i4 5155 ldx [%g5 + PCB_RW + (13*8)], %i5 5156 ldx [%g5 + PCB_RW + (14*8)], %i6 5157 ldx [%g5 + PCB_RW + (15*8)], %i7 5158 5159 #ifdef DEBUG 5160 stx %g0, [%g5 + PCB_RW + (14*8)] ! DEBUG mark that we've saved this one 5161 #endif 5162 5163 cmp %g5, %g6 5164 bgu,pt %xcc, 3b ! Next one? 5165 dec 8*16, %g5 5166 5167 stb %g0, [%g6 + PCB_NSAVED] ! Clear them out so we won't do this again 5168 GET_MAXCWP %g5 5169 add %g5, %g7, %g4 5170 dec 1, %g5 ! NWINDOWS-1-1 5171 wrpr %g5, 0, %cansave 5172 wrpr %g0, 0, %canrestore ! Make sure we have no freeloaders XXX 5173 wrpr %g0, WSTATE_USER, %wstate ! Save things to user space 5174 mov %g7, %g5 ! We already did one restore 5175 4: 5176 rdpr %canrestore, %g4 5177 inc %g4 5178 deccc %g5 5179 wrpr %g4, 0, %cleanwin ! Make *sure* we don't trap to cleanwin 5180 bge,a,pt %xcc, 4b ! return to starting regwin 5181 save %g0, %g0, %g0 ! This may force a datafault 5182 5183 #ifdef DEBUG 5184 wrpr %g0, 0, %tl 5185 #endif 5186 #ifdef TRAPSTATS 5187 set _C_LABEL(rftuld), %g5 5188 lduw [%g5], %g4 5189 inc %g4 5190 stw %g4, [%g5] 5191 #endif 5192 !! 5193 !! We can't take any save faults in here 'cause they will never be serviced 5194 !! 5195 5196 #ifdef DEBUG 5197 sethi %hi(CPCB), %g5 5198 LDPTR [%g5 + %lo(CPCB)], %g5 5199 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? 5200 tst %g5 5201 tnz %icc, 1; nop ! Debugger if we still have saved windows 5202 bne,a rft_user ! Try starting over again 5203 sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 5204 #endif 5205 /* 5206 * Set up our return trapframe so we can recover if we trap from here 5207 * on in. 5208 */ 5209 wrpr %g0, 1, %tl ! Set up the trap state 5210 wrpr %g2, 0, %tpc 5211 wrpr %g3, 0, %tnpc 5212 ba,pt %icc, 6f 5213 wrpr %g1, %g0, %tstate 5214 5215 5: 5216 /* 5217 * Set up our return trapframe so we can recover if we trap from here 5218 * on in. 5219 */ 5220 wrpr %g0, 1, %tl ! Set up the trap state 5221 wrpr %g2, 0, %tpc 5222 wrpr %g3, 0, %tnpc 5223 wrpr %g1, %g0, %tstate 5224 5225 /* 5226 * The restore instruction further down may cause the trap level 5227 * to exceed the maximum trap level on sun4v, so a manual fill 5228 * may be necessary. 5229 */ 5230 5231 #ifdef SUN4V 5232 sethi %hi(cputyp), %g5 5233 ld [%g5 + %lo(cputyp)], %g5 5234 cmp %g5, CPU_SUN4V 5235 bne,pt %icc, 1f 5236 nop 5237 5238 ! Only manual fill if the restore instruction will cause a fill trap 5239 rdpr %canrestore, %g5 5240 brnz %g5, 1f 5241 nop 5242 5243 ! Do a manual fill 5244 wr %g0, ASI_AIUS, %asi 5245 rdpr %cwp, %g4 5246 dec %g4 5247 wrpr %g4, 0, %cwp 5248 rft_user_fault_start: 5249 FILL ldxa, %sp+BIAS, 8, %asi 5250 rft_user_fault_end: 5251 restored 5252 inc %g4 5253 wrpr %g4, 0, %cwp 5254 1: 5255 #endif 5256 restore 5257 6: 5258 rdpr %canrestore, %g5 5259 wrpr %g5, 0, %cleanwin ! Force cleanup of kernel windows 5260 5261 #ifdef NOTDEF_DEBUG 5262 ldx [%g6 + CC64FSZ + STKB + TF_L + (0*8)], %g5! DEBUG -- get proper value for %l0 5263 cmp %l0, %g5 5264 be,a,pt %icc, 1f 5265 nop 5266 ! sir ! WATCHDOG 5267 set badregs, %g1 ! Save the suspect regs 5268 stw %l0, [%g1+(4*0)] 5269 stw %l1, [%g1+(4*1)] 5270 stw %l2, [%g1+(4*2)] 5271 stw %l3, [%g1+(4*3)] 5272 stw %l4, [%g1+(4*4)] 5273 stw %l5, [%g1+(4*5)] 5274 stw %l6, [%g1+(4*6)] 5275 stw %l7, [%g1+(4*7)] 5276 stw %i0, [%g1+(4*8)+(4*0)] 5277 stw %i1, [%g1+(4*8)+(4*1)] 5278 stw %i2, [%g1+(4*8)+(4*2)] 5279 stw %i3, [%g1+(4*8)+(4*3)] 5280 stw %i4, [%g1+(4*8)+(4*4)] 5281 stw %i5, [%g1+(4*8)+(4*5)] 5282 stw %i6, [%g1+(4*8)+(4*6)] 5283 stw %i7, [%g1+(4*8)+(4*7)] 5284 save 5285 inc %g7 5286 wrpr %g7, 0, %otherwin 5287 wrpr %g0, 0, %canrestore 5288 wrpr %g0, WSTATE_KERN, %wstate ! Need to know where our sp points 5289 set rft_wcnt, %g4 ! Restore nsaved before trapping 5290 sethi %hi(CPCB), %g6 5291 LDPTR [%g6 + %lo(CPCB)], %g6 5292 lduw [%g4], %g4 5293 stb %g4, [%g6 + PCB_NSAVED] 5294 ta 1 5295 sir 5296 .data 5297 badregs: 5298 .space 16*4 5299 .text 5300 1: 5301 #endif 5302 5303 rdpr %tstate, %g1 5304 rdpr %cwp, %g7 ! Find our cur window 5305 andn %g1, CWP, %g1 ! Clear it from %tstate 5306 wrpr %g1, %g7, %tstate ! Set %tstate with %cwp 5307 mov CTX_SECONDARY, %g1 ! Restore the user context 5308 GET_MMU_CONTEXTID %g4, %g1, %g3 5309 mov CTX_PRIMARY, %g2 5310 SET_MMU_CONTEXTID %g4, %g2, %g3 5311 sethi %hi(KERNBASE), %g7 ! Should not be needed due to retry 5312 membar #Sync ! Should not be needed due to retry 5313 flush %g7 ! Should not be needed due to retry 5314 5315 CLRTT 5316 #ifdef TRAPSTATS 5317 set _C_LABEL(rftudone), %g1 5318 lduw [%g1], %g2 5319 inc %g2 5320 stw %g2, [%g1] 5321 #endif 5322 #ifdef DEBUG 5323 sethi %hi(CPCB), %g5 5324 LDPTR [%g5 + %lo(CPCB)], %g5 5325 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? 5326 tst %g5 5327 tnz %icc, 1; nop ! Debugger if we still have saved windows! 5328 #endif 5329 wrpr %g0, 0, %pil ! Enable all interrupts 5330 retry 5331 5332 ! exported end marker for kernel gdb 5333 .globl _C_LABEL(endtrapcode) 5334 _C_LABEL(endtrapcode): 5335 5336 /* 5337 * Kernel entry point. 5338 * 5339 * The contract between bootloader and kernel is: 5340 * 5341 * %o0 OpenFirmware entry point, to keep Sun's updaters happy 5342 * %o1 Address of boot information vector (see bootinfo.h) 5343 * %o2 Length of the vector, in bytes 5344 * %o3 OpenFirmware entry point, to mimic Sun bootloader behavior 5345 * %o4 OpenFirmware, to meet earlier NetBSD kernels expectations 5346 */ 5347 .align 8 5348 start: 5349 dostart: 5350 /* 5351 * Startup. 5352 * 5353 * The Sun FCODE bootloader is nice and loads us where we want 5354 * to be. We have a full set of mappings already set up for us. 5355 * 5356 * I think we end up having an entire 16M allocated to us. 5357 * 5358 * We enter with the prom entry vector in %o0, dvec in %o1, 5359 * and the bootops vector in %o2. 5360 * 5361 * All we need to do is: 5362 * 5363 * 1: Save the prom vector 5364 * 5365 * 2: Create a decent stack for ourselves 5366 * 5367 * 3: Install the permanent 4MB kernel mapping 5368 * 5369 * 4: Call the C language initialization code 5370 * 5371 */ 5372 5373 /* 5374 * Set the psr into a known state: 5375 * Set supervisor mode, interrupt level >= 13, traps enabled 5376 */ 5377 wrpr %g0, 13, %pil 5378 wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate 5379 wr %g0, FPRS_FEF, %fprs ! Turn on FPU 5380 5381 /* 5382 * Step 2: Set up a v8-like stack if we need to 5383 */ 5384 5385 #ifdef _LP64 5386 btst 1, %sp 5387 bnz,pt %icc, 0f 5388 nop 5389 add %sp, -BIAS, %sp 5390 #else 5391 btst 1, %sp 5392 bz,pt %icc, 0f 5393 nop 5394 add %sp, BIAS, %sp 5395 #endif 5396 0: 5397 5398 call _C_LABEL(bootstrap) 5399 clr %g4 ! Clear data segment pointer 5400 5401 /* 5402 * Initialize the boot CPU. Basically: 5403 * 5404 * Locate the cpu_info structure for this CPU. 5405 * Establish a locked mapping for interrupt stack. 5406 * Switch to the initial stack. 5407 * Call the routine passed in in cpu_info->ci_spinup 5408 */ 5409 5410 #ifdef NO_VCACHE 5411 #define SUN4U_TTE_DATABITS SUN4U_TTE_L|SUN4U_TTE_CP|SUN4U_TTE_P|SUN4U_TTE_W 5412 #else 5413 #define SUN4U_TTE_DATABITS SUN4U_TTE_L|SUN4U_TTE_CP|SUN4U_TTE_CV|SUN4U_TTE_P|SUN4U_TTE_W 5414 #endif 5415 5416 5417 ENTRY_NOPROFILE(cpu_initialize) /* for cosmetic reasons - nicer backtrace */ 5418 5419 /* Cache the cputyp in %l6 for later use below */ 5420 sethi %hi(cputyp), %l6 5421 ld [%l6 + %lo(cputyp)], %l6 5422 5423 /* 5424 * Step 5: is no more. 5425 */ 5426 5427 /* 5428 * Step 6: hunt through cpus list and find the one that matches our cpuid 5429 */ 5430 5431 call _C_LABEL(cpu_myid) ! Retrieve cpuid in %o0 5432 mov %g0, %o0 5433 5434 sethi %hi(_C_LABEL(cpus)), %l1 5435 LDPTR [%l1 + %lo(_C_LABEL(cpus))], %l1 5436 0: 5437 ld [%l1 + CI_CPUID], %l3 ! Load CPUID 5438 cmp %l3, %o0 ! Does it match? 5439 bne,a,pt %icc, 0b ! no 5440 LDPTR [%l1 + CI_NEXT], %l1 ! Load next cpu_info pointer 5441 5442 /* 5443 * Get pointer to our cpu_info struct 5444 */ 5445 mov %l1, %l7 ! save cpu_info pointer 5446 ldx [%l1 + CI_PADDR], %l1 ! Load the interrupt stack's PA 5447 #ifdef SUN4V 5448 cmp %l6, CPU_SUN4V 5449 bne,pt %icc, 3f 5450 nop 5451 5452 /* sun4v */ 5453 call _C_LABEL(pmap_setup_intstack_sun4v) ! Call nice C function for mapping INTSTACK 5454 mov %l1, %o0 5455 ba 4f 5456 nop 5457 3: 5458 #endif 5459 /* sun4u */ 5460 sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 5461 sllx %l2, 32, %l2 ! Shift it into place 5462 5463 mov -1, %l3 ! Create a nice mask 5464 sllx %l3, 43, %l4 ! Mask off high bits 5465 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 5466 5467 andn %l1, %l4, %l1 ! Mask the phys page number 5468 5469 or %l2, %l1, %l1 ! Now take care of the high bits 5470 or %l1, SUN4U_TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 5471 5472 !! 5473 !! Now, map in the interrupt stack as context==0 5474 !! 5475 set TLB_TAG_ACCESS, %l5 5476 set INTSTACK, %l0 5477 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it 5478 stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it 5479 membar #Sync 5480 4: 5481 5482 !! Setup kernel stack (we rely on curlwp on this cpu 5483 !! being lwp0 here and its uarea is mapped special 5484 !! and already accessible here) 5485 flushw 5486 LDPTR [%l7 + CI_CPCB], %l0 ! load PCB/uarea pointer 5487 set 2*USPACE - TF_SIZE - CC64FSZ, %l1 5488 add %l1, %l0, %l0 5489 #ifdef _LP64 5490 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned 5491 sub %l0, BIAS, %l0 ! and biased 5492 #endif 5493 mov %l0, %sp 5494 flushw 5495 5496 #ifdef DEBUG 5497 set _C_LABEL(pmapdebug), %o1 5498 ld [%o1], %o1 5499 sethi %hi(0x40000), %o2 5500 btst %o2, %o1 5501 bz 0f 5502 5503 set 1f, %o0 ! Debug printf 5504 call _C_LABEL(prom_printf) 5505 nop 5506 .data 5507 1: 5508 .asciz "Setting trap base...\n" 5509 _ALIGN 5510 .text 5511 0: 5512 #endif 5513 /* 5514 * Step 7: change the trap base register, and install our TSB pointers 5515 */ 5516 5517 /* 5518 * install our TSB pointers 5519 */ 5520 5521 #ifdef SUN4V 5522 cmp %l6, CPU_SUN4V 5523 bne,pt %icc, 5f 5524 nop 5525 5526 /* sun4v */ 5527 LDPTR [%l7 + CI_TSB_DESC], %o0 5528 call _C_LABEL(pmap_setup_tsb_sun4v) 5529 nop 5530 ba 1f 5531 nop 5532 5: 5533 #endif 5534 /* sun4u */ 5535 sethi %hi(_C_LABEL(tsbsize)), %l2 5536 sethi %hi(0x1fff), %l3 5537 sethi %hi(TSB), %l4 5538 LDPTR [%l7 + CI_TSB_DMMU], %l0 5539 LDPTR [%l7 + CI_TSB_IMMU], %l1 5540 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 5541 or %l3, %lo(0x1fff), %l3 5542 or %l4, %lo(TSB), %l4 5543 5544 andn %l0, %l3, %l0 ! Mask off size and split bits 5545 or %l0, %l2, %l0 ! Make a TSB pointer 5546 stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer 5547 5548 andn %l1, %l3, %l1 ! Mask off size and split bits 5549 or %l1, %l2, %l1 ! Make a TSB pointer 5550 stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer 5551 membar #Sync 5552 set 1f, %l1 5553 flush %l1 5554 1: 5555 5556 /* set trap table */ 5557 #ifdef SUN4V 5558 cmp %l6, CPU_SUN4V 5559 bne,pt %icc, 6f 5560 nop 5561 /* sun4v */ 5562 set _C_LABEL(trapbase_sun4v), %l1 5563 GET_MMFSA %o1 5564 call _C_LABEL(prom_set_trap_table_sun4v) ! Now we should be running 100% from our handlers 5565 mov %l1, %o0 5566 5567 ba 7f 5568 nop 5569 6: 5570 #endif 5571 /* sun4u */ 5572 set _C_LABEL(trapbase), %l1 5573 call _C_LABEL(prom_set_trap_table_sun4u) ! Now we should be running 100% from our handlers 5574 mov %l1, %o0 5575 7: 5576 wrpr %l1, 0, %tba ! Make sure the PROM didn't foul up. 5577 5578 /* 5579 * Switch to the kernel mode and run away. 5580 */ 5581 wrpr %g0, WSTATE_KERN, %wstate 5582 5583 #ifdef DEBUG 5584 wrpr %g0, 1, %tl ! Debug -- start at tl==3 so we'll watchdog 5585 wrpr %g0, 0x1ff, %tt ! Debug -- clear out unused trap regs 5586 wrpr %g0, 0, %tpc 5587 wrpr %g0, 0, %tnpc 5588 wrpr %g0, 0, %tstate 5589 wrpr %g0, 0, %tl 5590 #endif 5591 5592 #ifdef DEBUG 5593 set _C_LABEL(pmapdebug), %o1 5594 ld [%o1], %o1 5595 sethi %hi(0x40000), %o2 5596 btst %o2, %o1 5597 bz 0f 5598 5599 LDPTR [%l7 + CI_SPINUP], %o1 5600 set 1f, %o0 ! Debug printf 5601 call _C_LABEL(prom_printf) 5602 mov %sp, %o2 5603 5604 .data 5605 1: 5606 .asciz "Calling startup routine %p with stack at %p...\n" 5607 _ALIGN 5608 .text 5609 0: 5610 #endif 5611 /* 5612 * Call our startup routine. 5613 */ 5614 5615 LDPTR [%l7 + CI_SPINUP], %o1 5616 5617 call %o1 ! Call routine 5618 clr %o0 ! our frame arg is ignored 5619 5620 set 1f, %o0 ! Main should never come back here 5621 call _C_LABEL(panic) 5622 nop 5623 .data 5624 1: 5625 .asciz "main() returned\n" 5626 _ALIGN 5627 .text 5628 5629 .align 8 5630 ENTRY(get_romtba) 5631 retl 5632 rdpr %tba, %o0 5633 5634 ENTRY(setcputyp) 5635 sethi %hi(cputyp), %o1 ! Trash %o1 assuming this is ok 5636 st %o0, [%o1 + %lo(cputyp)] 5637 retl 5638 nop 5639 5640 #ifdef MULTIPROCESSOR 5641 /* 5642 * cpu_mp_startup is called with: 5643 * 5644 * %g2 = cpu_args 5645 */ 5646 ENTRY(cpu_mp_startup) 5647 mov 1, %o0 5648 sllx %o0, 63, %o0 5649 wr %o0, TICK_CMPR ! XXXXXXX clear and disable %tick_cmpr for now 5650 wrpr %g0, 0, %cleanwin 5651 wrpr %g0, 0, %tl ! Make sure we're not in NUCLEUS mode 5652 wrpr %g0, WSTATE_KERN, %wstate 5653 wrpr %g0, PSTATE_KERN, %pstate 5654 flushw 5655 5656 /* Cache the cputyp in %l6 for later use below */ 5657 sethi %hi(cputyp), %l6 5658 ld [%l6 + %lo(cputyp)], %l6 5659 5660 /* 5661 * Get pointer to our cpu_info struct 5662 */ 5663 ldx [%g2 + CBA_CPUINFO], %l1 ! Load the interrupt stack's PA 5664 5665 #ifdef SUN4V 5666 cmp %l6, CPU_SUN4V 5667 bne,pt %icc, 3f 5668 nop 5669 5670 /* sun4v */ 5671 5672 sethi %hi(0x80000000), %l2 ! V=1|NFO=0|SW=0 5673 sllx %l2, 32, %l2 ! Shift it into place 5674 mov -1, %l3 ! Create a nice mask 5675 sllx %l3, 56, %l4 ! Mask off high 8 bits 5676 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 5677 andn %l1, %l4, %l1 ! Mask the phys page number into RA 5678 or %l2, %l1, %l1 ! Now take care of the 8 high bits V|NFO|SW 5679 or %l1, 0x0741, %l2 ! And low 13 bits IE=0|E=0|CP=1|CV=1|P=1| 5680 ! X=0|W=1|SW=00|SZ=0001 5681 5682 /* 5683 * Now, map in the interrupt stack & cpu_info as context==0 5684 */ 5685 5686 set INTSTACK, %o0 ! vaddr 5687 clr %o1 ! reserved 5688 mov %l2, %o2 ! tte 5689 mov MAP_DTLB, %o3 ! flags 5690 mov FT_MMU_MAP_PERM_ADDR, %o5 ! hv fast trap function 5691 ta ST_FAST_TRAP 5692 cmp %o0, 0 5693 be,pt %icc, 5f 5694 nop 5695 sir ! crash if mapping fails 5696 5: 5697 5698 /* 5699 * Set 0 as primary context XXX 5700 */ 5701 5702 mov CTX_PRIMARY, %o0 5703 SET_MMU_CONTEXTID_SUN4V %g0, %o0 5704 5705 ba 4f 5706 nop 5707 3: 5708 #endif 5709 5710 /* sun4u */ 5711 5712 sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 5713 sllx %l2, 32, %l2 ! Shift it into place 5714 mov -1, %l3 ! Create a nice mask 5715 sllx %l3, 43, %l4 ! Mask off high bits 5716 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 5717 andn %l1, %l4, %l1 ! Mask the phys page number 5718 or %l2, %l1, %l1 ! Now take care of the high bits 5719 or %l1, SUN4U_TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 5720 5721 /* 5722 * Now, map in the interrupt stack & cpu_info as context==0 5723 */ 5724 5725 set TLB_TAG_ACCESS, %l5 5726 set INTSTACK, %l0 5727 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it 5728 stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it 5729 5730 /* 5731 * Set 0 as primary context XXX 5732 */ 5733 5734 mov CTX_PRIMARY, %o0 5735 SET_MMU_CONTEXTID_SUN4U %g0, %o0 5736 5737 4: 5738 membar #Sync 5739 5740 /* 5741 * Temporarily use the interrupt stack 5742 */ 5743 #ifdef _LP64 5744 set ((EINTSTACK - CC64FSZ - TF_SIZE)) & ~0x0f - BIAS, %sp 5745 #else 5746 set EINTSTACK - CC64FSZ - TF_SIZE, %sp 5747 #endif 5748 set 1, %fp 5749 clr %i7 5750 5751 #ifdef SUN4V 5752 cmp %l6, CPU_SUN4V 5753 bne,pt %icc, 2f 5754 nop 5755 5756 /* sun4v */ 5757 5758 /* 5759 * install our TSB pointers 5760 */ 5761 5762 set CPUINFO_VA, %o0 5763 LDPTR [%o0 + CI_TSB_DESC], %o0 5764 call _C_LABEL(pmap_setup_tsb_sun4v) 5765 nop 5766 5767 /* set trap table */ 5768 5769 set _C_LABEL(trapbase_sun4v), %l1 5770 GET_MMFSA %o1 5771 call _C_LABEL(prom_set_trap_table_sun4v) 5772 mov %l1, %o0 5773 5774 ! Now we should be running 100% from our handlers 5775 ba 3f 5776 nop 5777 2: 5778 #endif 5779 /* sun4u */ 5780 5781 /* 5782 * install our TSB pointers 5783 */ 5784 5785 sethi %hi(CPUINFO_VA+CI_TSB_DMMU), %l0 5786 sethi %hi(CPUINFO_VA+CI_TSB_IMMU), %l1 5787 sethi %hi(_C_LABEL(tsbsize)), %l2 5788 sethi %hi(0x1fff), %l3 5789 sethi %hi(TSB), %l4 5790 LDPTR [%l0 + %lo(CPUINFO_VA+CI_TSB_DMMU)], %l0 5791 LDPTR [%l1 + %lo(CPUINFO_VA+CI_TSB_IMMU)], %l1 5792 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 5793 or %l3, %lo(0x1fff), %l3 5794 or %l4, %lo(TSB), %l4 5795 5796 andn %l0, %l3, %l0 ! Mask off size and split bits 5797 or %l0, %l2, %l0 ! Make a TSB pointer 5798 stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer 5799 membar #Sync 5800 5801 andn %l1, %l3, %l1 ! Mask off size and split bits 5802 or %l1, %l2, %l1 ! Make a TSB pointer 5803 stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer 5804 membar #Sync 5805 set 1f, %o0 5806 flush %o0 5807 1: 5808 5809 /* set trap table */ 5810 5811 set _C_LABEL(trapbase), %l1 5812 call _C_LABEL(prom_set_trap_table_sun4u) 5813 mov %l1, %o0 5814 3: 5815 wrpr %l1, 0, %tba ! Make sure the PROM didn't 5816 ! foul up. 5817 /* 5818 * Use this CPUs idlelewp's uarea stack 5819 */ 5820 sethi %hi(CPUINFO_VA+CI_IDLELWP), %l0 5821 LDPTR [%l0 + %lo(CPUINFO_VA+CI_IDLELWP)], %l0 5822 set USPACE - TF_SIZE - CC64FSZ, %l1 5823 LDPTR [%l0 + L_PCB], %l0 5824 add %l0, %l1, %l0 5825 #ifdef _LP64 5826 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned 5827 sub %l0, BIAS, %l0 ! and biased 5828 #endif 5829 mov %l0, %sp 5830 flushw 5831 5832 /* 5833 * Switch to the kernel mode and run away. 5834 */ 5835 wrpr %g0, 13, %pil 5836 wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate 5837 wr %g0, FPRS_FEF, %fprs ! Turn on FPU 5838 5839 call _C_LABEL(cpu_hatch) 5840 clr %g4 5841 5842 b _C_LABEL(idle_loop) 5843 clr %o0 5844 5845 NOTREACHED 5846 5847 .globl cpu_mp_startup_end 5848 cpu_mp_startup_end: 5849 #endif 5850 5851 /* 5852 * openfirmware(cell* param); 5853 * 5854 * OpenFirmware entry point 5855 * 5856 * If we're running in 32-bit mode we need to convert to a 64-bit stack 5857 * and 64-bit cells. The cells we'll allocate off the stack for simplicity. 5858 */ 5859 .align 8 5860 ENTRY(openfirmware) 5861 sethi %hi(romp), %o4 5862 andcc %sp, 1, %g0 5863 bz,pt %icc, 1f 5864 LDPTR [%o4+%lo(romp)], %o4 ! v9 stack, just load the addr and callit 5865 save %sp, -CC64FSZ, %sp 5866 rdpr %pil, %i2 5867 mov PIL_HIGH, %i3 5868 cmp %i3, %i2 5869 movle %icc, %i2, %i3 5870 wrpr %g0, %i3, %pil 5871 mov %i0, %o0 5872 mov %g1, %l1 5873 mov %g2, %l2 5874 mov %g3, %l3 5875 mov %g4, %l4 5876 mov %g5, %l5 5877 mov %g6, %l6 5878 mov %g7, %l7 5879 rdpr %pstate, %l0 5880 jmpl %i4, %o7 5881 #if !defined(_LP64) 5882 wrpr %g0, PSTATE_PROM, %pstate 5883 #else 5884 wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate 5885 #endif 5886 wrpr %l0, %g0, %pstate 5887 mov %l1, %g1 5888 mov %l2, %g2 5889 mov %l3, %g3 5890 mov %l4, %g4 5891 mov %l5, %g5 5892 mov %l6, %g6 5893 mov %l7, %g7 5894 wrpr %i2, 0, %pil 5895 ret 5896 restore %o0, %g0, %o0 5897 5898 1: ! v8 -- need to screw with stack & params 5899 #ifdef NOTDEF_DEBUG 5900 mov %o7, %o5 5901 call globreg_check 5902 nop 5903 mov %o5, %o7 5904 #endif 5905 save %sp, -CC64FSZ, %sp ! Get a new 64-bit stack frame 5906 add %sp, -BIAS, %sp 5907 rdpr %pstate, %l0 5908 srl %sp, 0, %sp 5909 rdpr %pil, %i2 ! s = splx(level) 5910 mov %i0, %o0 5911 mov PIL_HIGH, %i3 5912 mov %g1, %l1 5913 mov %g2, %l2 5914 cmp %i3, %i2 5915 mov %g3, %l3 5916 mov %g4, %l4 5917 mov %g5, %l5 5918 movle %icc, %i2, %i3 5919 mov %g6, %l6 5920 mov %g7, %l7 5921 wrpr %i3, %g0, %pil 5922 jmpl %i4, %o7 5923 ! Enable 64-bit addresses for the prom 5924 #if defined(_LP64) 5925 wrpr %g0, PSTATE_PROM, %pstate 5926 #else 5927 wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate 5928 #endif 5929 wrpr %l0, 0, %pstate 5930 wrpr %i2, 0, %pil 5931 mov %l1, %g1 5932 mov %l2, %g2 5933 mov %l3, %g3 5934 mov %l4, %g4 5935 mov %l5, %g5 5936 mov %l6, %g6 5937 mov %l7, %g7 5938 ret 5939 restore %o0, %g0, %o0 5940 5941 /* 5942 * void ofw_exit(cell_t args[]) 5943 */ 5944 ENTRY(openfirmware_exit) 5945 STACKFRAME(-CC64FSZ) 5946 flushw ! Flush register windows 5947 5948 wrpr %g0, PIL_HIGH, %pil ! Disable interrupts 5949 sethi %hi(romtba), %l5 5950 LDPTR [%l5 + %lo(romtba)], %l5 5951 wrpr %l5, 0, %tba ! restore the ofw trap table 5952 5953 /* Arrange locked kernel stack as PROM stack */ 5954 set EINTSTACK - CC64FSZ, %l5 5955 5956 andn %l5, 0x0f, %l5 ! Needs to be 16-byte aligned 5957 sub %l5, BIAS, %l5 ! and biased 5958 mov %l5, %sp 5959 flushw 5960 5961 sethi %hi(romp), %l6 5962 LDPTR [%l6 + %lo(romp)], %l6 5963 5964 mov CTX_PRIMARY, %l3 ! set context 0 5965 stxa %g0, [%l3] ASI_DMMU 5966 membar #Sync 5967 5968 wrpr %g0, PSTATE_PROM, %pstate ! Disable interrupts 5969 ! and enable 64-bit addresses 5970 wrpr %g0, 0, %tl ! force trap level 0 5971 call %l6 5972 mov %i0, %o0 5973 NOTREACHED 5974 5975 /* 5976 * sp_tlb_flush_pte_us(vaddr_t va, int ctx) 5977 * sp_tlb_flush_pte_usiii(vaddr_t va, int ctx) 5978 * 5979 * Flush tte from both IMMU and DMMU. 5980 * 5981 * This uses %o0-%o5 5982 */ 5983 .align 8 5984 ENTRY(sp_tlb_flush_pte_us) 5985 #ifdef DEBUG 5986 set pmapdebug, %o3 5987 lduw [%o3], %o3 5988 ! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 5989 btst 0x0020, %o3 5990 bz,pt %icc, 2f 5991 nop 5992 save %sp, -CC64FSZ, %sp 5993 set 1f, %o0 5994 mov %i1, %o1 5995 andn %i0, 0xfff, %o3 5996 or %o3, 0x010, %o3 5997 call _C_LABEL(printf) 5998 mov %i0, %o2 5999 restore 6000 .data 6001 1: 6002 .asciz "sp_tlb_flush_pte_us: demap ctx=%x va=%08x res=%x\n" 6003 _ALIGN 6004 .text 6005 2: 6006 #endif 6007 #ifdef MULTIPROCESSOR 6008 rdpr %pstate, %o3 6009 andn %o3, PSTATE_IE, %o4 ! disable interrupts 6010 wrpr %o4, 0, %pstate 6011 #endif 6012 srlx %o0, PG_SHIFT4U, %o0 ! drop unused va bits 6013 mov CTX_SECONDARY, %o2 6014 sllx %o0, PG_SHIFT4U, %o0 6015 ldxa [%o2] ASI_DMMU, %o5 ! Save secondary context 6016 sethi %hi(KERNBASE), %o4 6017 membar #LoadStore 6018 stxa %o1, [%o2] ASI_DMMU ! Insert context to demap 6019 membar #Sync 6020 or %o0, DEMAP_PAGE_SECONDARY, %o0 ! Demap page from secondary context only 6021 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6022 stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs 6023 #ifdef TLB_FLUSH_LOWVA 6024 srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries 6025 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6026 stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap 6027 #endif 6028 flush %o4 6029 stxa %o5, [%o2] ASI_DMMU ! Restore secondary context 6030 membar #Sync 6031 retl 6032 #ifdef MULTIPROCESSOR 6033 wrpr %o3, %pstate ! restore interrupts 6034 #else 6035 nop 6036 #endif 6037 6038 ENTRY(sp_tlb_flush_pte_usiii) 6039 #ifdef DEBUG 6040 set pmapdebug, %o3 6041 lduw [%o3], %o3 6042 ! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 6043 btst 0x0020, %o3 6044 bz,pt %icc, 2f 6045 nop 6046 save %sp, -CC64FSZ, %sp 6047 set 1f, %o0 6048 mov %i1, %o1 6049 andn %i0, 0xfff, %o3 6050 or %o3, 0x010, %o3 6051 call _C_LABEL(printf) 6052 mov %i0, %o2 6053 restore 6054 .data 6055 1: 6056 .asciz "sp_tlb_flush_pte_usiii: demap ctx=%x va=%08x res=%x\n" 6057 _ALIGN 6058 .text 6059 2: 6060 #endif 6061 ! %o0 = VA [in] 6062 ! %o1 = ctx value [in] / KERNBASE 6063 ! %o2 = CTX_PRIMARY 6064 ! %o3 = saved %tl 6065 ! %o4 = saved %pstate 6066 ! %o5 = saved primary ctx 6067 6068 ! Need this for UP as well 6069 rdpr %pstate, %o4 6070 andn %o4, PSTATE_IE, %o3 ! disable interrupts 6071 wrpr %o3, 0, %pstate 6072 6073 !! 6074 !! Cheetahs do not support flushing the IMMU from secondary context 6075 !! 6076 rdpr %tl, %o3 6077 mov CTX_PRIMARY, %o2 6078 brnz,pt %o3, 1f 6079 andn %o0, 0xfff, %o0 ! drop unused va bits 6080 wrpr %g0, 1, %tl ! Make sure we're NUCLEUS 6081 1: 6082 ldxa [%o2] ASI_DMMU, %o5 ! Save primary context 6083 membar #LoadStore 6084 stxa %o1, [%o2] ASI_DMMU ! Insert context to demap 6085 sethi %hi(KERNBASE), %o1 6086 membar #Sync 6087 or %o0, DEMAP_PAGE_PRIMARY, %o0 6088 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6089 membar #Sync 6090 stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs 6091 membar #Sync 6092 #ifdef TLB_FLUSH_LOWVA 6093 srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries 6094 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6095 membar #Sync 6096 stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap 6097 membar #Sync 6098 #endif 6099 flush %o1 6100 stxa %o5, [%o2] ASI_DMMU ! Restore primary context 6101 membar #Sync 6102 brnz,pt %o3, 1f 6103 flush %o1 6104 wrpr %g0, %o3, %tl ! Return to kernel mode. 6105 1: 6106 retl 6107 wrpr %o4, %pstate ! restore interrupts 6108 6109 6110 /* 6111 * sp_tlb_flush_all_us(void) 6112 * sp_tlb_flush_all_usiii(void) 6113 * 6114 * Flush all user TLB entries from both IMMU and DMMU. 6115 * We have both UltraSPARC I+II, and UltraSPARC >=III versions. 6116 */ 6117 .align 8 6118 ENTRY(sp_tlb_flush_all_us) 6119 rdpr %pstate, %o3 6120 andn %o3, PSTATE_IE, %o4 ! disable interrupts 6121 wrpr %o4, 0, %pstate 6122 set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 6123 set CTX_SECONDARY, %o4 6124 ldxa [%o4] ASI_DMMU, %o4 ! save secondary context 6125 set CTX_MASK, %o5 6126 membar #Sync 6127 6128 ! %o0 = loop counter 6129 ! %o1 = ctx value 6130 ! %o2 = TLB tag value 6131 ! %o3 = saved %pstate 6132 ! %o4 = saved primary ctx 6133 ! %o5 = CTX_MASK 6134 ! %xx = saved %tl 6135 6136 0: 6137 ldxa [%o0] ASI_DMMU_TLB_TAG, %o2 ! fetch the TLB tag 6138 andcc %o2, %o5, %o1 ! context 0? 6139 bz,pt %xcc, 1f ! if so, skip 6140 mov CTX_SECONDARY, %o2 6141 6142 stxa %o1, [%o2] ASI_DMMU ! set the context 6143 set DEMAP_CTX_SECONDARY, %o2 6144 membar #Sync 6145 stxa %o2, [%o2] ASI_DMMU_DEMAP ! do the demap 6146 membar #Sync 6147 6148 1: 6149 dec 8, %o0 6150 brgz,pt %o0, 0b ! loop over all entries 6151 nop 6152 6153 /* 6154 * now do the IMMU 6155 */ 6156 6157 set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 6158 6159 0: 6160 ldxa [%o0] ASI_IMMU_TLB_TAG, %o2 ! fetch the TLB tag 6161 andcc %o2, %o5, %o1 ! context 0? 6162 bz,pt %xcc, 1f ! if so, skip 6163 mov CTX_SECONDARY, %o2 6164 6165 stxa %o1, [%o2] ASI_DMMU ! set the context 6166 set DEMAP_CTX_SECONDARY, %o2 6167 membar #Sync 6168 stxa %o2, [%o2] ASI_IMMU_DEMAP ! do the demap 6169 membar #Sync 6170 6171 1: 6172 dec 8, %o0 6173 brgz,pt %o0, 0b ! loop over all entries 6174 nop 6175 6176 set CTX_SECONDARY, %o2 6177 stxa %o4, [%o2] ASI_DMMU ! restore secondary ctx 6178 sethi %hi(KERNBASE), %o4 6179 membar #Sync 6180 flush %o4 6181 retl 6182 wrpr %o3, %pstate 6183 6184 .align 8 6185 ENTRY(sp_tlb_flush_all_usiii) 6186 rdpr %tl, %o5 6187 brnz,pt %o5, 1f 6188 set DEMAP_ALL, %o2 6189 wrpr 1, %tl 6190 1: 6191 rdpr %pstate, %o3 6192 andn %o3, PSTATE_IE, %o4 ! disable interrupts 6193 wrpr %o4, 0, %pstate 6194 6195 stxa %o2, [%o2] ASI_IMMU_DEMAP 6196 membar #Sync 6197 stxa %o2, [%o2] ASI_DMMU_DEMAP 6198 6199 sethi %hi(KERNBASE), %o4 6200 membar #Sync 6201 flush %o4 6202 6203 wrpr %o5, %tl 6204 retl 6205 wrpr %o3, %pstate 6206 6207 /* 6208 * sp_blast_dcache(int dcache_size, int dcache_line_size) 6209 * sp_blast_dcache_disabled(int dcache_size, int dcache_line_size) 6210 * 6211 * Clear out all of D$ regardless of contents. The latter one also 6212 * disables the D$ while doing so. 6213 */ 6214 .align 8 6215 ENTRY(sp_blast_dcache) 6216 /* 6217 * We turn off interrupts for the duration to prevent RED exceptions. 6218 */ 6219 #ifdef PROF 6220 save %sp, -CC64FSZ, %sp 6221 #endif 6222 6223 rdpr %pstate, %o3 6224 sub %o0, %o1, %o0 6225 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6226 wrpr %o4, 0, %pstate 6227 1: 6228 stxa %g0, [%o0] ASI_DCACHE_TAG 6229 membar #Sync 6230 brnz,pt %o0, 1b 6231 sub %o0, %o1, %o0 6232 6233 sethi %hi(KERNBASE), %o2 6234 flush %o2 6235 membar #Sync 6236 #ifdef PROF 6237 wrpr %o3, %pstate 6238 ret 6239 restore 6240 #else 6241 retl 6242 wrpr %o3, %pstate 6243 #endif 6244 6245 .align 8 6246 ENTRY(sp_blast_dcache_disabled) 6247 /* 6248 * We turn off interrupts for the duration to prevent RED exceptions. 6249 */ 6250 #ifdef PROF 6251 save %sp, -CC64FSZ, %sp 6252 #endif 6253 6254 rdpr %pstate, %o3 6255 sub %o0, %o1, %o0 6256 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6257 wrpr %o4, 0, %pstate 6258 6259 ldxa [%g0] ASI_MCCR, %o5 6260 andn %o5, MCCR_DCACHE_EN, %o4 ! Turn off the D$ 6261 stxa %o4, [%g0] ASI_MCCR 6262 flush %g0 6263 6264 1: 6265 stxa %g0, [%o0] ASI_DCACHE_TAG 6266 membar #Sync 6267 brnz,pt %o0, 1b 6268 sub %o0, %o1, %o0 6269 6270 sethi %hi(KERNBASE), %o2 6271 flush %o2 6272 membar #Sync 6273 6274 stxa %o5, [%g0] ASI_MCCR ! Restore the D$ 6275 flush %g0 6276 #ifdef PROF 6277 wrpr %o3, %pstate 6278 ret 6279 restore 6280 #else 6281 retl 6282 wrpr %o3, %pstate 6283 #endif 6284 6285 #ifdef MULTIPROCESSOR 6286 /* 6287 * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size) 6288 * 6289 * Clear out all of D$ regardless of contents 6290 * 6291 * On entry: 6292 * %g2 = dcache_size 6293 * %g3 = dcache_line_size 6294 */ 6295 .align 8 6296 ENTRY(sparc64_ipi_blast_dcache) 6297 sub %g2, %g3, %g2 6298 1: 6299 stxa %g0, [%g2] ASI_DCACHE_TAG 6300 membar #Sync 6301 brnz,pt %g2, 1b 6302 sub %g2, %g3, %g2 6303 6304 sethi %hi(KERNBASE), %g5 6305 flush %g5 6306 membar #Sync 6307 6308 ba,a ret_from_intr_vector 6309 nop 6310 #endif /* MULTIPROCESSOR */ 6311 6312 /* 6313 * blast_icache_us() 6314 * blast_icache_usiii() 6315 * 6316 * Clear out all of I$ regardless of contents 6317 * Does not modify %o0 6318 * 6319 * We turn off interrupts for the duration to prevent RED exceptions. 6320 * For the Cheetah version, we also have to to turn off the I$ during this as 6321 * ASI_ICACHE_TAG accesses interfere with coherency. 6322 */ 6323 .align 8 6324 ENTRY(blast_icache_us) 6325 rdpr %pstate, %o3 6326 sethi %hi(icache_size), %o1 6327 ld [%o1 + %lo(icache_size)], %o1 6328 sethi %hi(icache_line_size), %o2 6329 ld [%o2 + %lo(icache_line_size)], %o2 6330 sub %o1, %o2, %o1 6331 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6332 wrpr %o4, 0, %pstate 6333 1: 6334 stxa %g0, [%o1] ASI_ICACHE_TAG 6335 brnz,pt %o1, 1b 6336 sub %o1, %o2, %o1 6337 sethi %hi(KERNBASE), %o5 6338 flush %o5 6339 membar #Sync 6340 retl 6341 wrpr %o3, %pstate 6342 6343 .align 8 6344 ENTRY(blast_icache_usiii) 6345 rdpr %pstate, %o3 6346 sethi %hi(icache_size), %o1 6347 ld [%o1 + %lo(icache_size)], %o1 6348 sethi %hi(icache_line_size), %o2 6349 ld [%o2 + %lo(icache_line_size)], %o2 6350 sub %o1, %o2, %o1 6351 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6352 wrpr %o4, 0, %pstate 6353 ldxa [%g0] ASI_MCCR, %o5 6354 andn %o5, MCCR_ICACHE_EN, %o4 ! Turn off the I$ 6355 stxa %o4, [%g0] ASI_MCCR 6356 flush %g0 6357 1: 6358 stxa %g0, [%o1] ASI_ICACHE_TAG 6359 membar #Sync 6360 brnz,pt %o1, 1b 6361 sub %o1, %o2, %o1 6362 stxa %o5, [%g0] ASI_MCCR ! Restore the I$ 6363 flush %g0 6364 retl 6365 wrpr %o3, %pstate 6366 6367 /* 6368 * dcache_flush_page_us(paddr_t pa) 6369 * dcache_flush_page_usiii(paddr_t pa) 6370 * 6371 * Clear one page from D$. 6372 * 6373 */ 6374 .align 8 6375 ENTRY(dcache_flush_page_us) 6376 #ifndef _LP64 6377 COMBINE(%o0, %o1, %o0) 6378 #endif 6379 mov -1, %o1 ! Generate mask for tag: bits [29..2] 6380 srlx %o0, 13-2, %o2 ! Tag is PA bits <40:13> in bits <29:2> 6381 clr %o4 6382 srl %o1, 2, %o1 ! Now we have bits <29:0> set 6383 set (2*NBPG), %o5 6384 ba,pt %icc, 1f 6385 andn %o1, 3, %o1 ! Now we have bits <29:2> set 6386 6387 .align 8 6388 1: 6389 ldxa [%o4] ASI_DCACHE_TAG, %o3 6390 mov %o4, %o0 6391 deccc 32, %o5 6392 bl,pn %icc, 2f 6393 inc 32, %o4 6394 6395 xor %o3, %o2, %o3 6396 andcc %o3, %o1, %g0 6397 bne,pt %xcc, 1b 6398 membar #LoadStore 6399 6400 stxa %g0, [%o0] ASI_DCACHE_TAG 6401 ba,pt %icc, 1b 6402 membar #StoreLoad 6403 2: 6404 6405 sethi %hi(KERNBASE), %o5 6406 flush %o5 6407 retl 6408 membar #Sync 6409 6410 .align 8 6411 ENTRY(dcache_flush_page_usiii) 6412 #ifndef _LP64 6413 COMBINE(%o0, %o1, %o0) 6414 #endif 6415 set NBPG, %o1 6416 sethi %hi(dcache_line_size), %o2 6417 add %o0, %o1, %o1 ! end address 6418 ld [%o2 + %lo(dcache_line_size)], %o2 6419 6420 1: 6421 stxa %g0, [%o0] ASI_DCACHE_INVALIDATE 6422 add %o0, %o2, %o0 6423 cmp %o0, %o1 6424 bl,pt %xcc, 1b 6425 nop 6426 6427 sethi %hi(KERNBASE), %o5 6428 flush %o5 6429 retl 6430 membar #Sync 6431 6432 /* 6433 * cache_flush_phys_us(paddr_t, psize_t, int); 6434 * cache_flush_phys_usiii(paddr_t, psize_t, int); 6435 * 6436 * Clear a set of paddrs from the D$, I$ and if param3 is 6437 * non-zero, E$. (E$ is not supported yet). 6438 */ 6439 6440 .align 8 6441 ENTRY(cache_flush_phys_us) 6442 #ifndef _LP64 6443 COMBINE(%o0, %o1, %o0) 6444 COMBINE(%o2, %o3, %o1) 6445 mov %o4, %o2 6446 #endif 6447 #ifdef DEBUG 6448 tst %o2 ! Want to clear E$? 6449 tnz 1 ! Error! 6450 #endif 6451 add %o0, %o1, %o1 ! End PA 6452 dec %o1 6453 6454 !! 6455 !! Both D$ and I$ tags match pa bits 42-13, but 6456 !! they are shifted different amounts. So we'll 6457 !! generate a mask for bits 40-13. 6458 !! 6459 6460 mov -1, %o2 ! Generate mask for tag: bits [40..13] 6461 srl %o2, 5, %o2 ! 32-5 = [27..0] 6462 sllx %o2, 13, %o2 ! 27+13 = [40..13] 6463 6464 and %o2, %o0, %o0 ! Mask away uninteresting bits 6465 and %o2, %o1, %o1 ! (probably not necessary) 6466 6467 set (2*NBPG), %o5 6468 clr %o4 6469 1: 6470 ldxa [%o4] ASI_DCACHE_TAG, %o3 6471 sllx %o3, 40-29, %o3 ! Shift D$ tag into place 6472 and %o3, %o2, %o3 ! Mask out trash 6473 6474 cmp %o0, %o3 6475 blt,pt %xcc, 2f ! Too low 6476 cmp %o1, %o3 6477 bgt,pt %xcc, 2f ! Too high 6478 nop 6479 6480 membar #LoadStore 6481 stxa %g0, [%o4] ASI_DCACHE_TAG ! Just right 6482 membar #Sync 6483 2: 6484 ldda [%o4] ASI_ICACHE_TAG, %g0 ! Tag goes in %g1 6485 sllx %g1, 40-35, %g1 ! Shift I$ tag into place 6486 and %g1, %o2, %g1 ! Mask out trash 6487 cmp %o0, %g1 6488 blt,pt %xcc, 3f 6489 cmp %o1, %g1 6490 bgt,pt %xcc, 3f 6491 nop 6492 stxa %g0, [%o4] ASI_ICACHE_TAG 6493 3: 6494 membar #StoreLoad 6495 dec 32, %o5 6496 brgz,pt %o5, 1b 6497 inc 32, %o4 6498 6499 sethi %hi(KERNBASE), %o5 6500 flush %o5 6501 retl 6502 membar #Sync 6503 6504 .align 8 6505 ENTRY(cache_flush_phys_usiii) 6506 #ifndef _LP64 6507 COMBINE(%o0, %o1, %o0) 6508 COMBINE(%o2, %o3, %o1) 6509 mov %o4, %o2 6510 #endif 6511 #ifdef DEBUG 6512 tst %o2 ! Want to clear E$? 6513 tnz 1 ! Error! 6514 #endif 6515 add %o0, %o1, %o1 ! End PA 6516 sethi %hi(dcache_line_size), %o3 6517 ld [%o3 + %lo(dcache_line_size)], %o3 6518 sethi %hi(KERNBASE), %o5 6519 1: 6520 stxa %g0, [%o0] ASI_DCACHE_INVALIDATE 6521 add %o0, %o3, %o0 6522 cmp %o0, %o1 6523 bl,pt %xcc, 1b 6524 nop 6525 6526 /* don't need to flush the I$ on cheetah */ 6527 6528 flush %o5 6529 retl 6530 membar #Sync 6531 6532 #ifdef COMPAT_16 6533 #ifdef _LP64 6534 /* 6535 * XXXXX Still needs lotsa cleanup after sendsig is complete and offsets are known 6536 * 6537 * The following code is copied to the top of the user stack when each 6538 * process is exec'ed, and signals are `trampolined' off it. 6539 * 6540 * When this code is run, the stack looks like: 6541 * [%sp] 128 bytes to which registers can be dumped 6542 * [%sp + 128] signal number (goes in %o0) 6543 * [%sp + 128 + 4] signal code (goes in %o1) 6544 * [%sp + 128 + 8] first word of saved state (sigcontext) 6545 * . 6546 * . 6547 * . 6548 * [%sp + NNN] last word of saved state 6549 * (followed by previous stack contents or top of signal stack). 6550 * The address of the function to call is in %g1; the old %g1 and %o0 6551 * have already been saved in the sigcontext. We are running in a clean 6552 * window, all previous windows now being saved to the stack. 6553 * 6554 * Note that [%sp + 128 + 8] == %sp + 128 + 16. The copy at %sp+128+8 6555 * will eventually be removed, with a hole left in its place, if things 6556 * work out. 6557 */ 6558 ENTRY_NOPROFILE(sigcode) 6559 /* 6560 * XXX the `save' and `restore' below are unnecessary: should 6561 * replace with simple arithmetic on %sp 6562 * 6563 * Make room on the stack for 64 %f registers + %fsr. This comes 6564 * out to 64*4+8 or 264 bytes, but this must be aligned to a multiple 6565 * of 64, or 320 bytes. 6566 */ 6567 save %sp, -CC64FSZ - 320, %sp 6568 mov %g2, %l2 ! save globals in %l registers 6569 mov %g3, %l3 6570 mov %g4, %l4 6571 mov %g5, %l5 6572 mov %g6, %l6 6573 mov %g7, %l7 6574 /* 6575 * Saving the fpu registers is expensive, so do it iff it is 6576 * enabled and dirty. 6577 */ 6578 rd %fprs, %l0 6579 btst FPRS_DL|FPRS_DU, %l0 ! All clean? 6580 bz,pt %icc, 2f 6581 btst FPRS_DL, %l0 ! test dl 6582 bz,pt %icc, 1f 6583 btst FPRS_DU, %l0 ! test du 6584 6585 ! fpu is enabled, oh well 6586 stx %fsr, [%sp + CC64FSZ + BIAS + 0] 6587 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6588 andn %l0, BLOCK_ALIGN, %l0 ! do a block store 6589 stda %f0, [%l0] ASI_BLK_P 6590 inc BLOCK_SIZE, %l0 6591 stda %f16, [%l0] ASI_BLK_P 6592 1: 6593 bz,pt %icc, 2f 6594 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6595 andn %l0, BLOCK_ALIGN, %l0 ! do a block store 6596 add %l0, 2*BLOCK_SIZE, %l0 ! and skip what we already stored 6597 stda %f32, [%l0] ASI_BLK_P 6598 inc BLOCK_SIZE, %l0 6599 stda %f48, [%l0] ASI_BLK_P 6600 2: 6601 membar #Sync 6602 rd %fprs, %l0 ! reload fprs copy, for checking after 6603 rd %y, %l1 ! in any case, save %y 6604 lduw [%fp + BIAS + 128], %o0 ! sig 6605 lduw [%fp + BIAS + 128 + 4], %o1 ! code 6606 call %g1 ! (*sa->sa_handler)(sig,code,scp) 6607 add %fp, BIAS + 128 + 8, %o2 ! scp 6608 wr %l1, %g0, %y ! in any case, restore %y 6609 6610 /* 6611 * Now that the handler has returned, re-establish all the state 6612 * we just saved above, then do a sigreturn. 6613 */ 6614 btst FPRS_DL|FPRS_DU, %l0 ! All clean? 6615 bz,pt %icc, 2f 6616 btst FPRS_DL, %l0 ! test dl 6617 bz,pt %icc, 1f 6618 btst FPRS_DU, %l0 ! test du 6619 6620 ldx [%sp + CC64FSZ + BIAS + 0], %fsr 6621 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6622 andn %l0, BLOCK_ALIGN, %l0 ! do a block load 6623 ldda [%l0] ASI_BLK_P, %f0 6624 inc BLOCK_SIZE, %l0 6625 ldda [%l0] ASI_BLK_P, %f16 6626 1: 6627 bz,pt %icc, 2f 6628 nop 6629 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6630 andn %l0, BLOCK_ALIGN, %l0 ! do a block load 6631 inc 2*BLOCK_SIZE, %l0 ! and skip what we already loaded 6632 ldda [%l0] ASI_BLK_P, %f32 6633 inc BLOCK_SIZE, %l0 6634 ldda [%l0] ASI_BLK_P, %f48 6635 2: 6636 mov %l2, %g2 6637 mov %l3, %g3 6638 mov %l4, %g4 6639 mov %l5, %g5 6640 mov %l6, %g6 6641 mov %l7, %g7 6642 membar #Sync 6643 6644 restore %g0, SYS_compat_16___sigreturn14, %g1 ! get registers back & set syscall # 6645 add %sp, BIAS + 128 + 8, %o0! compute scp 6646 ! andn %o0, 0x0f, %o0 6647 t ST_SYSCALL ! sigreturn(scp) 6648 ! sigreturn does not return unless it fails 6649 mov SYS_exit, %g1 ! exit(errno) 6650 t ST_SYSCALL 6651 /* NOTREACHED */ 6652 6653 .globl _C_LABEL(esigcode) 6654 _C_LABEL(esigcode): 6655 #endif 6656 6657 #if !defined(_LP64) 6658 6659 #define SIGCODE_NAME sigcode 6660 #define ESIGCODE_NAME esigcode 6661 #define SIGRETURN_NAME SYS_compat_16___sigreturn14 6662 #define EXIT_NAME SYS_exit 6663 6664 #include "sigcode32.s" 6665 6666 #endif 6667 #endif 6668 6669 /* 6670 * getfp() - get stack frame pointer 6671 */ 6672 ENTRY(getfp) 6673 retl 6674 mov %fp, %o0 6675 6676 /* 6677 * Call optional cpu_idle handler if provided 6678 */ 6679 ENTRY(cpu_idle) 6680 set CPUINFO_VA, %o0 6681 LDPTR [%o0 + CI_IDLESPIN], %o1 6682 tst %o1 6683 bz 1f 6684 nop 6685 jmp %o1 6686 nop 6687 1: 6688 retl 6689 nop 6690 6691 /* 6692 * cpu_switchto() switches to an lwp to run and runs it, saving the 6693 * current one away. 6694 * 6695 * struct lwp * cpu_switchto(struct lwp *current, struct lwp *next) 6696 * Switch to the specified next LWP 6697 * Arguments: 6698 * i0 'struct lwp *' of the current LWP 6699 * i1 'struct lwp *' of the LWP to switch to 6700 * i2 'bool' of the flag returning to a softint LWP or not 6701 * Returns: 6702 * the old lwp switched away from 6703 */ 6704 ENTRY(cpu_switchto) 6705 save %sp, -CC64FSZ, %sp 6706 /* 6707 * REGISTER USAGE AT THIS POINT: 6708 * %l1 = newpcb 6709 * %l3 = new trapframe 6710 * %l4 = new l->l_proc 6711 * %l5 = pcb of oldlwp 6712 * %l6 = %hi(CPCB) 6713 * %l7 = %hi(CURLWP) 6714 * %i0 = oldlwp 6715 * %i1 = lwp 6716 * %i2 = returning 6717 * %o0 = tmp 1 6718 * %o1 = tmp 2 6719 * %o2 = tmp 3 6720 * %o3 = tmp 4 6721 */ 6722 6723 flushw ! save all register windows except this one 6724 wrpr %g0, PSTATE_KERN, %pstate ! make sure we're on normal globals 6725 ! with traps turned off 6726 6727 sethi %hi(CPCB), %l6 6728 6729 rdpr %pstate, %o1 ! oldpstate = %pstate; 6730 LDPTR [%i0 + L_PCB], %l5 6731 6732 stx %i7, [%l5 + PCB_PC] 6733 stx %i6, [%l5 + PCB_SP] 6734 sth %o1, [%l5 + PCB_PSTATE] 6735 6736 rdpr %cwp, %o2 ! Useless 6737 stb %o2, [%l5 + PCB_CWP] 6738 6739 sethi %hi(CURLWP), %l7 6740 6741 LDPTR [%i1 + L_PCB], %l1 ! newpcb = l->l_pcb; 6742 6743 /* 6744 * Load the new lwp. To load, we must change stacks and 6745 * alter cpcb and the window control registers, hence we must 6746 * keep interrupts disabled. 6747 * 6748 * Issue barriers to coordinate mutex_exit on this CPU with 6749 * mutex_vector_enter on another CPU. 6750 * 6751 * 1. Any prior mutex_exit by oldlwp must be visible to other 6752 * CPUs before we set ci_curlwp := newlwp on this one, 6753 * requiring a store-before-store barrier. 6754 * 6755 * 2. ci_curlwp := newlwp must be visible on all other CPUs 6756 * before any subsequent mutex_exit by newlwp can even test 6757 * whether there might be waiters, requiring a 6758 * store-before-load barrier. 6759 * 6760 * See kern_mutex.c for details -- this is necessary for 6761 * adaptive mutexes to detect whether the lwp is on the CPU in 6762 * order to safely block without requiring atomic r/m/w in 6763 * mutex_exit. 6764 */ 6765 6766 membar #StoreStore 6767 STPTR %i1, [%l7 + %lo(CURLWP)] ! curlwp = l; 6768 membar #StoreLoad 6769 STPTR %l1, [%l6 + %lo(CPCB)] ! cpcb = newpcb; 6770 6771 ldx [%l1 + PCB_SP], %i6 6772 ldx [%l1 + PCB_PC], %i7 6773 6774 wrpr %g0, 0, %otherwin ! These two insns should be redundant 6775 wrpr %g0, 0, %canrestore 6776 GET_MAXCWP %o3 6777 wrpr %g0, %o3, %cleanwin 6778 dec 1, %o3 ! CANSAVE + CANRESTORE + OTHERWIN = MAXCWP - 1 6779 /* Skip the rest if returning to a interrupted LWP. */ 6780 brnz,pn %i2, Lsw_noras 6781 wrpr %o3, %cansave 6782 6783 /* finally, enable traps */ 6784 wrpr %g0, PSTATE_INTR, %pstate 6785 6786 !flushw 6787 !membar #Sync 6788 6789 /* 6790 * Check for restartable atomic sequences (RAS) 6791 */ 6792 LDPTR [%i1 + L_PROC], %l4 ! now %l4 points to p 6793 mov %l4, %o0 ! p is first arg to ras_lookup 6794 LDPTR [%o0 + P_RASLIST], %o1 ! any RAS in p? 6795 brz,pt %o1, Lsw_noras ! no, skip RAS check 6796 LDPTR [%i1 + L_TF], %l3 ! pointer to trap frame 6797 call _C_LABEL(ras_lookup) 6798 ldx [%l3 + TF_PC], %o1 6799 cmp %o0, -1 6800 be,pt CCCR, Lsw_noras 6801 add %o0, 4, %o1 6802 stx %o0, [%l3 + TF_PC] ! store rewound %pc 6803 stx %o1, [%l3 + TF_NPC] ! and %npc 6804 6805 Lsw_noras: 6806 6807 /* 6808 * We are resuming the process that was running at the 6809 * call to switch(). Just set psr ipl and return. 6810 */ 6811 ! wrpr %g0, 0, %cleanwin ! DEBUG 6812 clr %g4 ! This needs to point to the base of the data segment 6813 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 6814 !wrpr %g0, PSTATE_INTR, %pstate 6815 ret 6816 restore %i0, %g0, %o0 ! return old curlwp 6817 6818 #ifdef __HAVE_FAST_SOFTINTS 6819 /* 6820 * Switch to the LWP assigned to handle interrupts from the given 6821 * source. We borrow the VM context from the interrupted LWP. 6822 * 6823 * int softint_fastintr(void *l) 6824 * 6825 * Arguments: 6826 * i0 softint lwp 6827 */ 6828 ENTRY(softint_fastintr) 6829 save %sp, -CC64FSZ, %sp 6830 set CPUINFO_VA, %l0 ! l0 = curcpu() 6831 rdpr %pil, %l7 ! l7 = splhigh() 6832 wrpr %g0, PIL_HIGH, %pil 6833 LDPTR [%l0 + CI_EINTSTACK], %l6 ! l6 = ci_eintstack 6834 add %sp, -CC64FSZ, %l2 ! ci_eintstack = sp - CC64FSZ 6835 STPTR %l2, [%l0 + CI_EINTSTACK] ! save intstack for nested intr 6836 6837 mov %i0, %o0 ! o0/i0 = softint lwp 6838 mov %l7, %o1 ! o1/i1 = ipl 6839 save %sp, -CC64FSZ, %sp ! make one more register window 6840 flushw ! and save all 6841 6842 sethi %hi(CURLWP), %l7 6843 sethi %hi(CPCB), %l6 6844 LDPTR [%l7 + %lo(CURLWP)], %l0 ! l0 = interrupted lwp (curlwp) 6845 6846 /* save interrupted lwp/pcb info */ 6847 sethi %hi(softint_fastintr_ret - 8), %o0 ! trampoline function 6848 LDPTR [%l0 + L_PCB], %l5 ! l5 = interrupted pcb 6849 or %o0, %lo(softint_fastintr_ret - 8), %o0 6850 stx %i6, [%l5 + PCB_SP] 6851 stx %o0, [%l5 + PCB_PC] 6852 rdpr %pstate, %o1 6853 rdpr %cwp, %o2 6854 sth %o1, [%l5 + PCB_PSTATE] 6855 stb %o2, [%l5 + PCB_CWP] 6856 6857 /* switch to softint lwp */ 6858 sethi %hi(USPACE - TF_SIZE - CC64FSZ - STKB), %o3 6859 LDPTR [%i0 + L_PCB], %l1 ! l1 = softint pcb 6860 or %o3, %lo(USPACE - TF_SIZE - CC64FSZ - STKB), %o3 6861 membar #StoreStore /* for mutex_enter; see cpu_switchto */ 6862 STPTR %i0, [%l7 + %lo(CURLWP)] 6863 /* 6864 * No need for barrier after ci->ci_curlwp = softlwp -- when we 6865 * enter a softint lwp, it can't be holding any mutexes, so it 6866 * can't release any until after it has acquired them, so we 6867 * need not participate in the protocol with mutex_vector_enter 6868 * barriers here. 6869 */ 6870 add %l1, %o3, %i6 6871 STPTR %l1, [%l6 + %lo(CPCB)] 6872 stx %i6, [%l1 + PCB_SP] 6873 add %i6, -CC64FSZ, %sp ! new stack 6874 6875 /* now switched, then invoke MI dispatcher */ 6876 mov %i1, %o1 6877 call _C_LABEL(softint_dispatch) 6878 mov %l0, %o0 6879 6880 /* switch back to interrupted lwp */ 6881 ldx [%l5 + PCB_SP], %i6 6882 membar #StoreStore /* for mutex_enter; see cpu_switchto */ 6883 STPTR %l0, [%l7 + %lo(CURLWP)] 6884 membar #StoreLoad /* for mutex_enter; see cpu_switchto */ 6885 STPTR %l5, [%l6 + %lo(CPCB)] 6886 6887 restore ! rewind register window 6888 6889 STPTR %l6, [%l0 + CI_EINTSTACK] ! restore ci_eintstack 6890 wrpr %g0, %l7, %pil ! restore ipl 6891 ret 6892 restore %g0, 1, %o0 6893 6894 /* 6895 * Trampoline function that gets returned to by cpu_switchto() when 6896 * an interrupt handler blocks. 6897 * 6898 * Arguments: 6899 * o0 old lwp from cpu_switchto() 6900 * 6901 * from softint_fastintr(): 6902 * l0 CPUINFO_VA 6903 * l6 saved ci_eintstack 6904 * l7 saved ipl 6905 */ 6906 softint_fastintr_ret: 6907 /* re-adjust after mi_switch() */ 6908 ld [%l0 + CI_MTX_COUNT], %o1 6909 inc %o1 ! ci_mtx_count++ 6910 st %o1, [%l0 + CI_MTX_COUNT] 6911 6912 STPTR %l6, [%l0 + CI_EINTSTACK] ! restore ci_eintstack 6913 wrpr %g0, %l7, %pil ! restore ipl 6914 ret 6915 restore %g0, 1, %o0 6916 6917 #endif /* __HAVE_FAST_SOFTINTS */ 6918 6919 /* 6920 * Snapshot the current process so that stack frames are up to date. 6921 * Only used just before a crash dump. 6922 */ 6923 ENTRY(snapshot) 6924 rdpr %pstate, %o1 ! save psr 6925 stx %o7, [%o0 + PCB_PC] ! save pc 6926 stx %o6, [%o0 + PCB_SP] ! save sp 6927 rdpr %pil, %o2 6928 sth %o1, [%o0 + PCB_PSTATE] 6929 rdpr %cwp, %o3 6930 stb %o2, [%o0 + PCB_PIL] 6931 stb %o3, [%o0 + PCB_CWP] 6932 6933 flushw 6934 save %sp, -CC64FSZ, %sp 6935 flushw 6936 ret 6937 restore 6938 6939 /* 6940 * cpu_lwp_fork() arranges for lwp_trampoline() to run when the 6941 * nascent lwp is selected by switch(). 6942 * 6943 * The switch frame will contain pointer to struct lwp of this lwp in 6944 * %l2, a pointer to the function to call in %l0, and an argument to 6945 * pass to it in %l1 (we abuse the callee-saved registers). 6946 * 6947 * We enter lwp_trampoline as if we are "returning" from 6948 * cpu_switchto(), so %o0 contains previous lwp (the one we are 6949 * switching from) that we pass to lwp_startup(). 6950 * 6951 * If the function *(%l0) returns, we arrange for an immediate return 6952 * to user mode. This happens in two known cases: after execve(2) of 6953 * init, and when returning a child to user mode after a fork(2). 6954 * 6955 * If were setting up a kernel thread, the function *(%l0) will not 6956 * return. 6957 */ 6958 ENTRY(lwp_trampoline) 6959 /* 6960 * Note: cpu_lwp_fork() has set up a stack frame for us to run 6961 * in, so we can call other functions from here without using 6962 * `save ... restore'. 6963 */ 6964 6965 ! newlwp in %l2, oldlwp in %o0 6966 call lwp_startup 6967 mov %l2, %o1 6968 6969 call %l0 ! re-use current frame 6970 mov %l1, %o0 6971 6972 /* 6973 * Here we finish up as in syscall, but simplified. 6974 */ 6975 b return_from_trap 6976 nop 6977 6978 /* 6979 * pmap_zero_page_phys(pa) 6980 * 6981 * Zero one page physically addressed 6982 * 6983 * Block load/store ASIs do not exist for physical addresses, 6984 * so we won't use them. 6985 * 6986 * We will execute a flush at the end to sync the I$. 6987 * 6988 * This version expects to have the dcache_flush_page_all(pa) 6989 * to have been called before calling into here. 6990 */ 6991 ENTRY(pmap_zero_page_phys) 6992 #ifndef _LP64 6993 COMBINE(%o0, %o1, %o0) 6994 #endif 6995 #ifdef DEBUG 6996 set pmapdebug, %o4 6997 ld [%o4], %o4 6998 btst 0x80, %o4 ! PDB_COPY 6999 bz,pt %icc, 3f 7000 nop 7001 save %sp, -CC64FSZ, %sp 7002 set 2f, %o0 7003 call printf 7004 mov %i0, %o1 7005 ! ta 1; nop 7006 restore 7007 .data 7008 2: .asciz "pmap_zero_page(%p)\n" 7009 _ALIGN 7010 .text 7011 3: 7012 #endif 7013 set NBPG, %o2 ! Loop count 7014 wr %g0, ASI_PHYS_CACHED, %asi 7015 1: 7016 /* Unroll the loop 8 times */ 7017 stxa %g0, [%o0 + 0x00] %asi 7018 deccc 0x40, %o2 7019 stxa %g0, [%o0 + 0x08] %asi 7020 stxa %g0, [%o0 + 0x10] %asi 7021 stxa %g0, [%o0 + 0x18] %asi 7022 stxa %g0, [%o0 + 0x20] %asi 7023 stxa %g0, [%o0 + 0x28] %asi 7024 stxa %g0, [%o0 + 0x30] %asi 7025 stxa %g0, [%o0 + 0x38] %asi 7026 bg,pt %icc, 1b 7027 inc 0x40, %o0 7028 7029 sethi %hi(KERNBASE), %o3 7030 flush %o3 7031 retl 7032 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Make C code happy 7033 7034 /* 7035 * pmap_copy_page_phys(paddr_t src, paddr_t dst) 7036 * 7037 * Copy one page physically addressed 7038 * We need to use a global reg for ldxa/stxa 7039 * so the top 32-bits cannot be lost if we take 7040 * a trap and need to save our stack frame to a 7041 * 32-bit stack. We will unroll the loop by 4 to 7042 * improve performance. 7043 * 7044 * This version expects to have the dcache_flush_page_all(pa) 7045 * to have been called before calling into here. 7046 * 7047 */ 7048 ENTRY(pmap_copy_page_phys) 7049 #ifndef _LP64 7050 COMBINE(%o0, %o1, %o0) 7051 COMBINE(%o2, %o3, %o1) 7052 #endif 7053 #ifdef DEBUG 7054 set pmapdebug, %o4 7055 ld [%o4], %o4 7056 btst 0x80, %o4 ! PDB_COPY 7057 bz,pt %icc, 3f 7058 nop 7059 save %sp, -CC64FSZ, %sp 7060 mov %i0, %o1 7061 set 2f, %o0 7062 call printf 7063 mov %i1, %o2 7064 ! ta 1; nop 7065 restore 7066 .data 7067 2: .asciz "pmap_copy_page(%p,%p)\n" 7068 _ALIGN 7069 .text 7070 3: 7071 #endif 7072 #if 1 7073 set NBPG, %o2 7074 wr %g0, ASI_PHYS_CACHED, %asi 7075 1: 7076 ldxa [%o0 + 0x00] %asi, %g1 7077 ldxa [%o0 + 0x08] %asi, %o3 7078 ldxa [%o0 + 0x10] %asi, %o4 7079 ldxa [%o0 + 0x18] %asi, %o5 7080 inc 0x20, %o0 7081 deccc 0x20, %o2 7082 stxa %g1, [%o1 + 0x00] %asi 7083 stxa %o3, [%o1 + 0x08] %asi 7084 stxa %o4, [%o1 + 0x10] %asi 7085 stxa %o5, [%o1 + 0x18] %asi 7086 bg,pt %icc, 1b ! We don't care about pages >4GB 7087 inc 0x20, %o1 7088 retl 7089 wr %g0, ASI_PRIMARY_NOFAULT, %asi 7090 #else 7091 set NBPG, %o3 7092 add %o3, %o0, %o3 7093 mov %g1, %o4 ! Save g1 7094 1: 7095 ldxa [%o0] ASI_PHYS_CACHED, %g1 7096 inc 8, %o0 7097 cmp %o0, %o3 7098 stxa %g1, [%o1] ASI_PHYS_CACHED 7099 bl,pt %icc, 1b ! We don't care about pages >4GB 7100 inc 8, %o1 7101 retl 7102 mov %o4, %g1 ! Restore g1 7103 #endif 7104 7105 /* 7106 * extern int64_t pseg_get_real(struct pmap *pm, vaddr_t addr); 7107 * 7108 * Return TTE at addr in pmap. Uses physical addressing only. 7109 * pmap->pm_physaddr must by the physical address of pm_segs 7110 * 7111 */ 7112 ENTRY(pseg_get_real) 7113 ! flushw ! Make sure we don't have stack probs & lose hibits of %o 7114 #ifndef _LP64 7115 clruw %o1 ! Zero extend 7116 #endif 7117 ldx [%o0 + PM_PHYS], %o2 ! pmap->pm_segs 7118 7119 srax %o1, HOLESHIFT, %o3 ! Check for valid address 7120 brz,pt %o3, 0f ! Should be zero or -1 7121 inc %o3 ! Make -1 -> 0 7122 brnz,pn %o3, 1f ! Error! In hole! 7123 0: 7124 srlx %o1, STSHIFT, %o3 7125 and %o3, STMASK, %o3 ! Index into pm_segs 7126 sll %o3, 3, %o3 7127 add %o2, %o3, %o2 7128 DLFLUSH(%o2,%o3) 7129 ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page directory pointer 7130 DLFLUSH2(%o3) 7131 7132 srlx %o1, PDSHIFT, %o3 7133 and %o3, PDMASK, %o3 7134 sll %o3, 3, %o3 7135 brz,pn %o2, 1f ! NULL entry? check somewhere else 7136 add %o2, %o3, %o2 7137 DLFLUSH(%o2,%o3) 7138 ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page table pointer 7139 DLFLUSH2(%o3) 7140 7141 srlx %o1, PTSHIFT, %o3 ! Convert to ptab offset 7142 and %o3, PTMASK, %o3 7143 sll %o3, 3, %o3 7144 brz,pn %o2, 1f ! NULL entry? check somewhere else 7145 add %o2, %o3, %o2 7146 DLFLUSH(%o2,%o3) 7147 ldxa [%o2] ASI_PHYS_CACHED, %o0 7148 DLFLUSH2(%o3) 7149 brgez,pn %o0, 1f ! Entry invalid? Punt 7150 btst 1, %sp 7151 bz,pn %icc, 0f ! 64-bit mode? 7152 nop 7153 retl ! Yes, return full value 7154 nop 7155 0: 7156 #if 1 7157 srl %o0, 0, %o1 7158 retl ! No, generate a %o0:%o1 double 7159 srlx %o0, 32, %o0 7160 #else 7161 DLFLUSH(%o2,%o3) 7162 ldda [%o2] ASI_PHYS_CACHED, %o0 7163 DLFLUSH2(%o3) 7164 retl ! No, generate a %o0:%o1 double 7165 nop 7166 #endif 7167 1: 7168 #ifndef _LP64 7169 clr %o1 7170 #endif 7171 retl 7172 clr %o0 7173 7174 /* 7175 * In 32-bit mode: 7176 * 7177 * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, 7178 * int64_t tte %o2:%o3, paddr_t spare %o4:%o5); 7179 * 7180 * In 64-bit mode: 7181 * 7182 * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, 7183 * int64_t tte %o2, paddr_t spare %o3); 7184 * 7185 * Set a pseg entry to a particular TTE value. Return values are: 7186 * 7187 * -2 addr in hole 7188 * 0 success (spare was not used if given) 7189 * 1 failure (spare was not given, but one is needed) 7190 * 2 success (spare was given, used for L2) 7191 * 3 failure (spare was given, used for L2, another is needed for L3) 7192 * 4 success (spare was given, used for L3) 7193 * 7194 * rv == 0 success, spare not used if one was given 7195 * rv & 4 spare was used for L3 7196 * rv & 2 spare was used for L2 7197 * rv & 1 failure, spare is needed 7198 * 7199 * (NB: nobody in pmap checks for the virtual hole, so the system will hang.) 7200 * The way to call this is: first just call it without a spare page. 7201 * If that fails, allocate a page and try again, passing the paddr of the 7202 * new page as the spare. 7203 * If spare is non-zero it is assumed to be the address of a zeroed physical 7204 * page that can be used to generate a directory table or page table if needed. 7205 * 7206 * We keep track of valid (A_TLB_V bit set) and wired (A_TLB_TSB_LOCK bit set) 7207 * mappings that are set here. We check both bits on the new data entered 7208 * and increment counts, as well as decrementing counts if the bits are set 7209 * in the value replaced by this call. 7210 * The counters are 32 bit or 64 bit wide, depending on the kernel type we are 7211 * running! 7212 */ 7213 ENTRY(pseg_set_real) 7214 #ifndef _LP64 7215 clruw %o1 ! Zero extend 7216 COMBINE(%o2, %o3, %o2) 7217 COMBINE(%o4, %o5, %o3) 7218 #endif 7219 !! 7220 !! However we managed to get here we now have: 7221 !! 7222 !! %o0 = *pmap 7223 !! %o1 = addr 7224 !! %o2 = tte 7225 !! %o3 = paddr of spare page 7226 !! 7227 srax %o1, HOLESHIFT, %o4 ! Check for valid address 7228 brz,pt %o4, 0f ! Should be zero or -1 7229 inc %o4 ! Make -1 -> 0 7230 brz,pt %o4, 0f 7231 nop 7232 #ifdef DEBUG 7233 ta 1 ! Break into debugger 7234 #endif 7235 retl 7236 mov -2, %o0 ! Error -- in hole! 7237 7238 0: 7239 ldx [%o0 + PM_PHYS], %o4 ! pmap->pm_segs 7240 clr %g1 7241 srlx %o1, STSHIFT, %o5 7242 and %o5, STMASK, %o5 7243 sll %o5, 3, %o5 7244 add %o4, %o5, %o4 7245 0: 7246 DLFLUSH(%o4,%g5) 7247 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load page directory pointer 7248 DLFLUSH2(%g5) 7249 7250 brnz,a,pt %o5, 0f ! Null pointer? 7251 mov %o5, %o4 7252 brz,pn %o3, 9f ! Have a spare? 7253 mov %o3, %o5 7254 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 7255 brnz,pn %o5, 0b ! Something changed? 7256 DLFLUSH(%o4, %o5) 7257 mov %o3, %o4 7258 mov 2, %g1 ! record spare used for L2 7259 clr %o3 ! and not available for L3 7260 0: 7261 srlx %o1, PDSHIFT, %o5 7262 and %o5, PDMASK, %o5 7263 sll %o5, 3, %o5 7264 add %o4, %o5, %o4 7265 0: 7266 DLFLUSH(%o4,%g5) 7267 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load table directory pointer 7268 DLFLUSH2(%g5) 7269 7270 brnz,a,pt %o5, 0f ! Null pointer? 7271 mov %o5, %o4 7272 brz,pn %o3, 9f ! Have a spare? 7273 mov %o3, %o5 7274 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 7275 brnz,pn %o5, 0b ! Something changed? 7276 DLFLUSH(%o4, %o4) 7277 mov %o3, %o4 7278 mov 4, %g1 ! record spare used for L3 7279 0: 7280 srlx %o1, PTSHIFT, %o5 ! Convert to ptab offset 7281 and %o5, PTMASK, %o5 7282 sll %o5, 3, %o5 7283 add %o5, %o4, %o4 7284 7285 DLFLUSH(%o4,%g5) 7286 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! save old value in %o5 7287 stxa %o2, [%o4] ASI_PHYS_CACHED ! Easier than shift+or 7288 DLFLUSH2(%g5) 7289 7290 !! at this point we have: 7291 !! %g1 = return value 7292 !! %o0 = struct pmap * (where the counts are) 7293 !! %o2 = new TTE 7294 !! %o5 = old TTE 7295 7296 !! see if stats needs an update 7297 #ifdef SUN4V 7298 sethi %hi(cputyp), %g5 7299 ld [%g5 + %lo(cputyp)], %g5 7300 cmp %g5, CPU_SUN4V 7301 bne,pt %icc, 0f 7302 nop 7303 sethi %hh(SUN4V_TLB_TSB_LOCK), %g5 7304 sllx %g5, 32, %g5 7305 ba 1f 7306 nop 7307 0: 7308 #endif 7309 set SUN4U_TLB_TSB_LOCK, %g5 7310 1: 7311 xor %o2, %o5, %o3 ! %o3 - what changed 7312 7313 brgez,pn %o3, 5f ! has resident changed? (we predict it has) 7314 btst %g5, %o3 ! has wired changed? 7315 7316 LDPTR [%o0 + PM_RESIDENT], %o1 ! gonna update resident count 7317 brlz %o2, 0f 7318 mov 1, %o4 7319 neg %o4 ! new is not resident -> decrement 7320 0: add %o1, %o4, %o1 7321 STPTR %o1, [%o0 + PM_RESIDENT] 7322 btst %g5, %o3 ! has wired changed? 7323 5: bz,pt %xcc, 8f ! we predict it's not 7324 btst %g5, %o2 ! don't waste delay slot, check if new one is wired 7325 LDPTR [%o0 + PM_WIRED], %o1 ! gonna update wired count 7326 bnz,pt %xcc, 0f ! if wired changes, we predict it increments 7327 mov 1, %o4 7328 neg %o4 ! new is not wired -> decrement 7329 0: add %o1, %o4, %o1 7330 STPTR %o1, [%o0 + PM_WIRED] 7331 8: retl 7332 mov %g1, %o0 ! return %g1 7333 7334 9: retl 7335 or %g1, 1, %o0 ! spare needed, return flags + 1 7336 7337 7338 /* 7339 * clearfpstate() 7340 * 7341 * Drops the current fpu state, without saving it. 7342 */ 7343 ENTRY(clearfpstate) 7344 rdpr %pstate, %o1 ! enable FPU 7345 wr %g0, FPRS_FEF, %fprs 7346 or %o1, PSTATE_PEF, %o1 7347 retl 7348 wrpr %o1, 0, %pstate 7349 7350 /* 7351 * savefpstate(f) struct fpstate *f; 7352 * 7353 * Store the current FPU state. 7354 * 7355 * Since the kernel may need to use the FPU and we have problems atomically 7356 * testing and enabling the FPU, we leave here with the FPRS_FEF bit set. 7357 * Normally this should be turned on in loadfpstate(). 7358 */ 7359 /* XXXXXXXXXX Assume caller created a proper stack frame */ 7360 ENTRY(savefpstate) 7361 ! flushw ! Make sure we don't have stack probs & lose hibits of %o 7362 rdpr %pstate, %o1 ! enable FP before we begin 7363 rd %fprs, %o5 7364 wr %g0, FPRS_FEF, %fprs 7365 or %o1, PSTATE_PEF, %o1 7366 wrpr %o1, 0, %pstate 7367 7368 stx %fsr, [%o0 + FS_FSR] ! f->fs_fsr = getfsr(); 7369 rd %gsr, %o4 ! Save %gsr 7370 st %o4, [%o0 + FS_GSR] 7371 7372 add %o0, FS_REGS, %o2 7373 #ifdef DIAGNOSTIC 7374 btst BLOCK_ALIGN, %o2 ! Needs to be re-executed 7375 bnz,pn %icc, 6f ! Check alignment 7376 #endif 7377 st %g0, [%o0 + FS_QSIZE] ! f->fs_qsize = 0; 7378 btst FPRS_DL|FPRS_DU, %o5 ! Both FPU halves clean? 7379 bz,pt %icc, 5f ! Then skip it 7380 7381 btst FPRS_DL, %o5 ! Lower FPU clean? 7382 membar #Sync 7383 bz,a,pt %icc, 1f ! Then skip it, but upper FPU not clean 7384 add %o2, 2*BLOCK_SIZE, %o2 ! Skip a block 7385 7386 stda %f0, [%o2] ASI_BLK_P ! f->fs_f0 = etc; 7387 inc BLOCK_SIZE, %o2 7388 stda %f16, [%o2] ASI_BLK_P 7389 7390 btst FPRS_DU, %o5 ! Upper FPU clean? 7391 bz,pt %icc, 2f ! Then skip it 7392 inc BLOCK_SIZE, %o2 7393 1: 7394 stda %f32, [%o2] ASI_BLK_P 7395 inc BLOCK_SIZE, %o2 7396 stda %f48, [%o2] ASI_BLK_P 7397 2: 7398 membar #Sync ! Finish operation so we can 7399 5: 7400 retl 7401 wr %g0, FPRS_FEF, %fprs ! Mark FPU clean 7402 7403 #ifdef DIAGNOSTIC 7404 !! 7405 !! Damn thing is *NOT* aligned on a 64-byte boundary 7406 !! 7407 6: 7408 wr %g0, FPRS_FEF, %fprs 7409 ! XXX -- we should panic instead of silently entering debugger 7410 ta 1 7411 retl 7412 nop 7413 #endif 7414 7415 /* 7416 * Load FPU state. 7417 */ 7418 /* XXXXXXXXXX Should test to see if we only need to do a partial restore */ 7419 ENTRY(loadfpstate) 7420 flushw ! Make sure we don't have stack probs & lose hibits of %o 7421 rdpr %pstate, %o1 ! enable FP before we begin 7422 ld [%o0 + FS_GSR], %o4 ! Restore %gsr 7423 set PSTATE_PEF, %o2 7424 wr %g0, FPRS_FEF, %fprs 7425 or %o1, %o2, %o1 7426 wrpr %o1, 0, %pstate 7427 ldx [%o0 + FS_FSR], %fsr ! setfsr(f->fs_fsr); 7428 add %o0, FS_REGS, %o3 ! This is zero... 7429 #ifdef DIAGNOSTIC 7430 btst BLOCK_ALIGN, %o3 7431 bne,pn %icc, 1f ! Only use block loads on aligned blocks 7432 #endif 7433 wr %o4, %g0, %gsr 7434 membar #Sync 7435 ldda [%o3] ASI_BLK_P, %f0 7436 inc BLOCK_SIZE, %o3 7437 ldda [%o3] ASI_BLK_P, %f16 7438 inc BLOCK_SIZE, %o3 7439 ldda [%o3] ASI_BLK_P, %f32 7440 inc BLOCK_SIZE, %o3 7441 ldda [%o3] ASI_BLK_P, %f48 7442 membar #Sync ! Make sure loads are complete 7443 retl 7444 wr %g0, FPRS_FEF, %fprs ! Clear dirty bits 7445 7446 #ifdef DIAGNOSTIC 7447 !! 7448 !! Damn thing is *NOT* aligned on a 64-byte boundary 7449 !! 7450 1: 7451 wr %g0, FPRS_FEF, %fprs ! Clear dirty bits 7452 ! XXX -- we should panic instead of silently entering debugger 7453 ta 1 7454 retl 7455 nop 7456 #endif 7457 7458 /* 7459 * ienab_bis(bis) int bis; 7460 * ienab_bic(bic) int bic; 7461 * 7462 * Set and clear bits in the interrupt register. 7463 */ 7464 7465 /* 7466 * sun4u has separate asr's for clearing/setting the interrupt mask. 7467 */ 7468 ENTRY(ienab_bis) 7469 retl 7470 wr %o0, 0, SET_SOFTINT ! SET_SOFTINT 7471 7472 ENTRY(ienab_bic) 7473 retl 7474 wr %o0, 0, CLEAR_SOFTINT ! CLEAR_SOFTINT 7475 7476 /* 7477 * send_softint(cpu, level, intrhand) 7478 * 7479 * Send a softint with an intrhand pointer so we can cause a vectored 7480 * interrupt instead of a polled interrupt. This does pretty much the same 7481 * as interrupt_vector. If cpu is -1 then send it to this CPU, if it's -2 7482 * send it to any CPU, otherwise send it to a particular CPU. 7483 * 7484 * XXXX Dispatching to different CPUs is not implemented yet. 7485 */ 7486 ENTRY(send_softint) 7487 rdpr %pstate, %g1 7488 andn %g1, PSTATE_IE, %g2 ! clear PSTATE.IE 7489 wrpr %g2, 0, %pstate 7490 7491 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %o3 7492 LDPTR [%o2 + IH_PEND], %o5 7493 or %o3, %lo(CPUINFO_VA+CI_INTRPENDING), %o3 7494 brnz %o5, 1f 7495 sll %o1, PTRSHFT, %o5 ! Find start of table for this IPL 7496 add %o3, %o5, %o3 7497 2: 7498 LDPTR [%o3], %o5 ! Load list head 7499 STPTR %o5, [%o2+IH_PEND] ! Link our intrhand node in 7500 mov %o2, %o4 7501 CASPTRA [%o3] ASI_N, %o5, %o4 7502 cmp %o4, %o5 ! Did it work? 7503 bne,pn CCCR, 2b ! No, try again 7504 .empty 7505 7506 mov 1, %o4 ! Change from level to bitmask 7507 sllx %o4, %o1, %o4 7508 wr %o4, 0, SET_SOFTINT ! SET_SOFTINT 7509 1: 7510 retl 7511 wrpr %g1, 0, %pstate ! restore PSTATE.IE 7512 7513 7514 #define MICROPERSEC (1000000) 7515 7516 /* 7517 * delay function 7518 * 7519 * void delay(N) -- delay N microseconds 7520 * 7521 * Register usage: %o0 = "N" number of usecs to go (counts down to zero) 7522 * %o1 = "timerblurb" (stays constant) 7523 * %o2 = counter for 1 usec (counts down from %o1 to zero) 7524 * 7525 * 7526 * ci_cpu_clockrate should be tuned during CPU probe to the CPU 7527 * clockrate in Hz 7528 * 7529 */ 7530 ENTRY(delay) ! %o0 = n 7531 #if 1 7532 rdpr %tick, %o1 ! Take timer snapshot 7533 sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %o2 7534 sethi %hi(MICROPERSEC), %o3 7535 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)], %o4 ! Get scale factor 7536 brnz,pt %o4, 0f 7537 or %o3, %lo(MICROPERSEC), %o3 7538 7539 !! Calculate ticks/usec 7540 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %o4 ! No, we need to calculate it 7541 udivx %o4, %o3, %o4 7542 stx %o4, [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)] ! Save it so we don't need to divide again 7543 0: 7544 7545 mulx %o0, %o4, %o0 ! Convert usec -> ticks 7546 rdpr %tick, %o2 ! Top of next itr 7547 1: 7548 sub %o2, %o1, %o3 ! How many ticks have gone by? 7549 sub %o0, %o3, %o4 ! Decrement count by that much 7550 movrgz %o3, %o4, %o0 ! But only if we're decrementing 7551 mov %o2, %o1 ! Remember last tick 7552 brgz,pt %o0, 1b ! Done? 7553 rdpr %tick, %o2 ! Get new tick 7554 7555 retl 7556 nop 7557 #else 7558 /* This code only works if %tick does not wrap */ 7559 rdpr %tick, %g1 ! Take timer snapshot 7560 sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %g2 7561 sethi %hi(MICROPERSEC), %o2 7562 ldx [%g2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %g2 ! Get scale factor 7563 or %o2, %lo(MICROPERSEC), %o2 7564 ! sethi %hi(_C_LABEL(timerblurb), %o5 ! This is if we plan to tune the clock 7565 ! ld [%o5 + %lo(_C_LABEL(timerblurb))], %o5 ! with respect to the counter/timer 7566 mulx %o0, %g2, %g2 ! Scale it: (usec * Hz) / 1 x 10^6 = ticks 7567 udivx %g2, %o2, %g2 7568 add %g1, %g2, %g2 7569 ! add %o5, %g2, %g2 5, %g2, %g2 ! But this gets complicated 7570 rdpr %tick, %g1 ! Top of next itr 7571 mov %g1, %g1 ! Erratum 50 7572 1: 7573 cmp %g1, %g2 7574 bl,a,pn %xcc, 1b ! Done? 7575 rdpr %tick, %g1 7576 7577 retl 7578 nop 7579 #endif 7580 /* 7581 * If something's wrong with the standard setup do this stupid loop 7582 * calibrated for a 143MHz processor. 7583 */ 7584 Lstupid_delay: 7585 set 142857143/MICROPERSEC, %o1 7586 Lstupid_loop: 7587 brnz,pt %o1, Lstupid_loop 7588 dec %o1 7589 brnz,pt %o0, Lstupid_delay 7590 dec %o0 7591 retl 7592 nop 7593 7594 /* 7595 * next_tick(long increment) 7596 * 7597 * Sets the %tick_cmpr register to fire off in `increment' machine 7598 * cycles in the future. Also handles %tick wraparound. In 32-bit 7599 * mode we're limited to a 32-bit increment. 7600 */ 7601 ENTRY(next_tick) 7602 rd TICK_CMPR, %o2 7603 rdpr %tick, %o1 7604 7605 mov 1, %o3 ! Mask off high bits of these registers 7606 sllx %o3, 63, %o3 7607 andn %o1, %o3, %o1 7608 andn %o2, %o3, %o2 7609 cmp %o1, %o2 ! Did we wrap? (tick < tick_cmpr) 7610 bgt,pt %icc, 1f 7611 add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. 7612 7613 /* 7614 * Handle the unlikely case of %tick wrapping. 7615 * 7616 * This should only happen every 10 years or more. 7617 * 7618 * We need to increment the time base by the size of %tick in 7619 * microseconds. This will require some divides and multiplies 7620 * which can take time. So we re-read %tick. 7621 * 7622 */ 7623 7624 /* XXXXX NOT IMPLEMENTED */ 7625 7626 7627 7628 1: 7629 add %o2, %o0, %o2 7630 andn %o2, %o3, %o4 7631 brlz,pn %o4, Ltick_ovflw 7632 cmp %o2, %o1 ! Has this tick passed? 7633 blt,pn %xcc, 1b ! Yes 7634 nop 7635 7636 #ifdef BB_ERRATA_1 7637 ba,a 2f 7638 nop 7639 #else 7640 retl 7641 wr %o2, TICK_CMPR 7642 #endif 7643 7644 Ltick_ovflw: 7645 /* 7646 * When we get here tick_cmpr has wrapped, but we don't know if %tick 7647 * has wrapped. If bit 62 is set then we have not wrapped and we can 7648 * use the current value of %o4 as %tick. Otherwise we need to return 7649 * to our loop with %o4 as %tick_cmpr (%o2). 7650 */ 7651 srlx %o3, 1, %o5 7652 btst %o5, %o1 7653 bz,pn %xcc, 1b 7654 mov %o4, %o2 7655 #ifdef BB_ERRATA_1 7656 ba,a 2f 7657 nop 7658 .align 64 7659 2: wr %o2, TICK_CMPR 7660 rd TICK_CMPR, %g0 7661 retl 7662 nop 7663 #else 7664 retl 7665 wr %o2, TICK_CMPR 7666 #endif 7667 7668 /* 7669 * next_stick(long increment) 7670 * 7671 * Sets the %stick_cmpr register to fire off in `increment' machine 7672 * cycles in the future. Also handles %stick wraparound. In 32-bit 7673 * mode we're limited to a 32-bit increment. 7674 */ 7675 ENTRY(next_stick) 7676 rd STICK_CMPR, %o2 7677 rd STICK, %o1 7678 7679 mov 1, %o3 ! Mask off high bits of these registers 7680 sllx %o3, 63, %o3 7681 andn %o1, %o3, %o1 7682 andn %o2, %o3, %o2 7683 cmp %o1, %o2 ! Did we wrap? (stick < stick_cmpr) 7684 bgt,pt %xcc, 1f 7685 add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. 7686 7687 /* 7688 * Handle the unlikely case of %stick wrapping. 7689 * 7690 * This should only happen every 10 years or more. 7691 * 7692 * We need to increment the time base by the size of %stick in 7693 * microseconds. This will require some divides and multiplies 7694 * which can take time. So we re-read %stick. 7695 * 7696 */ 7697 7698 /* XXXXX NOT IMPLEMENTED */ 7699 7700 7701 7702 1: 7703 add %o2, %o0, %o2 7704 andn %o2, %o3, %o4 7705 brlz,pn %o4, Lstick_ovflw 7706 cmp %o2, %o1 ! Has this stick passed? 7707 blt,pn %xcc, 1b ! Yes 7708 nop 7709 retl 7710 wr %o2, STICK_CMPR 7711 7712 Lstick_ovflw: 7713 /* 7714 * When we get here tick_cmpr has wrapped, but we don't know if %stick 7715 * has wrapped. If bit 62 is set then we have not wrapped and we can 7716 * use the current value of %o4 as %stick. Otherwise we need to return 7717 * to our loop with %o4 as %stick_cmpr (%o2). 7718 */ 7719 srlx %o3, 1, %o5 7720 btst %o5, %o1 7721 bz,pn %xcc, 1b 7722 mov %o4, %o2 7723 retl 7724 wr %o2, STICK_CMPR 7725 7726 /* 7727 * next_stick_init() 7728 * 7729 * Sets the %stick_cmpr register to the value retrieved from %stick so 7730 * next_stick() does not spend too much time in the function when called 7731 * for the first time. 7732 * This has been observed on (at least) a SPARC-T5 (sun4v) system where 7733 * the %stick_cmpr ends up being less than the %stick value and then 7734 * the stickitr() interrupt is never triggered. 7735 */ 7736 ENTRY(next_stick_init) 7737 rd STICK, %o0 7738 mov 1, %o1 ! Mask off high bits of the register 7739 sllx %o1, 63, %o1 7740 andn %o0, %o1, %o0 7741 retl 7742 wr %o0, STICK_CMPR 7743 7744 ENTRY(setjmp) 7745 save %sp, -CC64FSZ, %sp ! Need a frame to return to. 7746 flushw 7747 stx %fp, [%i0+0] ! 64-bit stack pointer 7748 stx %i7, [%i0+8] ! 64-bit return pc 7749 ret 7750 restore %g0, 0, %o0 7751 7752 .data 7753 Lpanic_ljmp: 7754 .asciz "longjmp botch" 7755 _ALIGN 7756 .text 7757 7758 ENTRY(longjmp) 7759 save %sp, -CC64FSZ, %sp ! prepare to restore to (old) frame 7760 flushw 7761 mov 1, %i2 7762 ldx [%i0+0], %fp ! get return stack 7763 ldx [%i0+8], %i7 ! get rpc 7764 ret 7765 restore %i2, 0, %o0 7766 7767 #if defined(DDB) || defined(KGDB) 7768 /* 7769 * Debug stuff. Dump the trap registers into buffer & set tl=0. 7770 * 7771 * %o0 = *ts 7772 */ 7773 ENTRY(savetstate) 7774 mov %o0, %o1 7775 rdpr %tl, %o0 7776 brz %o0, 2f 7777 mov %o0, %o2 7778 1: 7779 rdpr %tstate, %o3 7780 stx %o3, [%o1] 7781 deccc %o2 7782 inc 8, %o1 7783 rdpr %tpc, %o4 7784 stx %o4, [%o1] 7785 inc 8, %o1 7786 rdpr %tnpc, %o5 7787 stx %o5, [%o1] 7788 inc 8, %o1 7789 rdpr %tt, %o4 7790 stx %o4, [%o1] 7791 inc 8, %o1 7792 bnz 1b 7793 wrpr %o2, 0, %tl 7794 2: 7795 retl 7796 nop 7797 7798 /* 7799 * Debug stuff. Restore trap registers from buffer. 7800 * 7801 * %o0 = %tl 7802 * %o1 = *ts 7803 * 7804 * Maybe this should be re-written to increment tl instead of decrementing. 7805 */ 7806 ENTRY(restoretstate) 7807 flushw ! Make sure we don't have stack probs & lose hibits of %o 7808 brz,pn %o0, 2f 7809 mov %o0, %o2 7810 wrpr %o0, 0, %tl 7811 1: 7812 ldx [%o1], %o3 7813 deccc %o2 7814 inc 8, %o1 7815 wrpr %o3, 0, %tstate 7816 ldx [%o1], %o4 7817 inc 8, %o1 7818 wrpr %o4, 0, %tpc 7819 ldx [%o1], %o5 7820 inc 8, %o1 7821 wrpr %o5, 0, %tnpc 7822 ldx [%o1], %o4 7823 inc 8, %o1 7824 wrpr %o4, 0, %tt 7825 bnz 1b 7826 wrpr %o2, 0, %tl 7827 2: 7828 retl 7829 wrpr %o0, 0, %tl 7830 7831 /* 7832 * Switch to context in abs(%o0) 7833 */ 7834 ENTRY(switchtoctx_us) 7835 set DEMAP_CTX_SECONDARY, %o3 7836 stxa %o3, [%o3] ASI_DMMU_DEMAP 7837 mov CTX_SECONDARY, %o4 7838 stxa %o3, [%o3] ASI_IMMU_DEMAP 7839 membar #Sync 7840 stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid 7841 sethi %hi(KERNBASE), %o2 7842 membar #Sync 7843 flush %o2 7844 retl 7845 nop 7846 7847 ENTRY(switchtoctx_usiii) 7848 mov CTX_SECONDARY, %o4 7849 ldxa [%o4] ASI_DMMU, %o2 ! Load secondary context 7850 mov CTX_PRIMARY, %o5 7851 ldxa [%o5] ASI_DMMU, %o1 ! Save primary context 7852 membar #LoadStore 7853 stxa %o2, [%o5] ASI_DMMU ! Insert secondary for demap 7854 membar #Sync 7855 set DEMAP_CTX_PRIMARY, %o3 7856 stxa %o3, [%o3] ASI_DMMU_DEMAP 7857 membar #Sync 7858 stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid 7859 membar #Sync 7860 stxa %o1, [%o5] ASI_DMMU ! Restore primary context 7861 sethi %hi(KERNBASE), %o2 7862 membar #Sync 7863 flush %o2 7864 retl 7865 nop 7866 7867 #ifndef _LP64 7868 /* 7869 * Convert to 32-bit stack then call OF_sym2val() 7870 */ 7871 ENTRY(OF_sym2val32) 7872 save %sp, -CC64FSZ, %sp 7873 btst 7, %i0 7874 bnz,pn %icc, 1f 7875 add %sp, BIAS, %o1 7876 btst 1, %sp 7877 movnz %icc, %o1, %sp 7878 call _C_LABEL(OF_sym2val) 7879 mov %i0, %o0 7880 1: 7881 ret 7882 restore %o0, 0, %o0 7883 7884 /* 7885 * Convert to 32-bit stack then call OF_val2sym() 7886 */ 7887 ENTRY(OF_val2sym32) 7888 save %sp, -CC64FSZ, %sp 7889 btst 7, %i0 7890 bnz,pn %icc, 1f 7891 add %sp, BIAS, %o1 7892 btst 1, %sp 7893 movnz %icc, %o1, %sp 7894 call _C_LABEL(OF_val2sym) 7895 mov %i0, %o0 7896 1: 7897 ret 7898 restore %o0, 0, %o0 7899 #endif /* _LP64 */ 7900 #endif /* DDB */ 7901 7902 7903 #if defined(MULTIPROCESSOR) 7904 /* 7905 * IPI target function to setup a C compatible environment and call a MI function. 7906 * 7907 * On entry: 7908 * We are on one of the alternate set of globals 7909 * %g2 = function to call 7910 * %g3 = single argument to called function 7911 */ 7912 ENTRY(sparc64_ipi_ccall) 7913 #ifdef TRAPS_USE_IG 7914 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 7915 #endif 7916 TRAP_SETUP(-CC64FSZ-TF_SIZE) 7917 7918 #ifdef DEBUG 7919 rdpr %tt, %o1 ! debug 7920 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 7921 #endif 7922 mov %g3, %o0 ! save argument of function to call 7923 mov %g2, %o5 ! save function pointer 7924 7925 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 7926 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 7927 rdpr %tpc, %o2 ! (pc) 7928 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 7929 rdpr %tstate, %g1 7930 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 7931 rdpr %tnpc, %o3 7932 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 7933 rd %y, %o4 7934 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 7935 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 7936 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 7937 7938 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 7939 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 7940 stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] 7941 st %o4, [%sp + CC64FSZ + STKB + TF_Y] 7942 7943 rdpr %pil, %g5 7944 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 7945 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 7946 7947 rdpr %tl, %g7 7948 dec %g7 7949 movrlz %g7, %g0, %g7 7950 wrpr %g0, %g7, %tl 7951 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 7952 !! In our case we need to clear it before calling any C-code 7953 clr %g4 7954 wr %g0, ASI_NUCLEUS, %asi ! default kernel ASI 7955 7956 call %o5 ! call function 7957 nop 7958 7959 b return_from_trap ! and return from IPI 7960 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 7961 7962 #endif 7963 7964 ENTRY(paravirt_membar_sync) 7965 /* 7966 * Store-before-load ordering with respect to matching logic 7967 * on the hypervisor side. 7968 * 7969 * This is the same as membar_sync, but without patching or 7970 * conditionalizing away the MEMBAR instruction on uniprocessor 7971 * builds or boots -- because under virtualization, we still 7972 * have to coordinate with a `device' backed by a hypervisor 7973 * that is potentially on another physical CPU even if we 7974 * observe only one virtual CPU as the guest. 7975 * 7976 * See common/lib/libc/arch/sparc64/atomic/membar_ops.S for why 7977 * we avoid using the delay slot and keep this in sync with the 7978 * implementation of membar_sync there. 7979 */ 7980 membar #StoreLoad 7981 retl 7982 nop 7983 END(paravirt_membar_sync) 7984 7985 .data 7986 _ALIGN 7987 #if NKSYMS || defined(DDB) || defined(MODULAR) 7988 .globl _C_LABEL(esym) 7989 _C_LABEL(esym): 7990 POINTER 0 7991 .globl _C_LABEL(ssym) 7992 _C_LABEL(ssym): 7993 POINTER 0 7994 #endif 7995 .comm _C_LABEL(promvec), PTRSZ 7996 7997 #ifdef DEBUG 7998 .comm _C_LABEL(trapdebug), 4 7999 .comm _C_LABEL(pmapdebug), 4 8000 #endif 8001
Indexes created Sat Jul 04 00:25:45 UTC 2026