cpuswitch.S revision 1.45
1 /* $NetBSD: cpuswitch.S,v 1.45 2007/02/09 21:55:02 ad Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_arm32_pmap.h" 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 85 #include "assym.h" 86 #include <machine/param.h> 87 #include <machine/cpu.h> 88 #include <machine/frame.h> 89 #include <machine/asm.h> 90 91 /* LINTSTUB: include <sys/param.h> */ 92 93 #undef IRQdisable 94 #undef IRQenable 95 96 /* 97 * New experimental definitions of IRQdisable and IRQenable 98 * These keep FIQ's enabled since FIQ's are special. 99 */ 100 101 #define IRQdisable \ 102 mrs r14, cpsr ; \ 103 orr r14, r14, #(I32_bit) ; \ 104 msr cpsr_c, r14 ; \ 105 106 #define IRQenable \ 107 mrs r14, cpsr ; \ 108 bic r14, r14, #(I32_bit) ; \ 109 msr cpsr_c, r14 ; \ 110 111 /* 112 * These are used for switching the translation table/DACR. 113 * Since the vector page can be invalid for a short time, we must 114 * disable both regular IRQs *and* FIQs. 115 * 116 * XXX: This is not necessary if the vector table is relocated. 117 */ 118 #define IRQdisableALL \ 119 mrs r14, cpsr ; \ 120 orr r14, r14, #(I32_bit | F32_bit) ; \ 121 msr cpsr_c, r14 122 123 #define IRQenableALL \ 124 mrs r14, cpsr ; \ 125 bic r14, r14, #(I32_bit | F32_bit) ; \ 126 msr cpsr_c, r14 127 128 .text 129 130 .Lwhichqs: 131 .word _C_LABEL(sched_whichqs) 132 133 .Lqs: 134 .word _C_LABEL(sched_qs) 135 136 /* 137 * cpuswitch() 138 * 139 * preforms a process context switch. 140 * This function has several entry points 141 */ 142 143 #ifdef MULTIPROCESSOR 144 .Lcpu_info_store: 145 .word _C_LABEL(cpu_info_store) 146 .Lcurlwp: 147 /* FIXME: This is bogus in the general case. */ 148 .word _C_LABEL(cpu_info_store) + CI_CURLWP 149 150 .Lcurpcb: 151 .word _C_LABEL(cpu_info_store) + CI_CURPCB 152 #else 153 .Lcurlwp: 154 .word _C_LABEL(curlwp) 155 156 .Lcurpcb: 157 .word _C_LABEL(curpcb) 158 #endif 159 160 .Lwant_resched: 161 .word _C_LABEL(want_resched) 162 163 .Lcpufuncs: 164 .word _C_LABEL(cpufuncs) 165 166 #ifndef MULTIPROCESSOR 167 .data 168 .global _C_LABEL(curpcb) 169 _C_LABEL(curpcb): 170 .word 0x00000000 171 .text 172 #endif 173 174 .Lblock_userspace_access: 175 .word _C_LABEL(block_userspace_access) 176 177 .Lcpu_do_powersave: 178 .word _C_LABEL(cpu_do_powersave) 179 180 .Lpmap_kernel_cstate: 181 .word (kernel_pmap_store + PMAP_CSTATE) 182 183 .Llast_cache_state_ptr: 184 .word _C_LABEL(pmap_cache_state) 185 186 /* 187 * Idle loop, exercised while waiting for a process to wake up. 188 * 189 * NOTE: When we jump back to .Lswitch_search, we must have a 190 * pointer to whichqs in r7, which is what it is when we arrive 191 * here. 192 */ 193 /* LINTSTUB: Ignore */ 194 ASENTRY_NP(idle) 195 ldr r6, .Lcpu_do_powersave 196 IRQenable /* Enable interrupts */ 197 ldr r6, [r6] /* r6 = cpu_do_powersave */ 198 199 bl _C_LABEL(sched_unlock_idle) 200 201 /* Drop to spl0 (returns the current spl level in r0). */ 202 #ifdef __NEWINTR 203 mov r0, #(IPL_NONE) 204 bl _C_LABEL(_spllower) 205 #else /* ! __NEWINTR */ 206 mov r0, #(_SPL_0) 207 bl _C_LABEL(splx) 208 #endif /* __NEWINTR */ 209 210 teq r6, #0 /* cpu_do_powersave non zero? */ 211 ldrne r6, .Lcpufuncs 212 mov r4, r0 /* Old interrupt level to r4 */ 213 ldrne r6, [r6, #(CF_SLEEP)] 214 215 /* 216 * Main idle loop. 217 * r6 points to power-save idle function if required, else NULL. 218 */ 219 1: ldr r3, [r7] /* r3 = sched_whichqs */ 220 teq r3, #0 221 bne 2f /* We have work to do */ 222 teq r6, #0 /* Powersave idle? */ 223 beq 1b /* Nope. Just sit-n-spin. */ 224 225 /* 226 * Before going into powersave idle mode, disable interrupts 227 * and check sched_whichqs one more time. 228 */ 229 IRQdisableALL 230 ldr r3, [r7] 231 mov r0, #0 232 teq r3, #0 /* sched_whichqs still zero? */ 233 moveq lr, pc 234 moveq pc, r6 /* If so, do powersave idle */ 235 IRQenableALL 236 b 1b /* Back around */ 237 238 /* 239 * sched_whichqs indicates that at least one lwp is ready to run. 240 * Restore the original interrupt priority level, grab the 241 * scheduler lock if necessary, and jump back into cpu_switch. 242 */ 243 2: mov r0, r4 244 bl _C_LABEL(splx) 245 adr lr, .Lswitch_search 246 b _C_LABEL(sched_lock_idle) 247 248 249 /* 250 * Find a new lwp to run, save the current context and 251 * load the new context 252 * 253 * Arguments: 254 * r0 'struct lwp *' of the current LWP 255 */ 256 257 ENTRY(cpu_switch) 258 /* 259 * Local register usage. Some of these registers are out of date. 260 * r1 = oldlwp 261 * r2 = spl level 262 * r3 = whichqs 263 * r4 = queue 264 * r5 = &qs[queue] 265 * r6 = newlwp 266 * r7 = scratch 267 */ 268 stmfd sp!, {r4-r7, lr} 269 270 /* 271 * Indicate that there is no longer a valid process (curlwp = 0). 272 * Zero the current PCB pointer while we're at it. 273 */ 274 ldr r7, .Lcurlwp 275 ldr r6, .Lcurpcb 276 mov r2, #0x00000000 277 str r2, [r7] /* curlwp = NULL */ 278 str r2, [r6] /* curpcb = NULL */ 279 280 /* stash the old lwp while we call functions */ 281 mov r5, r0 282 283 /* First phase : find a new lwp */ 284 ldr r7, .Lwhichqs 285 286 /* rem: r5 = old lwp */ 287 /* rem: r7 = &whichqs */ 288 289 .Lswitch_search: 290 IRQdisable 291 292 /* Do we have any active queues */ 293 ldr r3, [r7] 294 295 /* If not we must idle until we do. */ 296 teq r3, #0x00000000 297 beq _ASM_LABEL(idle) 298 299 /* put old lwp back in r1 */ 300 mov r1, r5 301 302 /* rem: r1 = old lwp */ 303 /* rem: r3 = whichqs */ 304 /* rem: interrupts are disabled */ 305 306 /* used further down, saves SA stall */ 307 ldr r6, .Lqs 308 309 /* 310 * We have found an active queue. Currently we do not know which queue 311 * is active just that one of them is. 312 */ 313 /* Non-Xscale version of the ffs algorithm devised by d.seal and 314 * posted to comp.sys.arm on 16 Feb 1994. 315 */ 316 rsb r5, r3, #0 317 ands r0, r3, r5 318 319 #ifndef __XSCALE__ 320 adr r5, .Lcpu_switch_ffs_table 321 322 /* X = R0 */ 323 orr r4, r0, r0, lsl #4 /* r4 = X * 0x11 */ 324 orr r4, r4, r4, lsl #6 /* r4 = X * 0x451 */ 325 rsb r4, r4, r4, lsl #16 /* r4 = X * 0x0450fbaf */ 326 327 /* now lookup in table indexed on top 6 bits of a4 */ 328 ldrb r4, [ r5, r4, lsr #26 ] 329 330 #else /* __XSCALE__ */ 331 clz r4, r0 332 rsb r4, r4, #31 333 #endif /* __XSCALE__ */ 334 335 /* rem: r0 = bit mask of chosen queue (1 << r4) */ 336 /* rem: r1 = old lwp */ 337 /* rem: r3 = whichqs */ 338 /* rem: r4 = queue number */ 339 /* rem: interrupts are disabled */ 340 341 /* Get the address of the queue (&qs[queue]) */ 342 add r5, r6, r4, lsl #3 343 344 /* 345 * Get the lwp from the queue and place the next process in 346 * the queue at the head. This basically unlinks the lwp at 347 * the head of the queue. 348 */ 349 ldr r6, [r5, #(L_FORW)] 350 351 #ifdef DIAGNOSTIC 352 cmp r6, r5 353 beq .Lswitch_bogons 354 #endif 355 356 /* rem: r6 = new lwp */ 357 ldr r7, [r6, #(L_FORW)] 358 str r7, [r5, #(L_FORW)] 359 360 /* 361 * Test to see if the queue is now empty. If the head of the queue 362 * points to the queue itself then there are no more lwps in 363 * the queue. We can therefore clear the queue not empty flag held 364 * in r3. 365 */ 366 367 teq r5, r7 368 biceq r3, r3, r0 369 370 /* rem: r0 = bit mask of chosen queue (1 << r4) - NOT NEEDED AN MORE */ 371 372 /* Fix the back pointer for the lwp now at the head of the queue. */ 373 ldr r0, [r6, #(L_BACK)] 374 str r0, [r7, #(L_BACK)] 375 376 /* Update the RAM copy of the queue not empty flags word. */ 377 ldreq r7, .Lwhichqs 378 streq r3, [r7] 379 380 /* rem: r1 = old lwp */ 381 /* rem: r3 = whichqs - NOT NEEDED ANY MORE */ 382 /* rem: r4 = queue number - NOT NEEDED ANY MORE */ 383 /* rem: r6 = new lwp */ 384 /* rem: interrupts are disabled */ 385 386 /* Clear the want_resched flag */ 387 ldr r7, .Lwant_resched 388 mov r0, #0x00000000 389 str r0, [r7] 390 391 /* 392 * Clear the back pointer of the lwp we have removed from 393 * the head of the queue. The new lwp is isolated now. 394 */ 395 str r0, [r6, #(L_BACK)] 396 397 /* 398 * unlock the sched_lock, but leave interrupts off, for now. 399 */ 400 mov r7, r1 401 bl _C_LABEL(sched_unlock_idle) 402 mov r1, r7 403 404 405 .Lswitch_resume: 406 /* rem: r1 = old lwp */ 407 /* rem: r4 = return value [not used if came from cpu_switchto()] */ 408 /* rem: r6 = new lwp */ 409 /* rem: interrupts are disabled */ 410 411 #ifdef MULTIPROCESSOR 412 /* XXX use curcpu() */ 413 ldr r0, .Lcpu_info_store 414 str r0, [r6, #(L_CPU)] 415 #else 416 /* l->l_cpu initialized in fork1() for single-processor */ 417 #endif 418 419 /* Process is now on a processor. */ 420 mov r0, #LSONPROC /* l->l_stat = LSONPROC */ 421 str r0, [r6, #(L_STAT)] 422 423 /* We have a new curlwp now so make a note it */ 424 ldr r7, .Lcurlwp 425 str r6, [r7] 426 427 /* Hook in a new pcb */ 428 ldr r7, .Lcurpcb 429 ldr r0, [r6, #(L_ADDR)] 430 str r0, [r7] 431 432 /* At this point we can allow IRQ's again. */ 433 IRQenable 434 435 /* rem: r1 = old lwp */ 436 /* rem: r4 = return value */ 437 /* rem: r6 = new lwp */ 438 /* rem: interrupts are enabled */ 439 440 /* 441 * If the new lwp is the same as the lwp that called 442 * cpu_switch() then we do not need to save and restore any 443 * contexts. This means we can make a quick exit. 444 * The test is simple if curlwp on entry (now in r1) is the 445 * same as the lwp removed from the queue we can jump to the exit. 446 */ 447 teq r1, r6 448 moveq r4, #0x00000000 /* default to "didn't switch" */ 449 beq .Lswitch_return 450 451 /* 452 * At this point, we are guaranteed to be switching to 453 * a new lwp. 454 */ 455 mov r4, #0x00000001 456 457 /* Remember the old lwp in r0 */ 458 mov r0, r1 459 460 /* 461 * If the old lwp on entry to cpu_switch was zero then the 462 * process that called it was exiting. This means that we do 463 * not need to save the current context. Instead we can jump 464 * straight to restoring the context for the new process. 465 */ 466 teq r0, #0x00000000 467 beq .Lswitch_exited 468 469 /* rem: r0 = old lwp */ 470 /* rem: r4 = return value */ 471 /* rem: r6 = new lwp */ 472 /* rem: interrupts are enabled */ 473 474 /* Stage two : Save old context */ 475 476 /* Get the user structure for the old lwp. */ 477 ldr r1, [r0, #(L_ADDR)] 478 479 /* Save all the registers in the old lwp's pcb */ 480 #ifndef __XSCALE__ 481 add r7, r1, #(PCB_R8) 482 stmia r7, {r8-r13} 483 #else 484 strd r8, [r1, #(PCB_R8)] 485 strd r10, [r1, #(PCB_R10)] 486 strd r12, [r1, #(PCB_R12)] 487 #endif 488 489 /* 490 * NOTE: We can now use r8-r13 until it is time to restore 491 * them for the new process. 492 */ 493 494 /* Remember the old PCB. */ 495 mov r8, r1 496 497 /* r1 now free! */ 498 499 /* Get the user structure for the new process in r9 */ 500 ldr r9, [r6, #(L_ADDR)] 501 502 /* 503 * This can be optimised... We know we want to go from SVC32 504 * mode to UND32 mode 505 */ 506 mrs r3, cpsr 507 bic r2, r3, #(PSR_MODE) 508 orr r2, r2, #(PSR_UND32_MODE | I32_bit) 509 msr cpsr_c, r2 510 511 str sp, [r8, #(PCB_UND_SP)] 512 513 msr cpsr_c, r3 /* Restore the old mode */ 514 515 /* rem: r0 = old lwp */ 516 /* rem: r4 = return value */ 517 /* rem: r6 = new lwp */ 518 /* rem: r8 = old PCB */ 519 /* rem: r9 = new PCB */ 520 /* rem: interrupts are enabled */ 521 522 /* What else needs to be saved Only FPA stuff when that is supported */ 523 524 /* Third phase : restore saved context */ 525 526 /* rem: r0 = old lwp */ 527 /* rem: r4 = return value */ 528 /* rem: r6 = new lwp */ 529 /* rem: r8 = old PCB */ 530 /* rem: r9 = new PCB */ 531 /* rem: interrupts are enabled */ 532 533 /* 534 * Get the new L1 table pointer into r11. If we're switching to 535 * an LWP with the same address space as the outgoing one, we can 536 * skip the cache purge and the TTB load. 537 * 538 * To avoid data dep stalls that would happen anyway, we try 539 * and get some useful work done in the mean time. 540 */ 541 ldr r10, [r8, #(PCB_PAGEDIR)] /* r10 = old L1 */ 542 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 543 544 ldr r0, [r8, #(PCB_DACR)] /* r0 = old DACR */ 545 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 546 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = &new_pmap->pm_cstate */ 547 ldr r5, .Llast_cache_state_ptr /* Previous thread's cstate */ 548 549 teq r10, r11 /* Same L1? */ 550 ldr r5, [r5] 551 cmpeq r0, r1 /* Same DACR? */ 552 beq .Lcs_context_switched /* yes! */ 553 554 ldr r3, .Lblock_userspace_access 555 mov r12, #0 556 cmp r5, #0 /* No last vm? (switch_exit) */ 557 beq .Lcs_cache_purge_skipped /* No, we can skip cache flsh */ 558 559 mov r2, #DOMAIN_CLIENT 560 cmp r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 561 beq .Lcs_cache_purge_skipped /* Yup. Don't flush cache */ 562 563 cmp r5, r8 /* Same userland VM space? */ 564 ldrneb r12, [r5, #(CS_CACHE_ID)] /* Last VM space cache state */ 565 566 /* 567 * We're definately switching to a new userland VM space, 568 * and the previous userland VM space has yet to be flushed 569 * from the cache/tlb. 570 * 571 * r12 holds the previous VM space's cs_cache_id state 572 */ 573 tst r12, #0xff /* Test cs_cache_id */ 574 beq .Lcs_cache_purge_skipped /* VM space is not in cache */ 575 576 /* 577 * Definately need to flush the cache. 578 * Mark the old VM space as NOT being resident in the cache. 579 */ 580 mov r2, #0x00000000 581 strb r2, [r5, #(CS_CACHE_ID)] 582 strb r2, [r5, #(CS_CACHE_D)] 583 584 /* 585 * Don't allow user space access between the purge and the switch. 586 */ 587 mov r2, #0x00000001 588 str r2, [r3] 589 590 stmfd sp!, {r0-r3} 591 ldr r1, .Lcpufuncs 592 mov lr, pc 593 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 594 ldmfd sp!, {r0-r3} 595 596 .Lcs_cache_purge_skipped: 597 /* rem: r1 = new DACR */ 598 /* rem: r3 = &block_userspace_access */ 599 /* rem: r4 = return value */ 600 /* rem: r5 = &old_pmap->pm_cstate (or NULL) */ 601 /* rem: r6 = new lwp */ 602 /* rem: r8 = &new_pmap->pm_cstate */ 603 /* rem: r9 = new PCB */ 604 /* rem: r10 = old L1 */ 605 /* rem: r11 = new L1 */ 606 607 mov r2, #0x00000000 608 ldr r7, [r9, #(PCB_PL1VEC)] 609 610 /* 611 * At this point we need to kill IRQ's again. 612 * 613 * XXXSCW: Don't need to block FIQs if vectors have been relocated 614 */ 615 IRQdisableALL 616 617 /* 618 * Interrupts are disabled so we can allow user space accesses again 619 * as none will occur until interrupts are re-enabled after the 620 * switch. 621 */ 622 str r2, [r3] 623 624 /* 625 * Ensure the vector table is accessible by fixing up the L1 626 */ 627 cmp r7, #0 /* No need to fixup vector table? */ 628 ldrne r2, [r7] /* But if yes, fetch current value */ 629 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 630 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for new context */ 631 cmpne r2, r0 /* Stuffing the same value? */ 632 #ifndef PMAP_INCLUDE_PTE_SYNC 633 strne r0, [r7] /* Nope, update it */ 634 #else 635 beq .Lcs_same_vector 636 str r0, [r7] /* Otherwise, update it */ 637 638 /* 639 * Need to sync the cache to make sure that last store is 640 * visible to the MMU. 641 */ 642 ldr r2, .Lcpufuncs 643 mov r0, r7 644 mov r1, #4 645 mov lr, pc 646 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 647 648 .Lcs_same_vector: 649 #endif /* PMAP_INCLUDE_PTE_SYNC */ 650 651 cmp r10, r11 /* Switching to the same L1? */ 652 ldr r10, .Lcpufuncs 653 beq .Lcs_same_l1 /* Yup. */ 654 655 /* 656 * Do a full context switch, including full TLB flush. 657 */ 658 mov r0, r11 659 mov lr, pc 660 ldr pc, [r10, #CF_CONTEXT_SWITCH] 661 662 /* 663 * Mark the old VM space as NOT being resident in the TLB 664 */ 665 mov r2, #0x00000000 666 cmp r5, #0 667 strneh r2, [r5, #(CS_TLB_ID)] 668 b .Lcs_context_switched 669 670 /* 671 * We're switching to a different process in the same L1. 672 * In this situation, we only need to flush the TLB for the 673 * vector_page mapping, and even then only if r7 is non-NULL. 674 */ 675 .Lcs_same_l1: 676 cmp r7, #0 677 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 678 movne lr, pc 679 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 680 681 .Lcs_context_switched: 682 /* rem: r8 = &new_pmap->pm_cstate */ 683 684 /* XXXSCW: Safe to re-enable FIQs here */ 685 686 /* 687 * The new VM space is live in the cache and TLB. 688 * Update its cache/tlb state, and if it's not the kernel 689 * pmap, update the 'last cache state' pointer. 690 */ 691 mov r2, #-1 692 ldr r5, .Lpmap_kernel_cstate 693 ldr r0, .Llast_cache_state_ptr 694 str r2, [r8, #(CS_ALL)] 695 cmp r5, r8 696 strne r8, [r0] 697 698 /* rem: r4 = return value */ 699 /* rem: r6 = new lwp */ 700 /* rem: r9 = new PCB */ 701 702 /* 703 * This can be optimised... We know we want to go from SVC32 704 * mode to UND32 mode 705 */ 706 mrs r3, cpsr 707 bic r2, r3, #(PSR_MODE) 708 orr r2, r2, #(PSR_UND32_MODE) 709 msr cpsr_c, r2 710 711 ldr sp, [r9, #(PCB_UND_SP)] 712 713 msr cpsr_c, r3 /* Restore the old mode */ 714 715 /* Restore all the save registers */ 716 #ifndef __XSCALE__ 717 add r7, r9, #PCB_R8 718 ldmia r7, {r8-r13} 719 720 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 721 #else 722 mov r7, r9 723 ldr r8, [r7, #(PCB_R8)] 724 ldr r9, [r7, #(PCB_R9)] 725 ldr r10, [r7, #(PCB_R10)] 726 ldr r11, [r7, #(PCB_R11)] 727 ldr r12, [r7, #(PCB_R12)] 728 ldr r13, [r7, #(PCB_SP)] 729 #endif 730 731 ldr r5, [r6, #(L_PROC)] /* fetch the proc for below */ 732 733 /* rem: r4 = return value */ 734 /* rem: r5 = new lwp's proc */ 735 /* rem: r6 = new lwp */ 736 /* rem: r7 = new pcb */ 737 738 #ifdef ARMFPE 739 add r0, r7, #(USER_SIZE) & 0x00ff 740 add r0, r0, #(USER_SIZE) & 0xff00 741 bl _C_LABEL(arm_fpe_core_changecontext) 742 #endif 743 744 /* We can enable interrupts again */ 745 IRQenableALL 746 747 /* rem: r4 = return value */ 748 /* rem: r5 = new lwp's proc */ 749 /* rem: r6 = new lwp */ 750 /* rem: r7 = new PCB */ 751 752 /* 753 * Check for restartable atomic sequences (RAS). 754 */ 755 756 ldr r2, [r5, #(P_RASLIST)] 757 ldr r1, [r7, #(PCB_TF)] /* r1 = trapframe (used below) */ 758 teq r2, #0 /* p->p_nras == 0? */ 759 bne .Lswitch_do_ras /* no, check for one */ 760 761 .Lswitch_return: 762 /* cpu_switch returns 1 == switched, 0 == didn't switch */ 763 mov r0, r4 764 765 /* 766 * Pull the registers that got pushed when either savectx() or 767 * cpu_switch() was called and return. 768 */ 769 ldmfd sp!, {r4-r7, pc} 770 771 .Lswitch_do_ras: 772 ldr r1, [r1, #(TF_PC)] /* second ras_lookup() arg */ 773 mov r0, r5 /* first ras_lookup() arg */ 774 bl _C_LABEL(ras_lookup) 775 cmn r0, #1 /* -1 means "not in a RAS" */ 776 ldrne r1, [r7, #(PCB_TF)] 777 strne r0, [r1, #(TF_PC)] 778 b .Lswitch_return 779 780 .Lswitch_exited: 781 /* 782 * We skip the cache purge because switch_exit() already did it. 783 * Load up registers the way .Lcs_cache_purge_skipped expects. 784 * Userspace access already blocked by switch_exit(). 785 */ 786 ldr r9, [r6, #(L_ADDR)] /* r9 = new PCB */ 787 ldr r3, .Lblock_userspace_access 788 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 789 mov r5, #0 /* No previous cache state */ 790 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 791 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = new cache state */ 792 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 793 b .Lcs_cache_purge_skipped 794 795 796 #ifdef DIAGNOSTIC 797 .Lswitch_bogons: 798 adr r0, .Lswitch_panic_str 799 bl _C_LABEL(panic) 800 1: nop 801 b 1b 802 803 .Lswitch_panic_str: 804 .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n" 805 #endif 806 807 /* 808 * cpu_switchto(struct lwp *current, struct lwp *next) 809 * Switch to the specified next LWP 810 * Arguments: 811 * 812 * r0 'struct lwp *' of the current LWP 813 * r1 'struct lwp *' of the LWP to switch to 814 */ 815 ENTRY(cpu_switchto) 816 stmfd sp!, {r4-r7, lr} 817 818 mov r6, r1 /* save new lwp */ 819 820 mov r5, r0 /* save old lwp */ 821 bl _C_LABEL(sched_unlock_idle) 822 mov r1, r5 823 824 IRQdisable 825 826 /* 827 * Okay, set up registers the way cpu_switch() wants them, 828 * and jump into the middle of it (where we bring up the 829 * new process). 830 * 831 * r1 = old lwp (r6 = new lwp) 832 */ 833 b .Lswitch_resume 834 835 /* 836 * void switch_exit(struct lwp *l, struct lwp *l0, void (*exit)(struct lwp *)); 837 * Switch to lwp0's saved context and deallocate the address space and kernel 838 * stack for l. Then jump into cpu_switch(), as if we were in lwp0 all along. 839 */ 840 841 /* LINTSTUB: Func: void switch_exit(struct lwp *l, struct lwp *l0, void (*func)(struct lwp *)) */ 842 ENTRY(switch_exit) 843 /* 844 * The process is going away, so we can use callee-saved 845 * registers here without having to save them. 846 */ 847 848 mov r4, r0 849 ldr r0, .Lcurlwp 850 851 mov r5, r1 852 ldr r1, .Lblock_userspace_access 853 854 mov r6, r2 855 856 /* 857 * r4 = lwp 858 * r5 = lwp0 859 * r6 = exit func 860 */ 861 862 mov r2, #0x00000000 /* curlwp = NULL */ 863 str r2, [r0] 864 865 /* 866 * We're about to clear both the cache and the TLB. 867 * Make sure to zap the 'last cache state' pointer since the 868 * pmap might be about to go away. Also ensure the outgoing 869 * VM space's cache state is marked as NOT resident in the 870 * cache, and that lwp0's cache state IS resident. 871 */ 872 ldr r7, [r4, #(L_ADDR)] /* r7 = old lwp's PCB */ 873 ldr r0, .Llast_cache_state_ptr /* Last userland cache state */ 874 ldr r9, [r7, #(PCB_CSTATE)] /* Fetch cache state pointer */ 875 ldr r3, [r5, #(L_ADDR)] /* r3 = lwp0's PCB */ 876 str r2, [r0] /* No previous cache state */ 877 str r2, [r9, #(CS_ALL)] /* Zap old lwp's cache state */ 878 ldr r3, [r3, #(PCB_CSTATE)] /* lwp0's cache state */ 879 mov r2, #-1 880 str r2, [r3, #(CS_ALL)] /* lwp0 is in da cache! */ 881 882 /* 883 * Don't allow user space access between the purge and the switch. 884 */ 885 mov r2, #0x00000001 886 str r2, [r1] 887 888 /* Switch to lwp0 context */ 889 890 ldr r9, .Lcpufuncs 891 mov lr, pc 892 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 893 894 ldr r0, [r7, #(PCB_PL1VEC)] 895 ldr r1, [r7, #(PCB_DACR)] 896 897 /* 898 * r0 = Pointer to L1 slot for vector_page (or NULL) 899 * r1 = lwp0's DACR 900 * r4 = lwp we're switching from 901 * r5 = lwp0 902 * r6 = exit func 903 * r7 = lwp0's PCB 904 * r9 = cpufuncs 905 */ 906 907 IRQdisableALL 908 909 /* 910 * Ensure the vector table is accessible by fixing up lwp0's L1 911 */ 912 cmp r0, #0 /* No need to fixup vector table? */ 913 ldrne r3, [r0] /* But if yes, fetch current value */ 914 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 915 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 916 cmpne r3, r2 /* Stuffing the same value? */ 917 strne r2, [r0] /* Store if not. */ 918 919 #ifdef PMAP_INCLUDE_PTE_SYNC 920 /* 921 * Need to sync the cache to make sure that last store is 922 * visible to the MMU. 923 */ 924 movne r1, #4 925 movne lr, pc 926 ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 927 #endif /* PMAP_INCLUDE_PTE_SYNC */ 928 929 /* 930 * Note: We don't do the same optimisation as cpu_switch() with 931 * respect to avoiding flushing the TLB if we're switching to 932 * the same L1 since this process' VM space may be about to go 933 * away, so we don't want *any* turds left in the TLB. 934 */ 935 936 /* Switch the memory to the new process */ 937 ldr r0, [r7, #(PCB_PAGEDIR)] 938 mov lr, pc 939 ldr pc, [r9, #CF_CONTEXT_SWITCH] 940 941 ldr r0, .Lcurpcb 942 943 /* Restore all the save registers */ 944 #ifndef __XSCALE__ 945 add r1, r7, #PCB_R8 946 ldmia r1, {r8-r13} 947 #else 948 ldr r8, [r7, #(PCB_R8)] 949 ldr r9, [r7, #(PCB_R9)] 950 ldr r10, [r7, #(PCB_R10)] 951 ldr r11, [r7, #(PCB_R11)] 952 ldr r12, [r7, #(PCB_R12)] 953 ldr r13, [r7, #(PCB_SP)] 954 #endif 955 str r7, [r0] /* curpcb = lwp0's PCB */ 956 957 IRQenableALL 958 959 /* 960 * Schedule the vmspace and stack to be freed. 961 */ 962 mov r0, r4 /* {lwp_}exit2(l) */ 963 mov lr, pc 964 mov pc, r6 965 966 bl _C_LABEL(sched_lock_idle) 967 968 ldr r7, .Lwhichqs /* r7 = &whichqs */ 969 mov r5, #0x00000000 /* r5 = old lwp = NULL */ 970 b .Lswitch_search 971 972 /* LINTSTUB: Func: void savectx(struct pcb *pcb) */ 973 ENTRY(savectx) 974 /* 975 * r0 = pcb 976 */ 977 978 /* Push registers.*/ 979 stmfd sp!, {r4-r7, lr} 980 981 /* Store all the registers in the process's pcb */ 982 #ifndef __XSCALE__ 983 add r2, r0, #(PCB_R8) 984 stmia r2, {r8-r13} 985 #else 986 strd r8, [r0, #(PCB_R8)] 987 strd r10, [r0, #(PCB_R10)] 988 strd r12, [r0, #(PCB_R12)] 989 #endif 990 991 /* Pull the regs of the stack */ 992 ldmfd sp!, {r4-r7, pc} 993 994 ENTRY(proc_trampoline) 995 #ifdef __NEWINTR 996 mov r0, #(IPL_NONE) 997 bl _C_LABEL(_spllower) 998 #else /* ! __NEWINTR */ 999 mov r0, #(_SPL_0) 1000 bl _C_LABEL(splx) 1001 #endif /* __NEWINTR */ 1002 1003 #ifdef MULTIPROCESSOR 1004 bl _C_LABEL(proc_trampoline_mp) 1005 #endif 1006 mov r0, r5 1007 mov r1, sp 1008 mov lr, pc 1009 mov pc, r4 1010 1011 /* Kill irq's */ 1012 mrs r0, cpsr 1013 orr r0, r0, #(I32_bit) 1014 msr cpsr_c, r0 1015 1016 PULLFRAME 1017 1018 movs pc, lr /* Exit */ 1019 1020 #ifndef __XSCALE__ 1021 .type .Lcpu_switch_ffs_table, _ASM_TYPE_OBJECT; 1022 .Lcpu_switch_ffs_table: 1023 /* same as ffs table but all nums are -1 from that */ 1024 /* 0 1 2 3 4 5 6 7 */ 1025 .byte 0, 0, 1, 12, 2, 6, 0, 13 /* 0- 7 */ 1026 .byte 3, 0, 7, 0, 0, 0, 0, 14 /* 8-15 */ 1027 .byte 10, 4, 0, 0, 8, 0, 0, 25 /* 16-23 */ 1028 .byte 0, 0, 0, 0, 0, 21, 27, 15 /* 24-31 */ 1029 .byte 31, 11, 5, 0, 0, 0, 0, 0 /* 32-39 */ 1030 .byte 9, 0, 0, 24, 0, 0, 20, 26 /* 40-47 */ 1031 .byte 30, 0, 0, 0, 0, 23, 0, 19 /* 48-55 */ 1032 .byte 29, 0, 22, 18, 28, 17, 16, 0 /* 56-63 */ 1033 #endif /* !__XSCALE_ */ 1034
Indexes created Sat Oct 25 10:09:55 GMT 2025