cpuswitch.S revision 1.36
1 /* $NetBSD: cpuswitch.S,v 1.36 2003/06/23 11:01:07 martin Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_arm32_pmap.h" 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 85 #include "assym.h" 86 #include <machine/param.h> 87 #include <machine/cpu.h> 88 #include <machine/frame.h> 89 #include <machine/asm.h> 90 91 /* LINTSTUB: include <sys/param.h> */ 92 93 #undef IRQdisable 94 #undef IRQenable 95 96 /* 97 * New experimental definitions of IRQdisable and IRQenable 98 * These keep FIQ's enabled since FIQ's are special. 99 */ 100 101 #define IRQdisable \ 102 mrs r14, cpsr ; \ 103 orr r14, r14, #(I32_bit) ; \ 104 msr cpsr_c, r14 ; \ 105 106 #define IRQenable \ 107 mrs r14, cpsr ; \ 108 bic r14, r14, #(I32_bit) ; \ 109 msr cpsr_c, r14 ; \ 110 111 /* 112 * These are used for switching the translation table/DACR. 113 * Since the vector page can be invalid for a short time, we must 114 * disable both regular IRQs *and* FIQs. 115 * 116 * XXX: This is not necessary if the vector table is relocated. 117 */ 118 #define IRQdisableALL \ 119 mrs r14, cpsr ; \ 120 orr r14, r14, #(I32_bit | F32_bit) ; \ 121 msr cpsr_c, r14 122 123 #define IRQenableALL \ 124 mrs r14, cpsr ; \ 125 bic r14, r14, #(I32_bit | F32_bit) ; \ 126 msr cpsr_c, r14 127 128 .text 129 130 .Lwhichqs: 131 .word _C_LABEL(sched_whichqs) 132 133 .Lqs: 134 .word _C_LABEL(sched_qs) 135 136 /* 137 * cpuswitch() 138 * 139 * preforms a process context switch. 140 * This function has several entry points 141 */ 142 143 #ifdef MULTIPROCESSOR 144 .Lcpu_info_store: 145 .word _C_LABEL(cpu_info_store) 146 .Lcurlwp: 147 /* FIXME: This is bogus in the general case. */ 148 .word _C_LABEL(cpu_info_store) + CI_CURLWP 149 150 .Lcurpcb: 151 .word _C_LABEL(cpu_info_store) + CI_CURPCB 152 #else 153 .Lcurlwp: 154 .word _C_LABEL(curlwp) 155 156 .Lcurpcb: 157 .word _C_LABEL(curpcb) 158 #endif 159 160 .Lwant_resched: 161 .word _C_LABEL(want_resched) 162 163 .Lcpufuncs: 164 .word _C_LABEL(cpufuncs) 165 166 #ifndef MULTIPROCESSOR 167 .data 168 .global _C_LABEL(curpcb) 169 _C_LABEL(curpcb): 170 .word 0x00000000 171 .text 172 #endif 173 174 .Lblock_userspace_access: 175 .word _C_LABEL(block_userspace_access) 176 177 .Lcpu_do_powersave: 178 .word _C_LABEL(cpu_do_powersave) 179 180 .Lpmap_kernel_cstate: 181 .word (kernel_pmap_store + PMAP_CSTATE) 182 183 .Llast_cache_state_ptr: 184 .word _C_LABEL(pmap_cache_state) 185 186 /* 187 * Idle loop, exercised while waiting for a process to wake up. 188 * 189 * NOTE: When we jump back to .Lswitch_search, we must have a 190 * pointer to whichqs in r7, which is what it is when we arrive 191 * here. 192 */ 193 /* LINTSTUB: Ignore */ 194 ASENTRY_NP(idle) 195 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 196 bl _C_LABEL(sched_unlock_idle) 197 #endif 198 ldr r3, .Lcpu_do_powersave 199 200 /* Enable interrupts */ 201 IRQenable 202 203 /* If we don't want to sleep, use a simpler loop. */ 204 ldr r3, [r3] /* r3 = cpu_do_powersave */ 205 teq r3, #0 206 bne 2f 207 208 /* Non-powersave idle. */ 209 1: /* should maybe do uvm pageidlezero stuff here */ 210 ldr r3, [r7] /* r3 = whichqs */ 211 teq r3, #0x00000000 212 bne .Lswitch_search 213 b 1b 214 215 2: /* Powersave idle. */ 216 ldr r4, .Lcpufuncs 217 3: ldr r3, [r7] /* r3 = whichqs */ 218 teq r3, #0x00000000 219 bne .Lswitch_search 220 221 /* if saving power, don't want to pageidlezero */ 222 mov r0, #0 223 adr lr, 3b 224 ldr pc, [r4, #(CF_SLEEP)] 225 /* loops back around */ 226 227 228 /* 229 * Find a new lwp to run, save the current context and 230 * load the new context 231 * 232 * Arguments: 233 * r0 'struct lwp *' of the current LWP 234 */ 235 236 ENTRY(cpu_switch) 237 /* 238 * Local register usage. Some of these registers are out of date. 239 * r1 = oldlwp 240 * r2 = spl level 241 * r3 = whichqs 242 * r4 = queue 243 * r5 = &qs[queue] 244 * r6 = newlwp 245 * r7 = scratch 246 */ 247 stmfd sp!, {r4-r7, lr} 248 249 /* 250 * Indicate that there is no longer a valid process (curlwp = 0). 251 * Zero the current PCB pointer while we're at it. 252 */ 253 ldr r7, .Lcurlwp 254 ldr r6, .Lcurpcb 255 mov r2, #0x00000000 256 str r2, [r7] /* curproc = NULL */ 257 str r2, [r6] /* curpcb = NULL */ 258 259 /* stash the old proc while we call functions */ 260 mov r5, r0 261 262 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 263 /* release the sched_lock before handling interrupts */ 264 bl _C_LABEL(sched_unlock_idle) 265 #endif 266 267 /* Lower the spl level to spl0 and get the current spl level. */ 268 #ifdef __NEWINTR 269 mov r0, #(IPL_NONE) 270 bl _C_LABEL(_spllower) 271 #else /* ! __NEWINTR */ 272 mov r0, #(_SPL_0) 273 bl _C_LABEL(splx) 274 #endif /* __NEWINTR */ 275 276 /* Push the old spl level onto the stack */ 277 str r0, [sp, #-0x0004]! 278 279 /* First phase : find a new lwp */ 280 281 ldr r7, .Lwhichqs 282 283 /* rem: r5 = old lwp */ 284 /* rem: r7 = &whichqs */ 285 286 .Lswitch_search: 287 IRQdisable 288 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 289 bl _C_LABEL(sched_lock_idle) 290 #endif 291 292 /* Do we have any active queues */ 293 ldr r3, [r7] 294 295 /* If not we must idle until we do. */ 296 teq r3, #0x00000000 297 beq _ASM_LABEL(idle) 298 299 /* put old proc back in r1 */ 300 mov r1, r5 301 302 /* rem: r1 = old lwp */ 303 /* rem: r3 = whichqs */ 304 /* rem: interrupts are disabled */ 305 306 /* 307 * We have found an active queue. Currently we do not know which queue 308 * is active just that one of them is. 309 */ 310 /* this is the ffs algorithm devised by d.seal and posted to 311 * comp.sys.arm on 16 Feb 1994. 312 */ 313 rsb r5, r3, #0 314 ands r0, r3, r5 315 316 adr r5, .Lcpu_switch_ffs_table 317 318 /* X = R0 */ 319 orr r4, r0, r0, lsl #4 /* r4 = X * 0x11 */ 320 orr r4, r4, r4, lsl #6 /* r4 = X * 0x451 */ 321 rsb r4, r4, r4, lsl #16 /* r4 = X * 0x0450fbaf */ 322 323 /* used further down, saves SA stall */ 324 ldr r6, .Lqs 325 326 /* now lookup in table indexed on top 6 bits of a4 */ 327 ldrb r4, [ r5, r4, lsr #26 ] 328 329 /* rem: r0 = bit mask of chosen queue (1 << r4) */ 330 /* rem: r1 = old lwp */ 331 /* rem: r3 = whichqs */ 332 /* rem: r4 = queue number */ 333 /* rem: interrupts are disabled */ 334 335 /* Get the address of the queue (&qs[queue]) */ 336 add r5, r6, r4, lsl #3 337 338 /* 339 * Get the lwp from the queue and place the next process in 340 * the queue at the head. This basically unlinks the lwp at 341 * the head of the queue. 342 */ 343 ldr r6, [r5, #(L_FORW)] 344 345 /* rem: r6 = new lwp */ 346 ldr r7, [r6, #(L_FORW)] 347 str r7, [r5, #(L_FORW)] 348 349 /* 350 * Test to see if the queue is now empty. If the head of the queue 351 * points to the queue itself then there are no more lwps in 352 * the queue. We can therefore clear the queue not empty flag held 353 * in r3. 354 */ 355 356 teq r5, r7 357 biceq r3, r3, r0 358 359 /* rem: r0 = bit mask of chosen queue (1 << r4) - NOT NEEDED AN MORE */ 360 361 /* Fix the back pointer for the lwp now at the head of the queue. */ 362 ldr r0, [r6, #(L_BACK)] 363 str r0, [r7, #(L_BACK)] 364 365 /* Update the RAM copy of the queue not empty flags word. */ 366 ldr r7, .Lwhichqs 367 str r3, [r7] 368 369 /* rem: r1 = old lwp */ 370 /* rem: r3 = whichqs - NOT NEEDED ANY MORE */ 371 /* rem: r4 = queue number - NOT NEEDED ANY MORE */ 372 /* rem: r6 = new lwp */ 373 /* rem: interrupts are disabled */ 374 375 /* Clear the want_resched flag */ 376 ldr r7, .Lwant_resched 377 mov r0, #0x00000000 378 str r0, [r7] 379 380 /* 381 * Clear the back pointer of the lwp we have removed from 382 * the head of the queue. The new lwp is isolated now. 383 */ 384 str r0, [r6, #(L_BACK)] 385 386 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 387 /* 388 * unlock the sched_lock, but leave interrupts off, for now. 389 */ 390 mov r7, r1 391 bl _C_LABEL(sched_unlock_idle) 392 mov r1, r7 393 #endif 394 395 .Lswitch_resume: 396 #ifdef MULTIPROCESSOR 397 /* XXX use curcpu() */ 398 ldr r0, .Lcpu_info_store 399 str r0, [r6, #(L_CPU)] 400 #else 401 /* l->l_cpu initialized in fork1() for single-processor */ 402 #endif 403 404 /* Process is now on a processor. */ 405 mov r0, #LSONPROC /* l->l_stat = LSONPROC */ 406 str r0, [r6, #(L_STAT)] 407 408 /* We have a new curlwp now so make a note it */ 409 ldr r7, .Lcurlwp 410 str r6, [r7] 411 412 /* Hook in a new pcb */ 413 ldr r7, .Lcurpcb 414 ldr r0, [r6, #(L_ADDR)] 415 str r0, [r7] 416 417 /* At this point we can allow IRQ's again. */ 418 IRQenable 419 420 /* rem: r1 = old lwp */ 421 /* rem: r4 = return value */ 422 /* rem: r6 = new process */ 423 /* rem: interrupts are enabled */ 424 425 /* 426 * If the new process is the same as the process that called 427 * cpu_switch() then we do not need to save and restore any 428 * contexts. This means we can make a quick exit. 429 * The test is simple if curlwp on entry (now in r1) is the 430 * same as the proc removed from the queue we can jump to the exit. 431 */ 432 teq r1, r6 433 moveq r4, #0x00000000 /* default to "didn't switch" */ 434 beq .Lswitch_return 435 436 /* 437 * At this point, we are guaranteed to be switching to 438 * a new lwp. 439 */ 440 mov r4, #0x00000001 441 442 /* Remember the old lwp in r0 */ 443 mov r0, r1 444 445 /* 446 * If the old lwp on entry to cpu_switch was zero then the 447 * process that called it was exiting. This means that we do 448 * not need to save the current context. Instead we can jump 449 * straight to restoring the context for the new process. 450 */ 451 teq r0, #0x00000000 452 beq .Lswitch_exited 453 454 /* rem: r0 = old lwp */ 455 /* rem: r4 = return value */ 456 /* rem: r6 = new process */ 457 /* rem: interrupts are enabled */ 458 459 /* Stage two : Save old context */ 460 461 /* Get the user structure for the old lwp. */ 462 ldr r1, [r0, #(L_ADDR)] 463 464 /* Save all the registers in the old lwp's pcb */ 465 add r7, r1, #(PCB_R8) 466 stmia r7, {r8-r13} 467 468 /* 469 * NOTE: We can now use r8-r13 until it is time to restore 470 * them for the new process. 471 */ 472 473 /* Remember the old PCB. */ 474 mov r8, r1 475 476 /* r1 now free! */ 477 478 /* Get the user structure for the new process in r9 */ 479 ldr r9, [r6, #(L_ADDR)] 480 481 /* 482 * This can be optimised... We know we want to go from SVC32 483 * mode to UND32 mode 484 */ 485 mrs r3, cpsr 486 bic r2, r3, #(PSR_MODE) 487 orr r2, r2, #(PSR_UND32_MODE | I32_bit) 488 msr cpsr_c, r2 489 490 str sp, [r8, #(PCB_UND_SP)] 491 492 msr cpsr_c, r3 /* Restore the old mode */ 493 494 /* rem: r0 = old lwp */ 495 /* rem: r4 = return value */ 496 /* rem: r6 = new process */ 497 /* rem: r8 = old PCB */ 498 /* rem: r9 = new PCB */ 499 /* rem: interrupts are enabled */ 500 501 /* What else needs to be saved Only FPA stuff when that is supported */ 502 503 /* Third phase : restore saved context */ 504 505 /* rem: r0 = old lwp */ 506 /* rem: r4 = return value */ 507 /* rem: r6 = new lwp */ 508 /* rem: r8 = old PCB */ 509 /* rem: r9 = new PCB */ 510 /* rem: interrupts are enabled */ 511 512 /* 513 * Get the new L1 table pointer into r11. If we're switching to 514 * an LWP with the same address space as the outgoing one, we can 515 * skip the cache purge and the TTB load. 516 * 517 * To avoid data dep stalls that would happen anyway, we try 518 * and get some useful work done in the mean time. 519 */ 520 ldr r10, [r8, #(PCB_PAGEDIR)] /* r10 = old L1 */ 521 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 522 523 ldr r0, [r8, #(PCB_DACR)] /* r0 = old DACR */ 524 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 525 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = &new_pmap->pm_cstate */ 526 ldr r5, .Llast_cache_state_ptr /* Previous thread's cstate */ 527 528 teq r10, r11 /* Same L1? */ 529 ldr r5, [r5] 530 cmpeq r0, r1 /* Same DACR? */ 531 beq .Lcs_context_switched /* yes! */ 532 533 ldr r3, .Lblock_userspace_access 534 mov r12, #0 535 cmp r5, #0 /* No last vm? (switch_exit) */ 536 beq .Lcs_cache_purge_skipped /* No, we can skip cache flsh */ 537 538 mov r2, #DOMAIN_CLIENT 539 cmp r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 540 beq .Lcs_cache_purge_skipped /* Yup. Don't flush cache */ 541 542 cmp r5, r8 /* Same userland VM space? */ 543 ldrneb r12, [r5, #(CS_CACHE_ID)] /* Last VM space cache state */ 544 545 /* 546 * We're definately switching to a new userland VM space, 547 * and the previous userland VM space has yet to be flushed 548 * from the cache/tlb. 549 * 550 * r12 holds the previous VM space's cs_cache_id state 551 */ 552 tst r12, #0xff /* Test cs_cache_id */ 553 beq .Lcs_cache_purge_skipped /* VM space is not in cache */ 554 555 /* 556 * Definately need to flush the cache. 557 * Mark the old VM space as NOT being resident in the cache. 558 */ 559 mov r2, #0x00000000 560 strb r2, [r5, #(CS_CACHE_ID)] 561 strb r2, [r5, #(CS_CACHE_D)] 562 563 /* 564 * Don't allow user space access between the purge and the switch. 565 */ 566 mov r2, #0x00000001 567 str r2, [r3] 568 569 stmfd sp!, {r0-r3} 570 ldr r1, .Lcpufuncs 571 mov lr, pc 572 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 573 ldmfd sp!, {r0-r3} 574 575 .Lcs_cache_purge_skipped: 576 /* rem: r1 = new DACR */ 577 /* rem: r3 = &block_userspace_access */ 578 /* rem: r4 = return value */ 579 /* rem: r5 = &old_pmap->pm_cstate (or NULL) */ 580 /* rem: r6 = new lwp */ 581 /* rem: r8 = &new_pmap->pm_cstate */ 582 /* rem: r9 = new PCB */ 583 /* rem: r10 = old L1 */ 584 /* rem: r11 = new L1 */ 585 586 mov r2, #0x00000000 587 ldr r7, [r9, #(PCB_PL1VEC)] 588 589 /* 590 * At this point we need to kill IRQ's again. 591 * 592 * XXXSCW: Don't need to block FIQs if vectors have been relocated 593 */ 594 IRQdisableALL 595 596 /* 597 * Interrupts are disabled so we can allow user space accesses again 598 * as none will occur until interrupts are re-enabled after the 599 * switch. 600 */ 601 str r2, [r3] 602 603 /* 604 * Ensure the vector table is accessible by fixing up the L1 605 */ 606 cmp r7, #0 /* No need to fixup vector table? */ 607 ldrne r2, [r7] /* But if yes, fetch current value */ 608 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 609 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for new context */ 610 cmpne r2, r0 /* Stuffing the same value? */ 611 #ifndef PMAP_INCLUDE_PTE_SYNC 612 strne r0, [r7] /* Nope, update it */ 613 #else 614 beq .Lcs_same_vector 615 str r0, [r7] /* Otherwise, update it */ 616 617 /* 618 * Need to sync the cache to make sure that last store is 619 * visible to the MMU. 620 */ 621 ldr r2, .Lcpufuncs 622 mov r0, r7 623 mov r1, #4 624 mov lr, pc 625 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 626 627 .Lcs_same_vector: 628 #endif /* PMAP_INCLUDE_PTE_SYNC */ 629 630 cmp r10, r11 /* Switching to the same L1? */ 631 ldr r10, .Lcpufuncs 632 beq .Lcs_same_l1 /* Yup. */ 633 634 /* 635 * Do a full context switch, including full TLB flush. 636 */ 637 mov r0, r11 638 mov lr, pc 639 ldr pc, [r10, #CF_CONTEXT_SWITCH] 640 641 /* 642 * Mark the old VM space as NOT being resident in the TLB 643 */ 644 mov r2, #0x00000000 645 cmp r5, #0 646 strneh r2, [r5, #(CS_TLB_ID)] 647 b .Lcs_context_switched 648 649 /* 650 * We're switching to a different process in the same L1. 651 * In this situation, we only need to flush the TLB for the 652 * vector_page mapping, and even then only if r7 is non-NULL. 653 */ 654 .Lcs_same_l1: 655 cmp r7, #0 656 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 657 movne lr, pc 658 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 659 660 .Lcs_context_switched: 661 /* rem: r8 = &new_pmap->pm_cstate */ 662 663 /* XXXSCW: Safe to re-enable FIQs here */ 664 665 /* 666 * The new VM space is live in the cache and TLB. 667 * Update its cache/tlb state, and if it's not the kernel 668 * pmap, update the 'last cache state' pointer. 669 */ 670 mov r2, #-1 671 ldr r5, .Lpmap_kernel_cstate 672 ldr r0, .Llast_cache_state_ptr 673 str r2, [r8, #(CS_ALL)] 674 cmp r5, r8 675 strne r8, [r0] 676 677 /* rem: r4 = return value */ 678 /* rem: r6 = new lwp */ 679 /* rem: r9 = new PCB */ 680 681 /* 682 * This can be optimised... We know we want to go from SVC32 683 * mode to UND32 mode 684 */ 685 mrs r3, cpsr 686 bic r2, r3, #(PSR_MODE) 687 orr r2, r2, #(PSR_UND32_MODE) 688 msr cpsr_c, r2 689 690 ldr sp, [r9, #(PCB_UND_SP)] 691 692 msr cpsr_c, r3 /* Restore the old mode */ 693 694 /* Restore all the save registers */ 695 add r7, r9, #PCB_R8 696 ldmia r7, {r8-r13} 697 698 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 699 700 ldr r5, [r6, #(L_PROC)] /* fetch the proc for below */ 701 702 /* rem: r4 = return value */ 703 /* rem: r5 = new lwp's proc */ 704 /* rem: r6 = new lwp */ 705 /* rem: r7 = new pcb */ 706 707 #ifdef ARMFPE 708 add r0, r7, #(USER_SIZE) & 0x00ff 709 add r0, r0, #(USER_SIZE) & 0xff00 710 bl _C_LABEL(arm_fpe_core_changecontext) 711 #endif 712 713 /* We can enable interrupts again */ 714 IRQenableALL 715 716 /* rem: r4 = return value */ 717 /* rem: r5 = new lwp's proc */ 718 /* rem: r6 = new lwp */ 719 /* rem: r7 = new PCB */ 720 721 /* 722 * Check for restartable atomic sequences (RAS). 723 */ 724 725 ldr r2, [r5, #(P_NRAS)] 726 ldr r4, [r7, #(PCB_TF)] /* r4 = trapframe (used below) */ 727 teq r2, #0 /* p->p_nras == 0? */ 728 bne .Lswitch_do_ras /* no, check for one */ 729 730 .Lswitch_return: 731 732 /* Get the spl level from the stack and update the current spl level */ 733 ldr r0, [sp], #0x0004 734 bl _C_LABEL(splx) 735 736 /* cpu_switch returns 1 == switched, 0 == didn't switch */ 737 mov r0, r4 738 739 /* 740 * Pull the registers that got pushed when either savectx() or 741 * cpu_switch() was called and return. 742 */ 743 ldmfd sp!, {r4-r7, pc} 744 745 .Lswitch_do_ras: 746 ldr r1, [r4, #(TF_PC)] /* second ras_lookup() arg */ 747 mov r0, r5 /* first ras_lookup() arg */ 748 bl _C_LABEL(ras_lookup) 749 cmn r0, #1 /* -1 means "not in a RAS" */ 750 strne r0, [r4, #(TF_PC)] 751 b .Lswitch_return 752 753 .Lswitch_exited: 754 /* 755 * We skip the cache purge because switch_exit() already did it. 756 * Load up registers the way .Lcs_cache_purge_skipped expects. 757 * Userpsace access already blocked by switch_exit(). 758 */ 759 ldr r9, [r6, #(L_ADDR)] /* r9 = new PCB */ 760 ldr r3, .Lblock_userspace_access 761 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 762 mov r5, #0 /* No previous cache state */ 763 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 764 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = new cache state */ 765 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 766 b .Lcs_cache_purge_skipped 767 768 /* 769 * cpu_switchto(struct lwp *current, struct lwp *next) 770 * Switch to the specified next LWP 771 * Arguments: 772 * 773 * r0 'struct lwp *' of the current LWP 774 * r1 'struct lwp *' of the LWP to switch to 775 */ 776 ENTRY(cpu_switchto) 777 stmfd sp!, {r4-r7, lr} 778 779 /* Lower the spl level to spl0 and get the current spl level. */ 780 mov r6, r0 /* save old lwp */ 781 mov r5, r1 /* save new lwp */ 782 783 #if defined(LOCKDEBUG) 784 /* release the sched_lock before handling interrupts */ 785 bl _C_LABEL(sched_unlock_idle) 786 #endif 787 788 #ifdef __NEWINTR 789 mov r0, #(IPL_NONE) 790 bl _C_LABEL(_spllower) 791 #else /* ! __NEWINTR */ 792 #ifdef spl0 793 mov r0, #(_SPL_0) 794 bl _C_LABEL(splx) 795 #else 796 bl _C_LABEL(spl0) 797 #endif /* spl0 */ 798 #endif /* __NEWINTR */ 799 800 /* Push the old spl level onto the stack */ 801 str r0, [sp, #-0x0004]! 802 803 IRQdisable 804 #if defined(LOCKDEBUG) 805 bl _C_LABEL(sched_lock_idle) 806 #endif 807 808 mov r0, r6 /* restore old lwp */ 809 mov r1, r5 /* restore new lwp */ 810 811 /* rem: r0 = old lwp */ 812 /* rem: r1 = new lwp */ 813 /* rem: interrupts are disabled */ 814 815 /* 816 * Okay, set up registers the way cpu_switch() wants them, 817 * and jump into the middle of it (where we bring up the 818 * new process). 819 */ 820 mov r6, r1 /* r6 = new lwp */ 821 #if defined(LOCKDEBUG) 822 mov r5, r0 /* preserve old lwp */ 823 bl _C_LABEL(sched_unlock_idle) 824 mov r1, r5 /* r1 = old lwp */ 825 #else 826 mov r1, r0 /* r1 = old lwp */ 827 #endif 828 b .Lswitch_resume 829 830 /* 831 * void switch_exit(struct lwp *l, struct lwp *l0, void (*exit)(struct lwp *)); 832 * Switch to lwp0's saved context and deallocate the address space and kernel 833 * stack for l. Then jump into cpu_switch(), as if we were in lwp0 all along. 834 */ 835 836 /* LINTSTUB: Func: void switch_exit(struct lwp *l, struct lwp *l0, void (*func)(struct lwp *)) */ 837 ENTRY(switch_exit) 838 /* 839 * The process is going away, so we can use callee-saved 840 * registers here without having to save them. 841 */ 842 843 mov r4, r0 844 ldr r0, .Lcurlwp 845 846 mov r5, r1 847 ldr r1, .Lblock_userspace_access 848 849 mov r6, r2 850 851 /* 852 * r4 = lwp 853 * r5 = lwp0 854 * r6 = exit func 855 */ 856 857 mov r2, #0x00000000 /* curlwp = NULL */ 858 str r2, [r0] 859 860 /* 861 * We're about to clear both the cache and the TLB. 862 * Make sure to zap the 'last cache state' pointer since the 863 * pmap might be about to go away. Also ensure the outgoing 864 * VM space's cache state is marked as NOT resident in the 865 * cache, and that lwp0's cache state IS resident. 866 */ 867 ldr r7, [r4, #(L_ADDR)] /* r7 = old lwp's PCB */ 868 ldr r0, .Llast_cache_state_ptr /* Last userland cache state */ 869 ldr r9, [r7, #(PCB_CSTATE)] /* Fetch cache state pointer */ 870 ldr r3, [r5, #(L_ADDR)] /* r3 = lwp0's PCB */ 871 str r2, [r0] /* No previous cache state */ 872 str r2, [r9, #(CS_ALL)] /* Zap old lwp's cache state */ 873 ldr r3, [r3, #(PCB_CSTATE)] /* lwp0's cache state */ 874 mov r2, #-1 875 str r2, [r3, #(CS_ALL)] /* lwp0 is in da cache! */ 876 877 /* 878 * Don't allow user space access between the purge and the switch. 879 */ 880 mov r2, #0x00000001 881 str r2, [r1] 882 883 /* Switch to lwp0 context */ 884 885 ldr r9, .Lcpufuncs 886 mov lr, pc 887 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 888 889 ldr r0, [r7, #(PCB_PL1VEC)] 890 ldr r1, [r7, #(PCB_DACR)] 891 892 /* 893 * r0 = Pointer to L1 slot for vector_page (or NULL) 894 * r1 = lwp0's DACR 895 * r4 = lwp we're switching from 896 * r5 = lwp0 897 * r6 = exit func 898 * r7 = lwp0's PCB 899 * r9 = cpufuncs 900 */ 901 902 IRQdisableALL 903 904 /* 905 * Ensure the vector table is accessible by fixing up lwp0's L1 906 */ 907 cmp r0, #0 /* No need to fixup vector table? */ 908 ldrne r3, [r0] /* But if yes, fetch current value */ 909 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 910 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 911 cmpne r3, r2 /* Stuffing the same value? */ 912 strne r2, [r0] /* Store if not. */ 913 914 #ifdef PMAP_INCLUDE_PTE_SYNC 915 /* 916 * Need to sync the cache to make sure that last store is 917 * visible to the MMU. 918 */ 919 movne r1, #4 920 movne lr, pc 921 ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 922 #endif /* PMAP_INCLUDE_PTE_SYNC */ 923 924 /* 925 * Note: We don't do the same optimisation as cpu_switch() with 926 * respect to avoiding flushing the TLB if we're switching to 927 * the same L1 since this process' VM space may be about to go 928 * away, so we don't want *any* turds left in the TLB. 929 */ 930 931 /* Switch the memory to the new process */ 932 ldr r0, [r7, #(PCB_PAGEDIR)] 933 mov lr, pc 934 ldr pc, [r9, #CF_CONTEXT_SWITCH] 935 936 ldr r0, .Lcurpcb 937 938 /* Restore all the save registers */ 939 add r1, r7, #PCB_R8 940 ldmia r1, {r8-r13} 941 942 str r7, [r0] /* curpcb = lwp0's PCB */ 943 944 IRQenableALL 945 946 /* 947 * Schedule the vmspace and stack to be freed. 948 */ 949 mov r0, r4 /* {lwp_}exit2(l) */ 950 mov lr, pc 951 mov pc, r6 952 953 ldr r7, .Lwhichqs /* r7 = &whichqs */ 954 mov r5, #0x00000000 /* r5 = old lwp = NULL */ 955 b .Lswitch_search 956 957 /* LINTSTUB: Func: void savectx(struct pcb *pcb) */ 958 ENTRY(savectx) 959 /* 960 * r0 = pcb 961 */ 962 963 /* Push registers.*/ 964 stmfd sp!, {r4-r7, lr} 965 966 /* Store all the registers in the process's pcb */ 967 add r2, r0, #(PCB_R8) 968 stmia r2, {r8-r13} 969 970 /* Pull the regs of the stack */ 971 ldmfd sp!, {r4-r7, pc} 972 973 ENTRY(proc_trampoline) 974 #ifdef MULTIPROCESSOR 975 bl _C_LABEL(proc_trampoline_mp) 976 #endif 977 mov r0, r5 978 mov r1, sp 979 mov lr, pc 980 mov pc, r4 981 982 /* Kill irq's */ 983 mrs r0, cpsr 984 orr r0, r0, #(I32_bit) 985 msr cpsr_c, r0 986 987 PULLFRAME 988 989 movs pc, lr /* Exit */ 990 991 .type .Lcpu_switch_ffs_table, _ASM_TYPE_OBJECT; 992 .Lcpu_switch_ffs_table: 993 /* same as ffs table but all nums are -1 from that */ 994 /* 0 1 2 3 4 5 6 7 */ 995 .byte 0, 0, 1, 12, 2, 6, 0, 13 /* 0- 7 */ 996 .byte 3, 0, 7, 0, 0, 0, 0, 14 /* 8-15 */ 997 .byte 10, 4, 0, 0, 8, 0, 0, 25 /* 16-23 */ 998 .byte 0, 0, 0, 0, 0, 21, 27, 15 /* 24-31 */ 999 .byte 31, 11, 5, 0, 0, 0, 0, 0 /* 32-39 */ 1000 .byte 9, 0, 0, 24, 0, 0, 20, 26 /* 40-47 */ 1001 .byte 30, 0, 0, 0, 0, 23, 0, 19 /* 48-55 */ 1002 .byte 29, 0, 22, 18, 28, 17, 16, 0 /* 56-63 */ 1003
Indexes created Tue Oct 21 16:10:11 GMT 2025