cpuswitch.S revision 1.35
1 /* $NetBSD: cpuswitch.S,v 1.35 2003/06/23 09:05:22 chris Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_arm32_pmap.h" 82 #include "opt_multiprocessor.h" 83 84 #include "assym.h" 85 #include <machine/param.h> 86 #include <machine/cpu.h> 87 #include <machine/frame.h> 88 #include <machine/asm.h> 89 90 /* LINTSTUB: include <sys/param.h> */ 91 92 #undef IRQdisable 93 #undef IRQenable 94 95 /* 96 * New experimental definitions of IRQdisable and IRQenable 97 * These keep FIQ's enabled since FIQ's are special. 98 */ 99 100 #define IRQdisable \ 101 mrs r14, cpsr ; \ 102 orr r14, r14, #(I32_bit) ; \ 103 msr cpsr_c, r14 ; \ 104 105 #define IRQenable \ 106 mrs r14, cpsr ; \ 107 bic r14, r14, #(I32_bit) ; \ 108 msr cpsr_c, r14 ; \ 109 110 /* 111 * These are used for switching the translation table/DACR. 112 * Since the vector page can be invalid for a short time, we must 113 * disable both regular IRQs *and* FIQs. 114 * 115 * XXX: This is not necessary if the vector table is relocated. 116 */ 117 #define IRQdisableALL \ 118 mrs r14, cpsr ; \ 119 orr r14, r14, #(I32_bit | F32_bit) ; \ 120 msr cpsr_c, r14 121 122 #define IRQenableALL \ 123 mrs r14, cpsr ; \ 124 bic r14, r14, #(I32_bit | F32_bit) ; \ 125 msr cpsr_c, r14 126 127 .text 128 129 .Lwhichqs: 130 .word _C_LABEL(sched_whichqs) 131 132 .Lqs: 133 .word _C_LABEL(sched_qs) 134 135 /* 136 * cpuswitch() 137 * 138 * preforms a process context switch. 139 * This function has several entry points 140 */ 141 142 #ifdef MULTIPROCESSOR 143 .Lcpu_info_store: 144 .word _C_LABEL(cpu_info_store) 145 .Lcurlwp: 146 /* FIXME: This is bogus in the general case. */ 147 .word _C_LABEL(cpu_info_store) + CI_CURLWP 148 149 .Lcurpcb: 150 .word _C_LABEL(cpu_info_store) + CI_CURPCB 151 #else 152 .Lcurlwp: 153 .word _C_LABEL(curlwp) 154 155 .Lcurpcb: 156 .word _C_LABEL(curpcb) 157 #endif 158 159 .Lwant_resched: 160 .word _C_LABEL(want_resched) 161 162 .Lcpufuncs: 163 .word _C_LABEL(cpufuncs) 164 165 #ifndef MULTIPROCESSOR 166 .data 167 .global _C_LABEL(curpcb) 168 _C_LABEL(curpcb): 169 .word 0x00000000 170 .text 171 #endif 172 173 .Lblock_userspace_access: 174 .word _C_LABEL(block_userspace_access) 175 176 .Lcpu_do_powersave: 177 .word _C_LABEL(cpu_do_powersave) 178 179 .Lpmap_kernel_cstate: 180 .word (kernel_pmap_store + PMAP_CSTATE) 181 182 .Llast_cache_state_ptr: 183 .word _C_LABEL(pmap_cache_state) 184 185 /* 186 * Idle loop, exercised while waiting for a process to wake up. 187 * 188 * NOTE: When we jump back to .Lswitch_search, we must have a 189 * pointer to whichqs in r7, which is what it is when we arrive 190 * here. 191 */ 192 /* LINTSTUB: Ignore */ 193 ASENTRY_NP(idle) 194 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 195 bl _C_LABEL(sched_unlock_idle) 196 #endif 197 ldr r3, .Lcpu_do_powersave 198 199 /* Enable interrupts */ 200 IRQenable 201 202 /* If we don't want to sleep, use a simpler loop. */ 203 ldr r3, [r3] /* r3 = cpu_do_powersave */ 204 teq r3, #0 205 bne 2f 206 207 /* Non-powersave idle. */ 208 1: /* should maybe do uvm pageidlezero stuff here */ 209 ldr r3, [r7] /* r3 = whichqs */ 210 teq r3, #0x00000000 211 bne .Lswitch_search 212 b 1b 213 214 2: /* Powersave idle. */ 215 ldr r4, .Lcpufuncs 216 3: ldr r3, [r7] /* r3 = whichqs */ 217 teq r3, #0x00000000 218 bne .Lswitch_search 219 220 /* if saving power, don't want to pageidlezero */ 221 mov r0, #0 222 adr lr, 3b 223 ldr pc, [r4, #(CF_SLEEP)] 224 /* loops back around */ 225 226 227 /* 228 * Find a new lwp to run, save the current context and 229 * load the new context 230 * 231 * Arguments: 232 * r0 'struct lwp *' of the current LWP 233 */ 234 235 ENTRY(cpu_switch) 236 /* 237 * Local register usage. Some of these registers are out of date. 238 * r1 = oldlwp 239 * r2 = spl level 240 * r3 = whichqs 241 * r4 = queue 242 * r5 = &qs[queue] 243 * r6 = newlwp 244 * r7 = scratch 245 */ 246 stmfd sp!, {r4-r7, lr} 247 248 /* 249 * Indicate that there is no longer a valid process (curlwp = 0). 250 * Zero the current PCB pointer while we're at it. 251 */ 252 ldr r7, .Lcurlwp 253 ldr r6, .Lcurpcb 254 mov r2, #0x00000000 255 str r2, [r7] /* curproc = NULL */ 256 str r2, [r6] /* curpcb = NULL */ 257 258 /* stash the old proc while we call functions */ 259 mov r5, r0 260 261 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 262 /* release the sched_lock before handling interrupts */ 263 bl _C_LABEL(sched_unlock_idle) 264 #endif 265 266 /* Lower the spl level to spl0 and get the current spl level. */ 267 #ifdef __NEWINTR 268 mov r0, #(IPL_NONE) 269 bl _C_LABEL(_spllower) 270 #else /* ! __NEWINTR */ 271 mov r0, #(_SPL_0) 272 bl _C_LABEL(splx) 273 #endif /* __NEWINTR */ 274 275 /* Push the old spl level onto the stack */ 276 str r0, [sp, #-0x0004]! 277 278 /* First phase : find a new lwp */ 279 280 ldr r7, .Lwhichqs 281 282 /* rem: r5 = old lwp */ 283 /* rem: r7 = &whichqs */ 284 285 .Lswitch_search: 286 IRQdisable 287 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 288 bl _C_LABEL(sched_lock_idle) 289 #endif 290 291 /* Do we have any active queues */ 292 ldr r3, [r7] 293 294 /* If not we must idle until we do. */ 295 teq r3, #0x00000000 296 beq _ASM_LABEL(idle) 297 298 /* put old proc back in r1 */ 299 mov r1, r5 300 301 /* rem: r1 = old lwp */ 302 /* rem: r3 = whichqs */ 303 /* rem: interrupts are disabled */ 304 305 /* 306 * We have found an active queue. Currently we do not know which queue 307 * is active just that one of them is. 308 */ 309 /* this is the ffs algorithm devised by d.seal and posted to 310 * comp.sys.arm on 16 Feb 1994. 311 */ 312 rsb r5, r3, #0 313 ands r0, r3, r5 314 315 adr r5, .Lcpu_switch_ffs_table 316 317 /* X = R0 */ 318 orr r4, r0, r0, lsl #4 /* r4 = X * 0x11 */ 319 orr r4, r4, r4, lsl #6 /* r4 = X * 0x451 */ 320 rsb r4, r4, r4, lsl #16 /* r4 = X * 0x0450fbaf */ 321 322 /* used further down, saves SA stall */ 323 ldr r6, .Lqs 324 325 /* now lookup in table indexed on top 6 bits of a4 */ 326 ldrb r4, [ r5, r4, lsr #26 ] 327 328 /* rem: r0 = bit mask of chosen queue (1 << r4) */ 329 /* rem: r1 = old lwp */ 330 /* rem: r3 = whichqs */ 331 /* rem: r4 = queue number */ 332 /* rem: interrupts are disabled */ 333 334 /* Get the address of the queue (&qs[queue]) */ 335 add r5, r6, r4, lsl #3 336 337 /* 338 * Get the lwp from the queue and place the next process in 339 * the queue at the head. This basically unlinks the lwp at 340 * the head of the queue. 341 */ 342 ldr r6, [r5, #(L_FORW)] 343 344 /* rem: r6 = new lwp */ 345 ldr r7, [r6, #(L_FORW)] 346 str r7, [r5, #(L_FORW)] 347 348 /* 349 * Test to see if the queue is now empty. If the head of the queue 350 * points to the queue itself then there are no more lwps in 351 * the queue. We can therefore clear the queue not empty flag held 352 * in r3. 353 */ 354 355 teq r5, r7 356 biceq r3, r3, r0 357 358 /* rem: r0 = bit mask of chosen queue (1 << r4) - NOT NEEDED AN MORE */ 359 360 /* Fix the back pointer for the lwp now at the head of the queue. */ 361 ldr r0, [r6, #(L_BACK)] 362 str r0, [r7, #(L_BACK)] 363 364 /* Update the RAM copy of the queue not empty flags word. */ 365 ldr r7, .Lwhichqs 366 str r3, [r7] 367 368 /* rem: r1 = old lwp */ 369 /* rem: r3 = whichqs - NOT NEEDED ANY MORE */ 370 /* rem: r4 = queue number - NOT NEEDED ANY MORE */ 371 /* rem: r6 = new lwp */ 372 /* rem: interrupts are disabled */ 373 374 /* Clear the want_resched flag */ 375 ldr r7, .Lwant_resched 376 mov r0, #0x00000000 377 str r0, [r7] 378 379 /* 380 * Clear the back pointer of the lwp we have removed from 381 * the head of the queue. The new lwp is isolated now. 382 */ 383 str r0, [r6, #(L_BACK)] 384 385 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 386 /* 387 * unlock the sched_lock, but leave interrupts off, for now. 388 */ 389 mov r7, r1 390 bl _C_LABEL(sched_unlock_idle) 391 mov r1, r7 392 #endif 393 394 .Lswitch_resume: 395 #ifdef MULTIPROCESSOR 396 /* XXX use curcpu() */ 397 ldr r0, .Lcpu_info_store 398 str r0, [r6, #(L_CPU)] 399 #else 400 /* l->l_cpu initialized in fork1() for single-processor */ 401 #endif 402 403 /* Process is now on a processor. */ 404 mov r0, #LSONPROC /* l->l_stat = LSONPROC */ 405 str r0, [r6, #(L_STAT)] 406 407 /* We have a new curlwp now so make a note it */ 408 ldr r7, .Lcurlwp 409 str r6, [r7] 410 411 /* Hook in a new pcb */ 412 ldr r7, .Lcurpcb 413 ldr r0, [r6, #(L_ADDR)] 414 str r0, [r7] 415 416 /* At this point we can allow IRQ's again. */ 417 IRQenable 418 419 /* rem: r1 = old lwp */ 420 /* rem: r4 = return value */ 421 /* rem: r6 = new process */ 422 /* rem: interrupts are enabled */ 423 424 /* 425 * If the new process is the same as the process that called 426 * cpu_switch() then we do not need to save and restore any 427 * contexts. This means we can make a quick exit. 428 * The test is simple if curlwp on entry (now in r1) is the 429 * same as the proc removed from the queue we can jump to the exit. 430 */ 431 teq r1, r6 432 moveq r4, #0x00000000 /* default to "didn't switch" */ 433 beq .Lswitch_return 434 435 /* 436 * At this point, we are guaranteed to be switching to 437 * a new lwp. 438 */ 439 mov r4, #0x00000001 440 441 /* Remember the old lwp in r0 */ 442 mov r0, r1 443 444 /* 445 * If the old lwp on entry to cpu_switch was zero then the 446 * process that called it was exiting. This means that we do 447 * not need to save the current context. Instead we can jump 448 * straight to restoring the context for the new process. 449 */ 450 teq r0, #0x00000000 451 beq .Lswitch_exited 452 453 /* rem: r0 = old lwp */ 454 /* rem: r4 = return value */ 455 /* rem: r6 = new process */ 456 /* rem: interrupts are enabled */ 457 458 /* Stage two : Save old context */ 459 460 /* Get the user structure for the old lwp. */ 461 ldr r1, [r0, #(L_ADDR)] 462 463 /* Save all the registers in the old lwp's pcb */ 464 add r7, r1, #(PCB_R8) 465 stmia r7, {r8-r13} 466 467 /* 468 * NOTE: We can now use r8-r13 until it is time to restore 469 * them for the new process. 470 */ 471 472 /* Remember the old PCB. */ 473 mov r8, r1 474 475 /* r1 now free! */ 476 477 /* Get the user structure for the new process in r9 */ 478 ldr r9, [r6, #(L_ADDR)] 479 480 /* 481 * This can be optimised... We know we want to go from SVC32 482 * mode to UND32 mode 483 */ 484 mrs r3, cpsr 485 bic r2, r3, #(PSR_MODE) 486 orr r2, r2, #(PSR_UND32_MODE | I32_bit) 487 msr cpsr_c, r2 488 489 str sp, [r8, #(PCB_UND_SP)] 490 491 msr cpsr_c, r3 /* Restore the old mode */ 492 493 /* rem: r0 = old lwp */ 494 /* rem: r4 = return value */ 495 /* rem: r6 = new process */ 496 /* rem: r8 = old PCB */ 497 /* rem: r9 = new PCB */ 498 /* rem: interrupts are enabled */ 499 500 /* What else needs to be saved Only FPA stuff when that is supported */ 501 502 /* Third phase : restore saved context */ 503 504 /* rem: r0 = old lwp */ 505 /* rem: r4 = return value */ 506 /* rem: r6 = new lwp */ 507 /* rem: r8 = old PCB */ 508 /* rem: r9 = new PCB */ 509 /* rem: interrupts are enabled */ 510 511 /* 512 * Get the new L1 table pointer into r11. If we're switching to 513 * an LWP with the same address space as the outgoing one, we can 514 * skip the cache purge and the TTB load. 515 * 516 * To avoid data dep stalls that would happen anyway, we try 517 * and get some useful work done in the mean time. 518 */ 519 ldr r10, [r8, #(PCB_PAGEDIR)] /* r10 = old L1 */ 520 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 521 522 ldr r0, [r8, #(PCB_DACR)] /* r0 = old DACR */ 523 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 524 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = &new_pmap->pm_cstate */ 525 ldr r5, .Llast_cache_state_ptr /* Previous thread's cstate */ 526 527 teq r10, r11 /* Same L1? */ 528 ldr r5, [r5] 529 cmpeq r0, r1 /* Same DACR? */ 530 beq .Lcs_context_switched /* yes! */ 531 532 ldr r3, .Lblock_userspace_access 533 mov r12, #0 534 cmp r5, #0 /* No last vm? (switch_exit) */ 535 beq .Lcs_cache_purge_skipped /* No, we can skip cache flsh */ 536 537 mov r2, #DOMAIN_CLIENT 538 cmp r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 539 beq .Lcs_cache_purge_skipped /* Yup. Don't flush cache */ 540 541 cmp r5, r8 /* Same userland VM space? */ 542 ldrneb r12, [r5, #(CS_CACHE_ID)] /* Last VM space cache state */ 543 544 /* 545 * We're definately switching to a new userland VM space, 546 * and the previous userland VM space has yet to be flushed 547 * from the cache/tlb. 548 * 549 * r12 holds the previous VM space's cs_cache_id state 550 */ 551 tst r12, #0xff /* Test cs_cache_id */ 552 beq .Lcs_cache_purge_skipped /* VM space is not in cache */ 553 554 /* 555 * Definately need to flush the cache. 556 * Mark the old VM space as NOT being resident in the cache. 557 */ 558 mov r2, #0x00000000 559 strb r2, [r5, #(CS_CACHE_ID)] 560 strb r2, [r5, #(CS_CACHE_D)] 561 562 /* 563 * Don't allow user space access between the purge and the switch. 564 */ 565 mov r2, #0x00000001 566 str r2, [r3] 567 568 stmfd sp!, {r0-r3} 569 ldr r1, .Lcpufuncs 570 mov lr, pc 571 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 572 ldmfd sp!, {r0-r3} 573 574 .Lcs_cache_purge_skipped: 575 /* rem: r1 = new DACR */ 576 /* rem: r3 = &block_userspace_access */ 577 /* rem: r4 = return value */ 578 /* rem: r5 = &old_pmap->pm_cstate (or NULL) */ 579 /* rem: r6 = new lwp */ 580 /* rem: r8 = &new_pmap->pm_cstate */ 581 /* rem: r9 = new PCB */ 582 /* rem: r10 = old L1 */ 583 /* rem: r11 = new L1 */ 584 585 mov r2, #0x00000000 586 ldr r7, [r9, #(PCB_PL1VEC)] 587 588 /* 589 * At this point we need to kill IRQ's again. 590 * 591 * XXXSCW: Don't need to block FIQs if vectors have been relocated 592 */ 593 IRQdisableALL 594 595 /* 596 * Interrupts are disabled so we can allow user space accesses again 597 * as none will occur until interrupts are re-enabled after the 598 * switch. 599 */ 600 str r2, [r3] 601 602 /* 603 * Ensure the vector table is accessible by fixing up the L1 604 */ 605 cmp r7, #0 /* No need to fixup vector table? */ 606 ldrne r2, [r7] /* But if yes, fetch current value */ 607 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 608 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for new context */ 609 cmpne r2, r0 /* Stuffing the same value? */ 610 #ifndef PMAP_INCLUDE_PTE_SYNC 611 strne r0, [r7] /* Nope, update it */ 612 #else 613 beq .Lcs_same_vector 614 str r0, [r7] /* Otherwise, update it */ 615 616 /* 617 * Need to sync the cache to make sure that last store is 618 * visible to the MMU. 619 */ 620 ldr r2, .Lcpufuncs 621 mov r0, r7 622 mov r1, #4 623 mov lr, pc 624 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 625 626 .Lcs_same_vector: 627 #endif /* PMAP_INCLUDE_PTE_SYNC */ 628 629 cmp r10, r11 /* Switching to the same L1? */ 630 ldr r10, .Lcpufuncs 631 beq .Lcs_same_l1 /* Yup. */ 632 633 /* 634 * Do a full context switch, including full TLB flush. 635 */ 636 mov r0, r11 637 mov lr, pc 638 ldr pc, [r10, #CF_CONTEXT_SWITCH] 639 640 /* 641 * Mark the old VM space as NOT being resident in the TLB 642 */ 643 mov r2, #0x00000000 644 cmp r5, #0 645 strneh r2, [r5, #(CS_TLB_ID)] 646 b .Lcs_context_switched 647 648 /* 649 * We're switching to a different process in the same L1. 650 * In this situation, we only need to flush the TLB for the 651 * vector_page mapping, and even then only if r7 is non-NULL. 652 */ 653 .Lcs_same_l1: 654 cmp r7, #0 655 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 656 movne lr, pc 657 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 658 659 .Lcs_context_switched: 660 /* rem: r8 = &new_pmap->pm_cstate */ 661 662 /* XXXSCW: Safe to re-enable FIQs here */ 663 664 /* 665 * The new VM space is live in the cache and TLB. 666 * Update its cache/tlb state, and if it's not the kernel 667 * pmap, update the 'last cache state' pointer. 668 */ 669 mov r2, #-1 670 ldr r5, .Lpmap_kernel_cstate 671 ldr r0, .Llast_cache_state_ptr 672 str r2, [r8, #(CS_ALL)] 673 cmp r5, r8 674 strne r8, [r0] 675 676 /* rem: r4 = return value */ 677 /* rem: r6 = new lwp */ 678 /* rem: r9 = new PCB */ 679 680 /* 681 * This can be optimised... We know we want to go from SVC32 682 * mode to UND32 mode 683 */ 684 mrs r3, cpsr 685 bic r2, r3, #(PSR_MODE) 686 orr r2, r2, #(PSR_UND32_MODE) 687 msr cpsr_c, r2 688 689 ldr sp, [r9, #(PCB_UND_SP)] 690 691 msr cpsr_c, r3 /* Restore the old mode */ 692 693 /* Restore all the save registers */ 694 add r7, r9, #PCB_R8 695 ldmia r7, {r8-r13} 696 697 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 698 699 ldr r5, [r6, #(L_PROC)] /* fetch the proc for below */ 700 701 /* rem: r4 = return value */ 702 /* rem: r5 = new lwp's proc */ 703 /* rem: r6 = new lwp */ 704 /* rem: r7 = new pcb */ 705 706 #ifdef ARMFPE 707 add r0, r7, #(USER_SIZE) & 0x00ff 708 add r0, r0, #(USER_SIZE) & 0xff00 709 bl _C_LABEL(arm_fpe_core_changecontext) 710 #endif 711 712 /* We can enable interrupts again */ 713 IRQenableALL 714 715 /* rem: r4 = return value */ 716 /* rem: r5 = new lwp's proc */ 717 /* rem: r6 = new lwp */ 718 /* rem: r7 = new PCB */ 719 720 /* 721 * Check for restartable atomic sequences (RAS). 722 */ 723 724 ldr r2, [r5, #(P_NRAS)] 725 ldr r4, [r7, #(PCB_TF)] /* r4 = trapframe (used below) */ 726 teq r2, #0 /* p->p_nras == 0? */ 727 bne .Lswitch_do_ras /* no, check for one */ 728 729 .Lswitch_return: 730 731 /* Get the spl level from the stack and update the current spl level */ 732 ldr r0, [sp], #0x0004 733 bl _C_LABEL(splx) 734 735 /* cpu_switch returns 1 == switched, 0 == didn't switch */ 736 mov r0, r4 737 738 /* 739 * Pull the registers that got pushed when either savectx() or 740 * cpu_switch() was called and return. 741 */ 742 ldmfd sp!, {r4-r7, pc} 743 744 .Lswitch_do_ras: 745 ldr r1, [r4, #(TF_PC)] /* second ras_lookup() arg */ 746 mov r0, r5 /* first ras_lookup() arg */ 747 bl _C_LABEL(ras_lookup) 748 cmn r0, #1 /* -1 means "not in a RAS" */ 749 strne r0, [r4, #(TF_PC)] 750 b .Lswitch_return 751 752 .Lswitch_exited: 753 /* 754 * We skip the cache purge because switch_exit() already did it. 755 * Load up registers the way .Lcs_cache_purge_skipped expects. 756 * Userpsace access already blocked by switch_exit(). 757 */ 758 ldr r9, [r6, #(L_ADDR)] /* r9 = new PCB */ 759 ldr r3, .Lblock_userspace_access 760 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 761 mov r5, #0 /* No previous cache state */ 762 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 763 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = new cache state */ 764 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 765 b .Lcs_cache_purge_skipped 766 767 /* 768 * cpu_switchto(struct lwp *current, struct lwp *next) 769 * Switch to the specified next LWP 770 * Arguments: 771 * 772 * r0 'struct lwp *' of the current LWP 773 * r1 'struct lwp *' of the LWP to switch to 774 */ 775 ENTRY(cpu_switchto) 776 stmfd sp!, {r4-r7, lr} 777 778 /* Lower the spl level to spl0 and get the current spl level. */ 779 mov r6, r0 /* save old lwp */ 780 mov r5, r1 /* save new lwp */ 781 782 #if defined(LOCKDEBUG) 783 /* release the sched_lock before handling interrupts */ 784 bl _C_LABEL(sched_unlock_idle) 785 #endif 786 787 #ifdef __NEWINTR 788 mov r0, #(IPL_NONE) 789 bl _C_LABEL(_spllower) 790 #else /* ! __NEWINTR */ 791 #ifdef spl0 792 mov r0, #(_SPL_0) 793 bl _C_LABEL(splx) 794 #else 795 bl _C_LABEL(spl0) 796 #endif /* spl0 */ 797 #endif /* __NEWINTR */ 798 799 /* Push the old spl level onto the stack */ 800 str r0, [sp, #-0x0004]! 801 802 IRQdisable 803 #if defined(LOCKDEBUG) 804 bl _C_LABEL(sched_lock_idle) 805 #endif 806 807 mov r0, r6 /* restore old lwp */ 808 mov r1, r5 /* restore new lwp */ 809 810 /* rem: r0 = old lwp */ 811 /* rem: r1 = new lwp */ 812 /* rem: interrupts are disabled */ 813 814 /* 815 * Okay, set up registers the way cpu_switch() wants them, 816 * and jump into the middle of it (where we bring up the 817 * new process). 818 */ 819 mov r6, r1 /* r6 = new lwp */ 820 #if defined(LOCKDEBUG) 821 mov r5, r0 /* preserve old lwp */ 822 bl _C_LABEL(sched_unlock_idle) 823 mov r1, r5 /* r1 = old lwp */ 824 #else 825 mov r1, r0 /* r1 = old lwp */ 826 #endif 827 b .Lswitch_resume 828 829 /* 830 * void switch_exit(struct lwp *l, struct lwp *l0, void (*exit)(struct lwp *)); 831 * Switch to lwp0's saved context and deallocate the address space and kernel 832 * stack for l. Then jump into cpu_switch(), as if we were in lwp0 all along. 833 */ 834 835 /* LINTSTUB: Func: void switch_exit(struct lwp *l, struct lwp *l0, void (*func)(struct lwp *)) */ 836 ENTRY(switch_exit) 837 /* 838 * The process is going away, so we can use callee-saved 839 * registers here without having to save them. 840 */ 841 842 mov r4, r0 843 ldr r0, .Lcurlwp 844 845 mov r5, r1 846 ldr r1, .Lblock_userspace_access 847 848 mov r6, r2 849 850 /* 851 * r4 = lwp 852 * r5 = lwp0 853 * r6 = exit func 854 */ 855 856 mov r2, #0x00000000 /* curlwp = NULL */ 857 str r2, [r0] 858 859 /* 860 * We're about to clear both the cache and the TLB. 861 * Make sure to zap the 'last cache state' pointer since the 862 * pmap might be about to go away. Also ensure the outgoing 863 * VM space's cache state is marked as NOT resident in the 864 * cache, and that lwp0's cache state IS resident. 865 */ 866 ldr r7, [r4, #(L_ADDR)] /* r7 = old lwp's PCB */ 867 ldr r0, .Llast_cache_state_ptr /* Last userland cache state */ 868 ldr r9, [r7, #(PCB_CSTATE)] /* Fetch cache state pointer */ 869 ldr r3, [r5, #(L_ADDR)] /* r3 = lwp0's PCB */ 870 str r2, [r0] /* No previous cache state */ 871 str r2, [r9, #(CS_ALL)] /* Zap old lwp's cache state */ 872 ldr r3, [r3, #(PCB_CSTATE)] /* lwp0's cache state */ 873 mov r2, #-1 874 str r2, [r3, #(CS_ALL)] /* lwp0 is in da cache! */ 875 876 /* 877 * Don't allow user space access between the purge and the switch. 878 */ 879 mov r2, #0x00000001 880 str r2, [r1] 881 882 /* Switch to lwp0 context */ 883 884 ldr r9, .Lcpufuncs 885 mov lr, pc 886 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 887 888 ldr r0, [r7, #(PCB_PL1VEC)] 889 ldr r1, [r7, #(PCB_DACR)] 890 891 /* 892 * r0 = Pointer to L1 slot for vector_page (or NULL) 893 * r1 = lwp0's DACR 894 * r4 = lwp we're switching from 895 * r5 = lwp0 896 * r6 = exit func 897 * r7 = lwp0's PCB 898 * r9 = cpufuncs 899 */ 900 901 IRQdisableALL 902 903 /* 904 * Ensure the vector table is accessible by fixing up lwp0's L1 905 */ 906 cmp r0, #0 /* No need to fixup vector table? */ 907 ldrne r3, [r0] /* But if yes, fetch current value */ 908 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 909 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 910 cmpne r3, r2 /* Stuffing the same value? */ 911 strne r2, [r0] /* Store if not. */ 912 913 #ifdef PMAP_INCLUDE_PTE_SYNC 914 /* 915 * Need to sync the cache to make sure that last store is 916 * visible to the MMU. 917 */ 918 movne r1, #4 919 movne lr, pc 920 ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 921 #endif /* PMAP_INCLUDE_PTE_SYNC */ 922 923 /* 924 * Note: We don't do the same optimisation as cpu_switch() with 925 * respect to avoiding flushing the TLB if we're switching to 926 * the same L1 since this process' VM space may be about to go 927 * away, so we don't want *any* turds left in the TLB. 928 */ 929 930 /* Switch the memory to the new process */ 931 ldr r0, [r7, #(PCB_PAGEDIR)] 932 mov lr, pc 933 ldr pc, [r9, #CF_CONTEXT_SWITCH] 934 935 ldr r0, .Lcurpcb 936 937 /* Restore all the save registers */ 938 add r1, r7, #PCB_R8 939 ldmia r1, {r8-r13} 940 941 str r7, [r0] /* curpcb = lwp0's PCB */ 942 943 IRQenableALL 944 945 /* 946 * Schedule the vmspace and stack to be freed. 947 */ 948 mov r0, r4 /* {lwp_}exit2(l) */ 949 mov lr, pc 950 mov pc, r6 951 952 ldr r7, .Lwhichqs /* r7 = &whichqs */ 953 mov r5, #0x00000000 /* r5 = old lwp = NULL */ 954 b .Lswitch_search 955 956 /* LINTSTUB: Func: void savectx(struct pcb *pcb) */ 957 ENTRY(savectx) 958 /* 959 * r0 = pcb 960 */ 961 962 /* Push registers.*/ 963 stmfd sp!, {r4-r7, lr} 964 965 /* Store all the registers in the process's pcb */ 966 add r2, r0, #(PCB_R8) 967 stmia r2, {r8-r13} 968 969 /* Pull the regs of the stack */ 970 ldmfd sp!, {r4-r7, pc} 971 972 ENTRY(proc_trampoline) 973 #ifdef MULTIPROCESSOR 974 bl _C_LABEL(proc_trampoline_mp) 975 #endif 976 mov r0, r5 977 mov r1, sp 978 mov lr, pc 979 mov pc, r4 980 981 /* Kill irq's */ 982 mrs r0, cpsr 983 orr r0, r0, #(I32_bit) 984 msr cpsr_c, r0 985 986 PULLFRAME 987 988 movs pc, lr /* Exit */ 989 990 .type .Lcpu_switch_ffs_table, _ASM_TYPE_OBJECT; 991 .Lcpu_switch_ffs_table: 992 /* same as ffs table but all nums are -1 from that */ 993 /* 0 1 2 3 4 5 6 7 */ 994 .byte 0, 0, 1, 12, 2, 6, 0, 13 /* 0- 7 */ 995 .byte 3, 0, 7, 0, 0, 0, 0, 14 /* 8-15 */ 996 .byte 10, 4, 0, 0, 8, 0, 0, 25 /* 16-23 */ 997 .byte 0, 0, 0, 0, 0, 21, 27, 15 /* 24-31 */ 998 .byte 31, 11, 5, 0, 0, 0, 0, 0 /* 32-39 */ 999 .byte 9, 0, 0, 24, 0, 0, 20, 26 /* 40-47 */ 1000 .byte 30, 0, 0, 0, 0, 23, 0, 19 /* 48-55 */ 1001 .byte 29, 0, 22, 18, 28, 17, 16, 0 /* 56-63 */ 1002
Indexes created Sat Oct 25 10:09:55 GMT 2025