cpuswitch.S revision 1.45.2.2
1 /* $NetBSD: cpuswitch.S,v 1.45.2.2 2007/03/29 10:52:27 skrll Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_arm32_pmap.h" 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 85 #include "assym.h" 86 #include <arm/arm32/pte.h> 87 #include <machine/param.h> 88 #include <machine/cpu.h> 89 #include <machine/frame.h> 90 #include <machine/asm.h> 91 92 /* LINTSTUB: include <sys/param.h> */ 93 94 #undef IRQdisable 95 #undef IRQenable 96 97 /* 98 * New experimental definitions of IRQdisable and IRQenable 99 * These keep FIQ's enabled since FIQ's are special. 100 */ 101 102 #define IRQdisable \ 103 mrs r14, cpsr ; \ 104 orr r14, r14, #(I32_bit) ; \ 105 msr cpsr_c, r14 ; \ 106 107 #define IRQenable \ 108 mrs r14, cpsr ; \ 109 bic r14, r14, #(I32_bit) ; \ 110 msr cpsr_c, r14 ; \ 111 112 /* 113 * These are used for switching the translation table/DACR. 114 * Since the vector page can be invalid for a short time, we must 115 * disable both regular IRQs *and* FIQs. 116 * 117 * XXX: This is not necessary if the vector table is relocated. 118 */ 119 #define IRQdisableALL \ 120 mrs r14, cpsr ; \ 121 orr r14, r14, #(I32_bit | F32_bit) ; \ 122 msr cpsr_c, r14 123 124 #define IRQenableALL \ 125 mrs r14, cpsr ; \ 126 bic r14, r14, #(I32_bit | F32_bit) ; \ 127 msr cpsr_c, r14 128 129 .text 130 131 #ifdef MULTIPROCESSOR 132 .Lcpu_info_store: 133 .word _C_LABEL(cpu_info_store) 134 .Lcurpcb: 135 .word _C_LABEL(cpu_info_store) + CI_CURPCB 136 #else 137 .Lcurpcb: 138 .word _C_LABEL(curpcb) 139 #endif 140 141 .Lcpufuncs: 142 .word _C_LABEL(cpufuncs) 143 144 #ifndef MULTIPROCESSOR 145 .data 146 .global _C_LABEL(curpcb) 147 _C_LABEL(curpcb): 148 .word 0x00000000 149 .text 150 #endif 151 152 .Lblock_userspace_access: 153 .word _C_LABEL(block_userspace_access) 154 155 .Lpmap_kernel_cstate: 156 .word (kernel_pmap_store + PMAP_CSTATE) 157 158 .Llast_cache_state_ptr: 159 .word _C_LABEL(pmap_cache_state) 160 161 /* 162 * struct lwp * 163 * cpu_switchto(struct lwp *current, struct lwp *next) 164 * Switch to the specified next LWP 165 * Arguments: 166 * 167 * r0 'struct lwp *' of the current LWP 168 * r1 'struct lwp *' of the LWP to switch to 169 */ 170 ENTRY(cpu_switchto) 171 stmfd sp!, {r4-r7, lr} 172 173 mov r6, r1 /* save new lwp */ 174 mov r4, r0 /* save old lwp, it's the return value */ 175 176 IRQdisable 177 178 #ifdef MULTIPROCESSOR 179 /* XXX use curcpu() */ 180 ldr r0, .Lcpu_info_store 181 str r0, [r6, #(L_CPU)] 182 #else 183 /* l->l_cpu initialized in fork1() for single-processor */ 184 #endif 185 186 /* Hook in a new pcb */ 187 ldr r7, .Lcurpcb 188 ldr r0, [r6, #(L_ADDR)] 189 str r0, [r7] 190 191 /* At this point we can allow IRQ's again. */ 192 IRQenable 193 194 /* rem: r4 = old lwp */ 195 /* rem: r6 = new lwp */ 196 /* rem: interrupts are enabled */ 197 198 /* 199 * If the new lwp is the same as the old lwp then we do not need to 200 * save and restore any contexts. This means we can make a quick exit. 201 */ 202 teq r4, r6 203 beq .Lswitch_return 204 205 /* 206 * If the old lwp on entry to cpu_switchto was zero then the 207 * process that called it was exiting. This means that we do 208 * not need to save the current context. Instead we can jump 209 * straight to restoring the context for the new process. 210 */ 211 teq r4, #0x00000000 212 beq .Lswitch_exited 213 214 /* rem: r4 = old lwp */ 215 /* rem: r6 = new lwp */ 216 /* rem: interrupts are enabled */ 217 218 /* Stage two : Save old context */ 219 220 /* Get the user structure for the old lwp. */ 221 ldr r1, [r4, #(L_ADDR)] 222 223 /* Save all the registers in the old lwp's pcb */ 224 #ifndef __XSCALE__ 225 add r7, r1, #(PCB_R8) 226 stmia r7, {r8-r13} 227 #else 228 strd r8, [r1, #(PCB_R8)] 229 strd r10, [r1, #(PCB_R10)] 230 strd r12, [r1, #(PCB_R12)] 231 #endif 232 233 /* 234 * NOTE: We can now use r8-r13 until it is time to restore 235 * them for the new process. 236 */ 237 238 /* rem: r1 = old lwp PCB */ 239 /* rem: r4 = old lwp */ 240 /* rem: r6 = new lwp */ 241 /* rem: interrupts are enabled */ 242 243 /* Remember the old PCB. */ 244 mov r8, r1 245 246 /* r1 now free! */ 247 248 /* Get the user structure for the new process in r9 */ 249 ldr r9, [r6, #(L_ADDR)] 250 251 /* 252 * This can be optimised... We know we want to go from SVC32 253 * mode to UND32 mode 254 */ 255 mrs r3, cpsr 256 bic r2, r3, #(PSR_MODE) 257 orr r2, r2, #(PSR_UND32_MODE | I32_bit) 258 msr cpsr_c, r2 259 260 str sp, [r8, #(PCB_UND_SP)] 261 262 msr cpsr_c, r3 /* Restore the old mode */ 263 264 /* What else needs to be saved Only FPA stuff when that is supported */ 265 266 /* Third phase : restore saved context */ 267 268 /* rem: r4 = old lwp */ 269 /* rem: r6 = new lwp */ 270 /* rem: r8 = old PCB */ 271 /* rem: r9 = new PCB */ 272 /* rem: interrupts are enabled */ 273 274 /* 275 * Get the new L1 table pointer into r11. If we're switching to 276 * an LWP with the same address space as the outgoing one, we can 277 * skip the cache purge and the TTB load. 278 * 279 * To avoid data dep stalls that would happen anyway, we try 280 * and get some useful work done in the mean time. 281 */ 282 ldr r10, [r8, #(PCB_PAGEDIR)] /* r10 = old L1 */ 283 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 284 285 ldr r0, [r8, #(PCB_DACR)] /* r0 = old DACR */ 286 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 287 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = &new_pmap->pm_cstate */ 288 ldr r5, .Llast_cache_state_ptr /* Previous thread's cstate */ 289 290 teq r10, r11 /* Same L1? */ 291 ldr r5, [r5] 292 cmpeq r0, r1 /* Same DACR? */ 293 beq .Lcs_context_switched /* yes! */ 294 295 ldr r3, .Lblock_userspace_access 296 mov r12, #0 297 cmp r5, #0 /* No last vm? (switch_exit) */ 298 beq .Lcs_cache_purge_skipped /* No, we can skip cache flsh */ 299 300 mov r2, #DOMAIN_CLIENT 301 cmp r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 302 beq .Lcs_cache_purge_skipped /* Yup. Don't flush cache */ 303 304 cmp r5, r8 /* Same userland VM space? */ 305 ldrneb r12, [r5, #(CS_CACHE_ID)] /* Last VM space cache state */ 306 307 /* 308 * We're definately switching to a new userland VM space, 309 * and the previous userland VM space has yet to be flushed 310 * from the cache/tlb. 311 * 312 * r12 holds the previous VM space's cs_cache_id state 313 */ 314 tst r12, #0xff /* Test cs_cache_id */ 315 beq .Lcs_cache_purge_skipped /* VM space is not in cache */ 316 317 /* 318 * Definately need to flush the cache. 319 * Mark the old VM space as NOT being resident in the cache. 320 */ 321 322 mov r2, #0x00000000 323 strb r2, [r5, #(CS_CACHE_ID)] 324 strb r2, [r5, #(CS_CACHE_D)] 325 326 .Lcs_cache_purge: 327 /* 328 * Don't allow user space access between the purge and the switch. 329 */ 330 mov r2, #0x00000001 331 str r2, [r3] 332 333 stmfd sp!, {r0-r3} 334 ldr r1, .Lcpufuncs 335 mov lr, pc 336 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 337 ldmfd sp!, {r0-r3} 338 339 .Lcs_cache_purge_skipped: 340 /* rem: r1 = new DACR */ 341 /* rem: r3 = &block_userspace_access */ 342 /* rem: r4 = old lwp */ 343 /* rem: r5 = &old_pmap->pm_cstate (or NULL) */ 344 /* rem: r6 = new lwp */ 345 /* rem: r8 = &new_pmap->pm_cstate */ 346 /* rem: r9 = new PCB */ 347 /* rem: r10 = old L1 */ 348 /* rem: r11 = new L1 */ 349 350 mov r2, #0x00000000 351 ldr r7, [r9, #(PCB_PL1VEC)] 352 353 /* 354 * At this point we need to kill IRQ's again. 355 * 356 * XXXSCW: Don't need to block FIQs if vectors have been relocated 357 */ 358 IRQdisableALL 359 360 /* 361 * Interrupts are disabled so we can allow user space accesses again 362 * as none will occur until interrupts are re-enabled after the 363 * switch. 364 */ 365 str r2, [r3] 366 367 /* 368 * Ensure the vector table is accessible by fixing up the L1 369 */ 370 cmp r7, #0 /* No need to fixup vector table? */ 371 ldrne r2, [r7] /* But if yes, fetch current value */ 372 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 373 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for new context */ 374 cmpne r2, r0 /* Stuffing the same value? */ 375 #ifndef PMAP_INCLUDE_PTE_SYNC 376 strne r0, [r7] /* Nope, update it */ 377 #else 378 beq .Lcs_same_vector 379 str r0, [r7] /* Otherwise, update it */ 380 381 /* 382 * Need to sync the cache to make sure that last store is 383 * visible to the MMU. 384 */ 385 ldr r2, .Lcpufuncs 386 mov r0, r7 387 mov r1, #4 388 mov lr, pc 389 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 390 391 .Lcs_same_vector: 392 #endif /* PMAP_INCLUDE_PTE_SYNC */ 393 394 cmp r10, r11 /* Switching to the same L1? */ 395 ldr r10, .Lcpufuncs 396 beq .Lcs_same_l1 /* Yup. */ 397 398 /* 399 * Do a full context switch, including full TLB flush. 400 */ 401 mov r0, r11 402 mov lr, pc 403 ldr pc, [r10, #CF_CONTEXT_SWITCH] 404 405 /* 406 * Mark the old VM space as NOT being resident in the TLB 407 */ 408 mov r2, #0x00000000 409 cmp r5, #0 410 strneh r2, [r5, #(CS_TLB_ID)] 411 b .Lcs_context_switched 412 413 /* 414 * We're switching to a different process in the same L1. 415 * In this situation, we only need to flush the TLB for the 416 * vector_page mapping, and even then only if r7 is non-NULL. 417 */ 418 .Lcs_same_l1: 419 cmp r7, #0 420 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 421 movne lr, pc 422 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 423 424 .Lcs_context_switched: 425 /* rem: r8 = &new_pmap->pm_cstate */ 426 427 /* XXXSCW: Safe to re-enable FIQs here */ 428 429 /* 430 * The new VM space is live in the cache and TLB. 431 * Update its cache/tlb state, and if it's not the kernel 432 * pmap, update the 'last cache state' pointer. 433 */ 434 mov r2, #-1 435 ldr r5, .Lpmap_kernel_cstate 436 ldr r0, .Llast_cache_state_ptr 437 str r2, [r8, #(CS_ALL)] 438 cmp r5, r8 439 strne r8, [r0] 440 441 /* rem: r4 = old lwp */ 442 /* rem: r6 = new lwp */ 443 /* rem: r9 = new PCB */ 444 445 /* 446 * This can be optimised... We know we want to go from SVC32 447 * mode to UND32 mode 448 */ 449 mrs r3, cpsr 450 bic r2, r3, #(PSR_MODE) 451 orr r2, r2, #(PSR_UND32_MODE) 452 msr cpsr_c, r2 453 454 ldr sp, [r9, #(PCB_UND_SP)] 455 456 msr cpsr_c, r3 /* Restore the old mode */ 457 458 /* Restore all the save registers */ 459 #ifndef __XSCALE__ 460 add r7, r9, #PCB_R8 461 ldmia r7, {r8-r13} 462 463 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 464 #else 465 mov r7, r9 466 ldr r8, [r7, #(PCB_R8)] 467 ldr r9, [r7, #(PCB_R9)] 468 ldr r10, [r7, #(PCB_R10)] 469 ldr r11, [r7, #(PCB_R11)] 470 ldr r12, [r7, #(PCB_R12)] 471 ldr r13, [r7, #(PCB_SP)] 472 #endif 473 474 ldr r5, [r6, #(L_PROC)] /* fetch the proc for below */ 475 476 /* rem: r4 = old lwp */ 477 /* rem: r5 = new lwp's proc */ 478 /* rem: r6 = new lwp */ 479 /* rem: r7 = new pcb */ 480 481 #ifdef ARMFPE 482 add r0, r7, #(USER_SIZE) & 0x00ff 483 add r0, r0, #(USER_SIZE) & 0xff00 484 bl _C_LABEL(arm_fpe_core_changecontext) 485 #endif 486 487 /* We can enable interrupts again */ 488 IRQenableALL 489 490 /* rem: r4 = old lwp */ 491 /* rem: r5 = new lwp's proc */ 492 /* rem: r6 = new lwp */ 493 /* rem: r7 = new PCB */ 494 495 /* 496 * Check for restartable atomic sequences (RAS). 497 */ 498 499 ldr r2, [r5, #(P_RASLIST)] 500 ldr r1, [r7, #(PCB_TF)] /* r1 = trapframe (used below) */ 501 teq r2, #0 /* p->p_nras == 0? */ 502 bne .Lswitch_do_ras /* no, check for one */ 503 504 .Lswitch_return: 505 /* cpu_switchto returns the old lwp */ 506 mov r0, r4 507 /* lwp_trampoline expects new lwp as it's second argument */ 508 mov r1, r6 509 510 /* 511 * Pull the registers that got pushed when either savectx() or 512 * cpu_switchto() was called and return. 513 */ 514 ldmfd sp!, {r4-r7, pc} 515 516 .Lswitch_do_ras: 517 ldr r1, [r1, #(TF_PC)] /* second ras_lookup() arg */ 518 mov r0, r5 /* first ras_lookup() arg */ 519 bl _C_LABEL(ras_lookup) 520 cmn r0, #1 /* -1 means "not in a RAS" */ 521 ldrne r1, [r7, #(PCB_TF)] 522 strne r0, [r1, #(TF_PC)] 523 b .Lswitch_return 524 525 .Lswitch_exited: 526 527 /* 528 * We're about to clear both the cache and the TLB. 529 * Make sure to zap the 'last cache state' pointer since the 530 * pmap might be about to go away. Also ensure the outgoing 531 * VM space's cache state is marked as NOT resident in the 532 * cache. 533 */ 534 535 /* rem: r4 = old lwp (NULL) */ 536 /* rem: r6 = new lwp */ 537 /* rem: interrupts are enabled */ 538 539 /* 540 * Load up registers the way .Lcs_purge_cache expects. 541 */ 542 543 ldr r3, .Lblock_userspace_access 544 ldr r9, [r6, #(L_ADDR)] /* r9 = new PCB */ 545 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 546 mov r5, #0 /* No previous cache state */ 547 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 548 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = new cache state */ 549 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 550 b .Lcs_cache_purge 551 552 /* LINTSTUB: Func: void savectx(struct pcb *pcb) */ 553 ENTRY(savectx) 554 /* 555 * r0 = pcb 556 */ 557 558 /* Push registers.*/ 559 stmfd sp!, {r4-r7, lr} 560 561 /* Store all the registers in the process's pcb */ 562 #ifndef __XSCALE__ 563 add r2, r0, #(PCB_R8) 564 stmia r2, {r8-r13} 565 #else 566 strd r8, [r0, #(PCB_R8)] 567 strd r10, [r0, #(PCB_R10)] 568 strd r12, [r0, #(PCB_R12)] 569 #endif 570 571 /* Pull the regs of the stack */ 572 ldmfd sp!, {r4-r7, pc} 573 574 ENTRY(lwp_trampoline) 575 bl _C_LABEL(lwp_startup) 576 577 mov r0, r5 578 mov r1, sp 579 mov lr, pc 580 mov pc, r4 581 582 /* Kill irq's */ 583 mrs r0, cpsr 584 orr r0, r0, #(I32_bit) 585 msr cpsr_c, r0 586 587 PULLFRAME 588 589 movs pc, lr /* Exit */ 590
Indexes created Mon Oct 27 17:09:47 GMT 2025