cpuswitch.S revision 1.41.16.3
1 /* $NetBSD: cpuswitch.S,v 1.41.16.3 2007/09/03 14:23:14 yamt Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_arm32_pmap.h" 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 85 #include "assym.h" 86 #include <arm/arm32/pte.h> 87 #include <machine/param.h> 88 #include <machine/cpu.h> 89 #include <machine/frame.h> 90 #include <machine/asm.h> 91 92 /* LINTSTUB: include <sys/param.h> */ 93 94 #undef IRQdisable 95 #undef IRQenable 96 97 /* 98 * New experimental definitions of IRQdisable and IRQenable 99 * These keep FIQ's enabled since FIQ's are special. 100 */ 101 102 #define IRQdisable \ 103 mrs r14, cpsr ; \ 104 orr r14, r14, #(I32_bit) ; \ 105 msr cpsr_c, r14 ; \ 106 107 #define IRQenable \ 108 mrs r14, cpsr ; \ 109 bic r14, r14, #(I32_bit) ; \ 110 msr cpsr_c, r14 ; \ 111 112 /* 113 * These are used for switching the translation table/DACR. 114 * Since the vector page can be invalid for a short time, we must 115 * disable both regular IRQs *and* FIQs. 116 * 117 * XXX: This is not necessary if the vector table is relocated. 118 */ 119 #define IRQdisableALL \ 120 mrs r14, cpsr ; \ 121 orr r14, r14, #(I32_bit | F32_bit) ; \ 122 msr cpsr_c, r14 123 124 #define IRQenableALL \ 125 mrs r14, cpsr ; \ 126 bic r14, r14, #(I32_bit | F32_bit) ; \ 127 msr cpsr_c, r14 128 129 .text 130 131 #ifdef MULTIPROCESSOR 132 .Lcpu_info_store: 133 .word _C_LABEL(cpu_info_store) 134 .Lcurlwp: 135 /* FIXME: This is bogus in the general case. */ 136 .word _C_LABEL(cpu_info_store) + CI_CURLWP 137 138 .Lcurpcb: 139 .word _C_LABEL(cpu_info_store) + CI_CURPCB 140 #else 141 .Lcurlwp: 142 .word _C_LABEL(curlwp) 143 144 .Lcurpcb: 145 .word _C_LABEL(curpcb) 146 #endif 147 148 .Lcpufuncs: 149 .word _C_LABEL(cpufuncs) 150 151 #ifndef MULTIPROCESSOR 152 .data 153 .global _C_LABEL(curpcb) 154 _C_LABEL(curpcb): 155 .word 0x00000000 156 .text 157 #endif 158 159 .Lblock_userspace_access: 160 .word _C_LABEL(block_userspace_access) 161 162 .Lpmap_kernel_cstate: 163 .word (kernel_pmap_store + PMAP_CSTATE) 164 165 .Llast_cache_state_ptr: 166 .word _C_LABEL(pmap_cache_state) 167 168 /* 169 * struct lwp * 170 * cpu_switchto(struct lwp *current, struct lwp *next) 171 * 172 * Switch to the specified next LWP 173 * Arguments: 174 * 175 * r0 'struct lwp *' of the current LWP 176 * r1 'struct lwp *' of the LWP to switch to 177 */ 178 ENTRY(cpu_switchto) 179 stmfd sp!, {r4-r7, lr} 180 181 mov r6, r1 /* save new lwp */ 182 mov r4, r0 /* save old lwp, it's the return value */ 183 184 IRQdisable 185 186 #ifdef MULTIPROCESSOR 187 /* XXX use curcpu() */ 188 ldr r0, .Lcpu_info_store 189 str r0, [r6, #(L_CPU)] 190 #else 191 /* l->l_cpu initialized in fork1() for single-processor */ 192 #endif 193 194 /* We have a new curlwp now so make a note it */ 195 ldr r7, .Lcurlwp 196 str r6, [r7] 197 198 /* Hook in a new pcb */ 199 ldr r7, .Lcurpcb 200 ldr r0, [r6, #(L_ADDR)] 201 str r0, [r7] 202 203 /* At this point we can allow IRQ's again. */ 204 IRQenable 205 206 /* rem: r4 = old lwp */ 207 /* rem: r6 = new lwp */ 208 /* rem: interrupts are enabled */ 209 210 /* 211 * If the old lwp on entry to cpu_switchto was zero then the 212 * process that called it was exiting. This means that we do 213 * not need to save the current context. Instead we can jump 214 * straight to restoring the context for the new process. 215 */ 216 teq r4, #0x00000000 217 beq .Lswitch_exited 218 219 /* rem: r4 = old lwp */ 220 /* rem: r6 = new lwp */ 221 /* rem: interrupts are enabled */ 222 223 /* Save old context */ 224 225 /* Get the user structure for the old lwp. */ 226 ldr r1, [r4, #(L_ADDR)] 227 228 /* Save all the registers in the old lwp's pcb */ 229 #ifndef __XSCALE__ 230 add r7, r1, #(PCB_R8) 231 stmia r7, {r8-r13} 232 #else 233 strd r8, [r1, #(PCB_R8)] 234 strd r10, [r1, #(PCB_R10)] 235 strd r12, [r1, #(PCB_R12)] 236 #endif 237 238 /* 239 * NOTE: We can now use r8-r13 until it is time to restore 240 * them for the new process. 241 */ 242 243 /* rem: r1 = old lwp PCB */ 244 /* rem: r4 = old lwp */ 245 /* rem: r6 = new lwp */ 246 /* rem: interrupts are enabled */ 247 248 /* Remember the old PCB. */ 249 mov r8, r1 250 251 /* r1 now free! */ 252 253 /* Get the user structure for the new process in r9 */ 254 ldr r9, [r6, #(L_ADDR)] 255 256 /* 257 * This can be optimised... We know we want to go from SVC32 258 * mode to UND32 mode 259 */ 260 mrs r3, cpsr 261 bic r2, r3, #(PSR_MODE) 262 orr r2, r2, #(PSR_UND32_MODE | I32_bit) 263 msr cpsr_c, r2 264 265 str sp, [r8, #(PCB_UND_SP)] 266 267 msr cpsr_c, r3 /* Restore the old mode */ 268 269 /* What else needs to be saved? Only FPA stuff when that is supported */ 270 271 /* Restore saved context */ 272 273 /* rem: r4 = old lwp */ 274 /* rem: r6 = new lwp */ 275 /* rem: r8 = old PCB */ 276 /* rem: r9 = new PCB */ 277 /* rem: interrupts are enabled */ 278 279 /* 280 * Get the new L1 table pointer into r11. If we're switching to 281 * an LWP with the same address space as the outgoing one, we can 282 * skip the cache purge and the TTB load. 283 * 284 * To avoid data dep stalls that would happen anyway, we try 285 * and get some useful work done in the mean time. 286 */ 287 ldr r10, [r8, #(PCB_PAGEDIR)] /* r10 = old L1 */ 288 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 289 290 ldr r0, [r8, #(PCB_DACR)] /* r0 = old DACR */ 291 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 292 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = &new_pmap->pm_cstate */ 293 ldr r5, .Llast_cache_state_ptr /* Previous thread's cstate */ 294 295 teq r10, r11 /* Same L1? */ 296 ldr r5, [r5] 297 cmpeq r0, r1 /* Same DACR? */ 298 beq .Lcs_context_switched /* yes! */ 299 300 ldr r3, .Lblock_userspace_access 301 mov r12, #0 302 cmp r5, #0 /* No last vm? (switch_exit) */ 303 beq .Lcs_cache_purge_skipped /* No, we can skip cache flsh */ 304 305 mov r2, #DOMAIN_CLIENT 306 cmp r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 307 beq .Lcs_cache_purge_skipped /* Yup. Don't flush cache */ 308 309 cmp r5, r8 /* Same userland VM space? */ 310 ldrneb r12, [r5, #(CS_CACHE_ID)] /* Last VM space cache state */ 311 312 /* 313 * We're definately switching to a new userland VM space, 314 * and the previous userland VM space has yet to be flushed 315 * from the cache/tlb. 316 * 317 * r12 holds the previous VM space's cs_cache_id state 318 */ 319 tst r12, #0xff /* Test cs_cache_id */ 320 beq .Lcs_cache_purge_skipped /* VM space is not in cache */ 321 322 /* 323 * Definately need to flush the cache. 324 * Mark the old VM space as NOT being resident in the cache. 325 */ 326 327 mov r2, #0x00000000 328 strb r2, [r5, #(CS_CACHE_ID)] 329 strb r2, [r5, #(CS_CACHE_D)] 330 331 .Lcs_cache_purge: 332 /* 333 * Don't allow user space access between the purge and the switch. 334 */ 335 mov r2, #0x00000001 336 str r2, [r3] 337 338 stmfd sp!, {r0-r3} 339 ldr r1, .Lcpufuncs 340 mov lr, pc 341 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 342 ldmfd sp!, {r0-r3} 343 344 .Lcs_cache_purge_skipped: 345 /* rem: r1 = new DACR */ 346 /* rem: r3 = &block_userspace_access */ 347 /* rem: r4 = old lwp */ 348 /* rem: r5 = &old_pmap->pm_cstate (or NULL) */ 349 /* rem: r6 = new lwp */ 350 /* rem: r8 = &new_pmap->pm_cstate */ 351 /* rem: r9 = new PCB */ 352 /* rem: r10 = old L1 */ 353 /* rem: r11 = new L1 */ 354 355 mov r2, #0x00000000 356 ldr r7, [r9, #(PCB_PL1VEC)] 357 358 /* 359 * At this point we need to kill IRQ's again. 360 * 361 * XXXSCW: Don't need to block FIQs if vectors have been relocated 362 */ 363 IRQdisableALL 364 365 /* 366 * Interrupts are disabled so we can allow user space accesses again 367 * as none will occur until interrupts are re-enabled after the 368 * switch. 369 */ 370 str r2, [r3] 371 372 /* 373 * Ensure the vector table is accessible by fixing up the L1 374 */ 375 cmp r7, #0 /* No need to fixup vector table? */ 376 ldrne r2, [r7] /* But if yes, fetch current value */ 377 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 378 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for new context */ 379 cmpne r2, r0 /* Stuffing the same value? */ 380 #ifndef PMAP_INCLUDE_PTE_SYNC 381 strne r0, [r7] /* Nope, update it */ 382 #else 383 beq .Lcs_same_vector 384 str r0, [r7] /* Otherwise, update it */ 385 386 /* 387 * Need to sync the cache to make sure that last store is 388 * visible to the MMU. 389 */ 390 ldr r2, .Lcpufuncs 391 mov r0, r7 392 mov r1, #4 393 mov lr, pc 394 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 395 396 .Lcs_same_vector: 397 #endif /* PMAP_INCLUDE_PTE_SYNC */ 398 399 cmp r10, r11 /* Switching to the same L1? */ 400 ldr r10, .Lcpufuncs 401 beq .Lcs_same_l1 /* Yup. */ 402 403 /* 404 * Do a full context switch, including full TLB flush. 405 */ 406 mov r0, r11 407 mov lr, pc 408 ldr pc, [r10, #CF_CONTEXT_SWITCH] 409 410 /* 411 * Mark the old VM space as NOT being resident in the TLB 412 */ 413 mov r2, #0x00000000 414 cmp r5, #0 415 strneh r2, [r5, #(CS_TLB_ID)] 416 b .Lcs_context_switched 417 418 /* 419 * We're switching to a different process in the same L1. 420 * In this situation, we only need to flush the TLB for the 421 * vector_page mapping, and even then only if r7 is non-NULL. 422 */ 423 .Lcs_same_l1: 424 cmp r7, #0 425 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 426 movne lr, pc 427 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 428 429 .Lcs_context_switched: 430 /* rem: r8 = &new_pmap->pm_cstate */ 431 432 /* XXXSCW: Safe to re-enable FIQs here */ 433 434 /* 435 * The new VM space is live in the cache and TLB. 436 * Update its cache/tlb state, and if it's not the kernel 437 * pmap, update the 'last cache state' pointer. 438 */ 439 mov r2, #-1 440 ldr r5, .Lpmap_kernel_cstate 441 ldr r0, .Llast_cache_state_ptr 442 str r2, [r8, #(CS_ALL)] 443 cmp r5, r8 444 strne r8, [r0] 445 446 /* rem: r4 = old lwp */ 447 /* rem: r6 = new lwp */ 448 /* rem: r9 = new PCB */ 449 450 /* 451 * This can be optimised... We know we want to go from SVC32 452 * mode to UND32 mode 453 */ 454 mrs r3, cpsr 455 bic r2, r3, #(PSR_MODE) 456 orr r2, r2, #(PSR_UND32_MODE) 457 msr cpsr_c, r2 458 459 ldr sp, [r9, #(PCB_UND_SP)] 460 461 msr cpsr_c, r3 /* Restore the old mode */ 462 463 /* Restore all the save registers */ 464 #ifndef __XSCALE__ 465 add r7, r9, #PCB_R8 466 ldmia r7, {r8-r13} 467 468 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 469 #else 470 mov r7, r9 471 ldr r8, [r7, #(PCB_R8)] 472 ldr r9, [r7, #(PCB_R9)] 473 ldr r10, [r7, #(PCB_R10)] 474 ldr r11, [r7, #(PCB_R11)] 475 ldr r12, [r7, #(PCB_R12)] 476 ldr r13, [r7, #(PCB_SP)] 477 #endif 478 479 ldr r5, [r6, #(L_PROC)] /* fetch the proc for below */ 480 481 /* rem: r4 = old lwp */ 482 /* rem: r5 = new lwp's proc */ 483 /* rem: r6 = new lwp */ 484 /* rem: r7 = new pcb */ 485 486 #ifdef ARMFPE 487 add r0, r7, #(USER_SIZE) & 0x00ff 488 add r0, r0, #(USER_SIZE) & 0xff00 489 bl _C_LABEL(arm_fpe_core_changecontext) 490 #endif 491 492 /* We can enable interrupts again */ 493 IRQenableALL 494 495 /* rem: r4 = old lwp */ 496 /* rem: r5 = new lwp's proc */ 497 /* rem: r6 = new lwp */ 498 /* rem: r7 = new PCB */ 499 500 /* 501 * Check for restartable atomic sequences (RAS). 502 */ 503 504 ldr r2, [r5, #(P_RASLIST)] 505 ldr r1, [r7, #(PCB_TF)] /* r1 = trapframe (used below) */ 506 teq r2, #0 /* p->p_nras == 0? */ 507 bne .Lswitch_do_ras /* no, check for one */ 508 509 .Lswitch_return: 510 /* cpu_switchto returns the old lwp */ 511 mov r0, r4 512 /* lwp_trampoline expects new lwp as it's second argument */ 513 mov r1, r6 514 515 /* 516 * Pull the registers that got pushed when either savectx() or 517 * cpu_switchto() was called and return. 518 */ 519 ldmfd sp!, {r4-r7, pc} 520 521 .Lswitch_do_ras: 522 ldr r1, [r1, #(TF_PC)] /* second ras_lookup() arg */ 523 mov r0, r5 /* first ras_lookup() arg */ 524 bl _C_LABEL(ras_lookup) 525 cmn r0, #1 /* -1 means "not in a RAS" */ 526 ldrne r1, [r7, #(PCB_TF)] 527 strne r0, [r1, #(TF_PC)] 528 b .Lswitch_return 529 530 .Lswitch_exited: 531 532 /* 533 * We're about to clear both the cache and the TLB. 534 * Make sure to zap the 'last cache state' pointer since the 535 * pmap might be about to go away. Also ensure the outgoing 536 * VM space's cache state is marked as NOT resident in the 537 * cache. 538 */ 539 540 /* rem: r4 = old lwp (NULL) */ 541 /* rem: r6 = new lwp */ 542 /* rem: interrupts are enabled */ 543 544 /* 545 * Load up registers the way .Lcs_purge_cache expects. 546 */ 547 548 ldr r3, .Lblock_userspace_access 549 ldr r9, [r6, #(L_ADDR)] /* r9 = new PCB */ 550 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 551 mov r5, #0 /* No previous cache state */ 552 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 553 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = new cache state */ 554 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 555 b .Lcs_cache_purge 556 557 /* LINTSTUB: Func: void savectx(struct pcb *pcb) */ 558 ENTRY(savectx) 559 /* 560 * r0 = pcb 561 */ 562 563 /* Push registers.*/ 564 stmfd sp!, {r4-r7, lr} 565 566 /* Store all the registers in the process's pcb */ 567 #ifndef __XSCALE__ 568 add r2, r0, #(PCB_R8) 569 stmia r2, {r8-r13} 570 #else 571 strd r8, [r0, #(PCB_R8)] 572 strd r10, [r0, #(PCB_R10)] 573 strd r12, [r0, #(PCB_R12)] 574 #endif 575 576 /* Pull the regs of the stack */ 577 ldmfd sp!, {r4-r7, pc} 578 579 ENTRY(lwp_trampoline) 580 bl _C_LABEL(lwp_startup) 581 582 mov r0, r5 583 mov r1, sp 584 mov lr, pc 585 mov pc, r4 586 587 /* Kill irq's */ 588 mrs r0, cpsr 589 orr r0, r0, #(I32_bit) 590 msr cpsr_c, r0 591 592 PULLFRAME 593 594 movs pc, lr /* Exit */ 595
Indexes created Sat Oct 25 16:10:12 GMT 2025