cpuswitch.S revision 1.103
1 /* $NetBSD: cpuswitch.S,v 1.103 2020/08/15 13:33:54 skrll Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_cpuoptions.h" 82 #include "opt_kasan.h" 83 #include "opt_lockdebug.h" 84 #include "opt_multiprocessor.h" 85 86 #include "assym.h" 87 #include <arm/asm.h> 88 #include <arm/locore.h> 89 90 RCSID("$NetBSD: cpuswitch.S,v 1.103 2020/08/15 13:33:54 skrll Exp $") 91 92 /* LINTSTUB: include <sys/param.h> */ 93 94 #ifdef FPU_VFP 95 .fpu vfpv2 96 #endif 97 98 #undef IRQdisable 99 #undef IRQenable 100 101 /* 102 * New experimental definitions of IRQdisable and IRQenable 103 * These keep FIQ's enabled since FIQ's are special. 104 */ 105 106 #ifdef _ARM_ARCH_6 107 #define IRQdisable cpsid i 108 #define IRQenable cpsie i 109 #else 110 #define IRQdisable \ 111 mrs r14, cpsr ; \ 112 orr r14, r14, #(I32_bit) ; \ 113 msr cpsr_c, r14 114 115 #define IRQenable \ 116 mrs r14, cpsr ; \ 117 bic r14, r14, #(I32_bit) ; \ 118 msr cpsr_c, r14 119 120 #endif 121 122 .text 123 124 /* 125 * struct lwp * 126 * cpu_switchto(struct lwp *current, struct lwp *next) 127 * 128 * Switch to the specified next LWP 129 * Arguments: 130 * 131 * r0 'struct lwp *' of the current LWP 132 * r1 'struct lwp *' of the LWP to switch to 133 * r2 returning 134 */ 135 ENTRY(cpu_switchto) 136 mov ip, sp 137 push {r4-r7, ip, lr} 138 139 /* move lwps into caller saved registers */ 140 mov r6, r1 141 mov r4, r0 142 143 #ifdef TPIDRPRW_IS_CURCPU 144 GET_CURCPU(r5) 145 #else 146 ldr r5, [r6, #L_CPU] /* get cpu from new lwp */ 147 #endif 148 149 /* rem: r4 = old lwp */ 150 /* rem: r5 = curcpu() */ 151 /* rem: r6 = new lwp */ 152 /* rem: interrupts are enabled */ 153 154 /* Save old context */ 155 156 /* Get the user structure for the old lwp. */ 157 ldr r7, [r4, #(L_PCB)] 158 159 /* Save all the registers in the old lwp's pcb */ 160 #if defined(_ARM_ARCH_DWORD_OK) 161 strd r8, r9, [r7, #(PCB_R8)] 162 strd r10, r11, [r7, #(PCB_R10)] 163 strd r12, r13, [r7, #(PCB_R12)] 164 #else 165 add r0, r7, #(PCB_R8) 166 stmia r0, {r8-r13} 167 #endif 168 169 #ifdef _ARM_ARCH_6 170 /* 171 * Save user read/write thread/process id register 172 */ 173 mrc p15, 0, r0, c13, c0, 2 174 str r0, [r7, #(PCB_USER_PID_RW)] 175 #endif 176 /* 177 * NOTE: We can now use r8-r13 until it is time to restore 178 * them for the new process. 179 */ 180 181 /* Restore saved context */ 182 183 /* rem: r4 = old lwp */ 184 /* rem: r5 = curcpu() */ 185 /* rem: r6 = new lwp */ 186 187 IRQdisable 188 #if defined(TPIDRPRW_IS_CURLWP) 189 mcr p15, 0, r6, c13, c0, 4 /* set current lwp */ 190 #endif 191 192 /* We have a new curlwp now so make a note of it */ 193 str r6, [r5, #(CI_CURLWP)] 194 195 #ifdef _ARM_ARCH_7 196 dmb /* see comments in kern_mutex.c */ 197 #endif 198 199 /* Get the new pcb */ 200 ldr r7, [r6, #(L_PCB)] 201 202 /* make sure we are using the new lwp's stack */ 203 ldr sp, [r7, #(PCB_KSP)] 204 205 /* At this point we can allow IRQ's again. */ 206 IRQenable 207 208 /* rem: r4 = old lwp */ 209 /* rem: r5 = curcpu() */ 210 /* rem: r6 = new lwp */ 211 /* rem: r7 = new pcb */ 212 /* rem: interrupts are enabled */ 213 214 /* 215 * If we are switching to a system lwp, don't bother restoring 216 * thread or vfp registers and skip the ras check. 217 */ 218 ldr r0, [r6, #(L_FLAG)] 219 tst r0, #(LW_SYSTEM) 220 bne .Lswitch_do_restore 221 222 #ifdef _ARM_ARCH_6 223 /* 224 * Restore user thread/process id registers 225 */ 226 ldr r0, [r7, #(PCB_USER_PID_RW)] 227 mcr p15, 0, r0, c13, c0, 2 228 ldr r0, [r6, #(L_PRIVATE)] 229 mcr p15, 0, r0, c13, c0, 3 230 #endif 231 232 #ifdef FPU_VFP 233 /* 234 * If we have a VFP, we need to load FPEXC. 235 */ 236 ldr r0, [r5, #(CI_VFP_ID)] 237 cmp r0, #0 238 ldrne r0, [r7, #(PCB_VFP_FPEXC)] 239 vmsrne fpexc, r0 240 #endif 241 242 /* 243 * Check for restartable atomic sequences (RAS). 244 */ 245 ldr r0, [r6, #(L_PROC)] /* fetch the proc for ras_lookup */ 246 ldr r2, [r0, #(P_RASLIST)] 247 cmp r2, #0 /* p->p_nras == 0? */ 248 beq .Lswitch_do_restore 249 250 /* we can use r8 since we haven't restored saved registers yet. */ 251 ldr r8, [r6, #(L_MD_TF)] /* r1 = trapframe (used below) */ 252 ldr r1, [r8, #(TF_PC)] /* second ras_lookup() arg */ 253 bl _C_LABEL(ras_lookup) 254 cmn r0, #1 /* -1 means "not in a RAS" */ 255 strne r0, [r8, #(TF_PC)] 256 257 /* rem: r4 = old lwp */ 258 /* rem: r5 = curcpu() */ 259 /* rem: r6 = new lwp */ 260 /* rem: r7 = new pcb */ 261 262 .Lswitch_do_restore: 263 /* Restore all the saved registers */ 264 #ifdef __XSCALE__ 265 ldr r8, [r7, #(PCB_R8)] 266 ldr r9, [r7, #(PCB_R9)] 267 ldr r10, [r7, #(PCB_R10)] 268 ldr r11, [r7, #(PCB_R11)] 269 ldr r12, [r7, #(PCB_R12)] 270 #elif defined(_ARM_ARCH_DWORD_OK) 271 ldrd r8, r9, [r7, #(PCB_R8)] 272 ldrd r10, r11, [r7, #(PCB_R10)] 273 ldr r12, [r7, #(PCB_R12)] 274 #else 275 add r0, r7, #PCB_R8 276 ldmia r0, {r8-r12} 277 #endif 278 279 /* Record the old lwp for pmap_activate()'s benefit */ 280 #ifndef ARM_MMU_EXTENDED 281 str r4, [r5, #CI_LASTLWP] 282 #endif 283 284 /* cpu_switchto returns the old lwp */ 285 mov r0, r4 286 /* lwp_trampoline expects new lwp as its second argument */ 287 mov r1, r6 288 289 #ifdef _ARM_ARCH_7 290 clrex /* cause any subsequent STREX* to fail */ 291 #endif 292 293 /* 294 * Pull the registers that got pushed when cpu_switchto() was called, 295 * and return. 296 */ 297 pop {r4-r7, ip, pc} 298 299 END(cpu_switchto) 300 301 ENTRY_NP(lwp_trampoline) 302 /* 303 * cpu_switchto gives us: 304 * arg0(r0) = old lwp 305 * arg1(r1) = new lwp 306 * setup by cpu_lwp_fork: 307 * r4 = func to call 308 * r5 = arg to func 309 * r6 = <unused> 310 * r7 = spsr mode 311 */ 312 bl _C_LABEL(lwp_startup) 313 314 mov fp, #0 /* top stack frame */ 315 mov r0, r5 316 mov r1, sp 317 #ifdef _ARM_ARCH_5 318 blx r4 319 #else 320 mov lr, pc 321 mov pc, r4 322 #endif 323 324 GET_CPSR(r0) 325 CPSID_I(r0, r0) /* Kill irq's */ 326 327 GET_CURCPU(r4) /* for DO_AST */ 328 DO_AST_AND_RESTORE_ALIGNMENT_FAULTS 329 PULLFRAME 330 331 movs pc, lr /* Exit */ 332 END(lwp_trampoline) 333 334 AST_ALIGNMENT_FAULT_LOCALS 335 336 #ifdef __HAVE_FAST_SOFTINTS 337 /* 338 * Called at IPL_HIGH 339 * r0 = new lwp 340 * r1 = ipl for softint_dispatch 341 */ 342 ENTRY_NP(softint_switch) 343 push {r4, r6, r7, lr} 344 345 ldr r7, [r0, #L_CPU] /* get curcpu */ 346 #if defined(TPIDRPRW_IS_CURLWP) 347 mrc p15, 0, r4, c13, c0, 4 /* get old lwp */ 348 #else 349 ldr r4, [r7, #(CI_CURLWP)] /* get old lwp */ 350 #endif 351 mrs r6, cpsr /* we need to save this */ 352 353 /* 354 * If the soft lwp blocks, it needs to return to softint_tramp 355 */ 356 mov r2, sp /* think ip */ 357 adr r3, softint_tramp /* think lr */ 358 push {r2-r3} 359 push {r4-r7} 360 361 mov r5, r0 /* save new lwp */ 362 363 ldr r2, [r4, #(L_PCB)] /* get old lwp's pcb */ 364 365 /* Save all the registers into the old lwp's pcb */ 366 #if defined(__XSCALE__) || defined(_ARM_ARCH_6) 367 strd r8, r9, [r2, #(PCB_R8)] 368 strd r10, r11, [r2, #(PCB_R10)] 369 strd r12, r13, [r2, #(PCB_R12)] 370 #else 371 add r3, r2, #(PCB_R8) 372 stmia r3, {r8-r13} 373 #endif 374 375 #ifdef _ARM_ARCH_6 376 /* 377 * Save user read/write thread/process id register in case it was 378 * set in userland. 379 */ 380 mrc p15, 0, r0, c13, c0, 2 381 str r0, [r2, #(PCB_USER_PID_RW)] 382 #endif 383 384 /* this is an invariant so load before disabling intrs */ 385 ldr r2, [r5, #(L_PCB)] /* get new lwp's pcb */ 386 387 IRQdisable 388 /* 389 * We're switching to a bound LWP so its l_cpu is already correct. 390 */ 391 #if defined(TPIDRPRW_IS_CURLWP) 392 mcr p15, 0, r5, c13, c0, 4 /* save new lwp */ 393 #endif 394 str r5, [r7, #(CI_CURLWP)] /* save new lwp */ 395 #ifdef _ARM_ARCH_7 396 dmb /* see comments in kern_mutex.c */ 397 #endif 398 399 #ifdef KASAN 400 mov r0, r5 401 bl _C_LABEL(kasan_softint) 402 #endif 403 404 /* 405 * Normally, we'd get {r8-r13} but since this is a softint lwp 406 * its existing state doesn't matter. We start the stack just 407 * below the trapframe. 408 */ 409 ldr sp, [r5, #(L_MD_TF)] /* get new lwp's stack ptr */ 410 411 /* At this point we can allow IRQ's again. */ 412 IRQenable 413 /* r1 still has ipl */ 414 mov r0, r4 /* r0 has pinned (old) lwp */ 415 bl _C_LABEL(softint_dispatch) 416 /* 417 * If we've returned, we need to change everything back and return. 418 */ 419 ldr r2, [r4, #(L_PCB)] /* get pinned lwp's pcb */ 420 421 /* 422 * We don't need to restore all the registers since another lwp was 423 * never executed. But we do need the SP from the formerly pinned lwp. 424 */ 425 426 IRQdisable 427 #if defined(TPIDRPRW_IS_CURLWP) 428 mcr p15, 0, r4, c13, c0, 4 /* restore pinned lwp */ 429 #endif 430 str r4, [r7, #(CI_CURLWP)] /* restore pinned lwp */ 431 ldr sp, [r2, #(PCB_KSP)] /* now running on the old stack. */ 432 433 /* At this point we can allow IRQ's again. */ 434 msr cpsr_c, r6 435 436 /* 437 * Grab the registers that got pushed at the start and return. 438 */ 439 pop {r4-r7, ip, lr} /* eat switch frame */ 440 pop {r4, r6, r7, pc} /* pop stack and return */ 441 442 END(softint_switch) 443 444 /* 445 * r0 = previous LWP (the soft lwp) 446 * r4 = original LWP (the current lwp) 447 * r6 = original CPSR 448 * r7 = curcpu() 449 */ 450 ENTRY_NP(softint_tramp) 451 ldr r3, [r7, #(CI_MTX_COUNT)] /* readjust after mi_switch */ 452 add r3, r3, #1 453 str r3, [r7, #(CI_MTX_COUNT)] 454 455 msr cpsr_c, r6 /* restore interrupts */ 456 pop {r4, r6, r7, pc} /* pop stack and return */ 457 END(softint_tramp) 458 #endif /* __HAVE_FAST_SOFTINTS */ 459
Indexes created Thu Oct 23 22:10:10 GMT 2025