cpuswitch.S revision 1.108
1 /* $NetBSD: cpuswitch.S,v 1.108 2025/10/07 10:35:06 skrll Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 /* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 */ 79 80 #include "opt_armfpe.h" 81 #include "opt_cpuoptions.h" 82 #include "opt_kasan.h" 83 #include "opt_lockdebug.h" 84 #include "opt_multiprocessor.h" 85 86 #include "assym.h" 87 #include <arm/asm.h> 88 #include <arm/locore.h> 89 90 RCSID("$NetBSD: cpuswitch.S,v 1.108 2025/10/07 10:35:06 skrll Exp $") 91 92 /* LINTSTUB: include <sys/param.h> */ 93 94 #ifdef FPU_VFP 95 .fpu vfpv2 96 #endif 97 98 .text 99 100 /* 101 * struct lwp * 102 * cpu_switchto(struct lwp *current, struct lwp *next) 103 * 104 * Switch to the specified next LWP 105 * Arguments: 106 * 107 * r0 'struct lwp *' of the current LWP 108 * r1 'struct lwp *' of the LWP to switch to 109 * r2 returning 110 */ 111 ENTRY(cpu_switchto) 112 mov ip, sp 113 push {r4-r7, ip, lr} 114 115 /* move lwps into callee saved registers */ 116 mov r6, r1 117 mov r4, r0 118 119 #ifdef TPIDRPRW_IS_CURCPU 120 GET_CURCPU(r5) 121 #else 122 ldr r5, [r6, #L_CPU] /* get cpu from new lwp */ 123 #endif 124 125 /* rem: r4 = old lwp */ 126 /* rem: r5 = curcpu() */ 127 /* rem: r6 = new lwp */ 128 /* rem: interrupts are enabled */ 129 130 /* Save old context */ 131 132 /* Get the user structure for the old lwp. */ 133 ldr r7, [r4, #(L_PCB)] 134 135 /* Save all the registers in the old lwp's pcb */ 136 #if defined(_ARM_ARCH_DWORD_OK) 137 strd r8, r9, [r7, #(PCB_R8)] 138 strd r10, r11, [r7, #(PCB_R10)] 139 strd r12, r13, [r7, #(PCB_R12)] 140 #else 141 add r0, r7, #(PCB_R8) 142 stmia r0, {r8-r13} 143 #endif 144 145 #ifdef _ARM_ARCH_6 146 /* 147 * Save user read/write thread/process id register 148 */ 149 mrc p15, 0, r0, c13, c0, 2 150 str r0, [r7, #(PCB_USER_PID_RW)] 151 #endif 152 /* 153 * NOTE: We can now use r8-r13 until it is time to restore 154 * them for the new process. 155 */ 156 157 /* Restore saved context */ 158 159 /* rem: r4 = old lwp */ 160 /* rem: r5 = curcpu() */ 161 /* rem: r6 = new lwp */ 162 163 IRQ_DISABLE(lr) 164 #if defined(TPIDRPRW_IS_CURLWP) 165 mcr p15, 0, r6, c13, c0, 4 /* set current lwp */ 166 #endif 167 168 /* 169 * Issue barriers to coordinate mutex_exit on this CPU with 170 * mutex_vector_enter on another CPU. 171 * 172 * 1. Any prior mutex_exit by oldlwp must be visible to other 173 * CPUs before we set ci_curlwp := newlwp on this one, 174 * requiring a store-before-store barrier. 175 * 176 * 2. ci_curlwp := newlwp must be visible on all other CPUs 177 * before any subsequent mutex_exit by newlwp can even test 178 * whether there might be waiters, requiring a 179 * store-before-load barrier. 180 * 181 * See kern_mutex.c for details -- this is necessary for 182 * adaptive mutexes to detect whether the lwp is on the CPU in 183 * order to safely block without requiring atomic r/m/w in 184 * mutex_exit. 185 */ 186 187 /* We have a new curlwp now so make a note of it */ 188 #ifdef _ARM_ARCH_7 189 dmb /* store-before-store */ 190 #endif 191 str r6, [r5, #(CI_CURLWP)] 192 #ifdef _ARM_ARCH_7 193 dmb /* store-before-load */ 194 #endif 195 196 /* Get the new pcb */ 197 ldr r7, [r6, #(L_PCB)] 198 199 /* make sure we are using the new lwp's stack */ 200 ldr sp, [r7, #(PCB_KSP)] 201 202 /* At this point we can allow IRQ's again. */ 203 IRQ_ENABLE(lr) 204 205 /* rem: r4 = old lwp */ 206 /* rem: r5 = curcpu() */ 207 /* rem: r6 = new lwp */ 208 /* rem: r7 = new pcb */ 209 /* rem: interrupts are enabled */ 210 211 /* 212 * If we are switching to a system lwp, don't bother restoring 213 * thread or vfp registers and skip the ras check. 214 */ 215 ldr r0, [r6, #(L_FLAG)] 216 tst r0, #(LW_SYSTEM) 217 bne .Lswitch_do_restore 218 219 #ifdef _ARM_ARCH_6 220 /* 221 * Restore user thread/process id registers 222 */ 223 ldr r0, [r7, #(PCB_USER_PID_RW)] 224 mcr p15, 0, r0, c13, c0, 2 225 ldr r0, [r6, #(L_PRIVATE)] 226 mcr p15, 0, r0, c13, c0, 3 227 #endif 228 229 #ifdef FPU_VFP 230 /* 231 * If we have a VFP, we need to load FPEXC. 232 */ 233 ldr r0, [r5, #(CI_VFP_ID)] 234 cmp r0, #0 235 ldrne r0, [r7, #(PCB_VFP_FPEXC)] 236 vmsrne fpexc, r0 237 #endif 238 239 /* 240 * Check for restartable atomic sequences (RAS). 241 */ 242 ldr r0, [r6, #(L_PROC)] /* fetch the proc for ras_lookup */ 243 ldr r2, [r0, #(P_RASLIST)] 244 cmp r2, #0 /* p->p_nras == 0? */ 245 beq .Lswitch_do_restore 246 247 /* we can use r8 since we haven't restored saved registers yet. */ 248 ldr r8, [r6, #(L_MD_TF)] /* r1 = trapframe (used below) */ 249 ldr r1, [r8, #(TF_PC)] /* second ras_lookup() arg */ 250 bl _C_LABEL(ras_lookup) 251 cmn r0, #1 /* -1 means "not in a RAS" */ 252 strne r0, [r8, #(TF_PC)] 253 254 /* rem: r4 = old lwp */ 255 /* rem: r5 = curcpu() */ 256 /* rem: r6 = new lwp */ 257 /* rem: r7 = new pcb */ 258 259 .Lswitch_do_restore: 260 /* Restore all the saved registers */ 261 #ifdef __XSCALE__ 262 ldr r8, [r7, #(PCB_R8)] 263 ldr r9, [r7, #(PCB_R9)] 264 ldr r10, [r7, #(PCB_R10)] 265 ldr r11, [r7, #(PCB_R11)] 266 ldr r12, [r7, #(PCB_R12)] 267 #elif defined(_ARM_ARCH_DWORD_OK) 268 ldrd r8, r9, [r7, #(PCB_R8)] 269 ldrd r10, r11, [r7, #(PCB_R10)] 270 ldr r12, [r7, #(PCB_R12)] 271 #else 272 add r0, r7, #PCB_R8 273 ldmia r0, {r8-r12} 274 #endif 275 276 /* Record the old lwp for pmap_activate()'s benefit */ 277 #ifndef ARM_MMU_EXTENDED 278 str r4, [r5, #CI_LASTLWP] 279 #endif 280 281 /* cpu_switchto returns the old lwp */ 282 mov r0, r4 283 /* lwp_trampoline expects new lwp as its second argument */ 284 mov r1, r6 285 286 #ifdef _ARM_ARCH_7 287 clrex /* cause any subsequent STREX* to fail */ 288 #endif 289 290 /* 291 * Pull the registers that got pushed when cpu_switchto() was called, 292 * and return. 293 */ 294 pop {r4-r7, ip, pc} 295 296 END(cpu_switchto) 297 298 ENTRY_NP(lwp_trampoline) 299 /* 300 * cpu_switchto gives us: 301 * arg0(r0) = old lwp 302 * arg1(r1) = new lwp 303 * setup by cpu_lwp_fork: 304 * r4 = func to call 305 * r5 = arg to func 306 * r6 = <unused> 307 * r7 = spsr mode 308 */ 309 bl _C_LABEL(lwp_startup) 310 311 mov fp, #0 /* top stack frame */ 312 mov r0, r5 313 mov r1, sp 314 #ifdef _ARM_ARCH_5 315 blx r4 316 #else 317 mov lr, pc 318 mov pc, r4 319 #endif 320 321 GET_CPSR(r0) 322 CPSID_I(r0, r0) /* Kill irq's */ 323 324 /* for DO_AST */ 325 GET_CURX(r4, r5) /* r4 = curcpu, r5 = curlwp */ 326 DO_AST_AND_RESTORE_ALIGNMENT_FAULTS 327 PULLFRAME 328 329 movs pc, lr /* Exit */ 330 END(lwp_trampoline) 331 332 AST_ALIGNMENT_FAULT_LOCALS 333 334 #ifdef __HAVE_FAST_SOFTINTS 335 /* 336 * Called at IPL_HIGH 337 * r0 = new lwp 338 * r1 = ipl for softint_dispatch 339 */ 340 ENTRY_NP(softint_switch) 341 push {r4, r6, r7, lr} 342 343 ldr r7, [r0, #L_CPU] /* get curcpu */ 344 #if defined(TPIDRPRW_IS_CURLWP) 345 mrc p15, 0, r4, c13, c0, 4 /* get old lwp */ 346 #else 347 ldr r4, [r7, #(CI_CURLWP)] /* get old lwp */ 348 #endif 349 mrs r6, cpsr /* we need to save this */ 350 351 /* 352 * If the soft lwp blocks, it needs to return to softint_tramp 353 */ 354 mov r2, sp /* think ip */ 355 adr r3, softint_tramp /* think lr */ 356 push {r2-r3} 357 push {r4-r7} 358 359 mov r5, r0 /* save new lwp */ 360 361 ldr r2, [r4, #(L_PCB)] /* get old lwp's pcb */ 362 363 /* Save all the registers into the old lwp's pcb */ 364 #if defined(__XSCALE__) || defined(_ARM_ARCH_6) 365 strd r8, r9, [r2, #(PCB_R8)] 366 strd r10, r11, [r2, #(PCB_R10)] 367 strd r12, r13, [r2, #(PCB_R12)] 368 #else 369 add r3, r2, #(PCB_R8) 370 stmia r3, {r8-r13} 371 #endif 372 373 #ifdef _ARM_ARCH_6 374 /* 375 * Save user read/write thread/process id register in case it was 376 * set in userland. 377 */ 378 mrc p15, 0, r0, c13, c0, 2 379 str r0, [r2, #(PCB_USER_PID_RW)] 380 #endif 381 382 /* this is an invariant so load before disabling intrs */ 383 ldr r2, [r5, #(L_PCB)] /* get new lwp's pcb */ 384 385 IRQ_DISABLE(lr) 386 387 /* 388 * We're switching to a bound LWP so its l_cpu is already correct. 389 */ 390 #if defined(TPIDRPRW_IS_CURLWP) 391 mcr p15, 0, r5, c13, c0, 4 /* save new lwp */ 392 #endif 393 #ifdef _ARM_ARCH_7 394 dmb /* for mutex_enter; see cpu_switchto */ 395 #endif 396 str r5, [r7, #(CI_CURLWP)] /* save new lwp */ 397 /* 398 * No need for barrier after ci->ci_curlwp = softlwp -- when we 399 * enter a softint lwp, it can't be holding any mutexes, so it 400 * can't release any until after it has acquired them, so we 401 * need not participate in the protocol with mutex_vector_enter 402 * barriers here. 403 */ 404 405 #ifdef KASAN 406 mov r0, r5 407 bl _C_LABEL(kasan_softint) 408 #endif 409 410 /* 411 * Normally, we'd get {r8-r13} but since this is a softint lwp 412 * its existing state doesn't matter. We start the stack just 413 * below the trapframe. 414 */ 415 ldr sp, [r5, #(L_MD_TF)] /* get new lwp's stack ptr */ 416 417 /* At this point we can allow IRQ's again. */ 418 IRQ_ENABLE(lr) 419 /* r1 still has ipl */ 420 mov r0, r4 /* r0 has pinned (old) lwp */ 421 bl _C_LABEL(softint_dispatch) 422 /* 423 * If we've returned, we need to change everything back and return. 424 */ 425 ldr r2, [r4, #(L_PCB)] /* get pinned lwp's pcb */ 426 427 /* 428 * We don't need to restore all the registers since another lwp was 429 * never executed. But we do need the SP from the formerly pinned lwp. 430 */ 431 432 IRQ_DISABLE(lr) 433 434 #if defined(TPIDRPRW_IS_CURLWP) 435 mcr p15, 0, r4, c13, c0, 4 /* restore pinned lwp */ 436 #endif 437 #ifdef _ARM_ARCH_7 438 dmb /* for mutex_enter; see cpu_switchto */ 439 #endif 440 str r4, [r7, #(CI_CURLWP)] /* restore pinned lwp */ 441 #ifdef _ARM_ARCH_7 442 dmb /* for mutex_enter; see cpu_switchto */ 443 #endif 444 ldr sp, [r2, #(PCB_KSP)] /* now running on the old stack. */ 445 446 /* At this point we can allow IRQ's again. */ 447 msr cpsr_c, r6 448 449 /* 450 * Grab the registers that got pushed at the start and return. 451 */ 452 pop {r4-r7, ip, lr} /* eat switch frame */ 453 pop {r4, r6, r7, pc} /* pop stack and return */ 454 455 END(softint_switch) 456 457 /* 458 * r0 = previous LWP (the soft lwp) 459 * r4 = original LWP (the current lwp) 460 * r6 = original CPSR 461 * r7 = curcpu() 462 */ 463 ENTRY_NP(softint_tramp) 464 ldr r3, [r7, #(CI_MTX_COUNT)] /* readjust after mi_switch */ 465 add r3, r3, #1 466 str r3, [r7, #(CI_MTX_COUNT)] 467 468 msr cpsr_c, r6 /* restore interrupts */ 469 pop {r4, r6, r7, pc} /* pop stack and return */ 470 END(softint_tramp) 471 #endif /* __HAVE_FAST_SOFTINTS */ 472
Indexes created Sat Oct 25 10:09:55 GMT 2025