1 /* $NetBSD: spl.S,v 1.58 2023/03/01 08:38:50 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2007, 2008, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <machine/asm.h> 33 __KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.58 2023/03/01 08:38:50 riastradh Exp $"); 34 35 #include "opt_ddb.h" 36 #include "opt_spldebug.h" 37 #include "opt_xen.h" 38 39 #include <machine/trap.h> 40 #include <machine/segments.h> 41 #include <machine/frameasm.h> 42 43 #include "assym.h" 44 45 .text 46 47 /* 48 * int splraise(int s); 49 */ 50 ENTRY(splraise) 51 movl 4(%esp),%edx 52 movzbl CPUVAR(ILEVEL),%eax 53 cmpl %edx,%eax 54 ja 1f 55 movb %dl,CPUVAR(ILEVEL) 56 1: 57 #ifdef SPLDEBUG 58 pushl %ebp 59 movl %esp,%ebp 60 pushl %eax 61 pushl %edx 62 call _C_LABEL(spldebug_raise) 63 addl $4,%esp 64 popl %eax 65 popl %ebp 66 #endif /* SPLDEBUG */ 67 ret 68 END(splraise) 69 70 #ifndef XENPV 71 /* 72 * void spllower(int s); 73 * 74 * spllower() for i486 and Pentium. Must be the same size as cx8_spllower(), 75 * that is, 96 bytes. This must use pushf/cli/popf as it is used early in boot 76 * where interrupts are disabled via eflags/IE. 77 */ 78 ENTRY(spllower) 79 HOTPATCH(HP_NAME_SPLLOWER, 96) 80 #ifdef SPLDEBUG 81 movl 4(%esp),%ecx 82 pushl %ebp 83 movl %esp,%ebp 84 pushl %ecx 85 call _C_LABEL(spldebug_lower) 86 addl $4,%esp 87 popl %ebp 88 #endif /* SPLDEBUG */ 89 movl 4(%esp),%ecx 90 cmpb CPUVAR(ILEVEL),%cl 91 jae 1f 92 movl CPUVAR(IUNMASK)(,%ecx,8),%edx 93 movl CPUVAR(IUNMASK)+4(,%ecx,8),%eax 94 PUSHF(%eax) 95 CLI(%eax) 96 testl CPUVAR(IPENDING),%edx 97 jnz 2f 98 testl CPUVAR(IPENDING)+4,%eax 99 jnz 2f 100 movb %cl,CPUVAR(ILEVEL) 101 POPF(%eax) 102 1: 103 ret 104 2: 105 popf 106 jmp _C_LABEL(Xspllower) 107 .align 32 108 END(spllower) 109 #else /* XENPV */ 110 STRONG_ALIAS(spllower, cx8_spllower) 111 #endif /* !XENPV */ 112 113 /* 114 * void cx8_spllower(int s); 115 * 116 * spllower() optimized for Pentium Pro and later, which have long pipelines 117 * that will be stalled by pushf/cli/popf. Must be the same size as 118 * spllower(), ie 96 bytes. Does not need to restore eflags/IE as is patched 119 * in once autoconf is underway. 120 * 121 * For cmpxchg8b, edx/ecx are the high words and eax/ebx the low. 122 * 123 * edx : eax = old level + high 24 bit old ipending / low 32 bit old ipending 124 * ecx : ebx = new level + high 24 bit old ipending / low 32 bit old ipending 125 */ 126 ENTRY(cx8_spllower) 127 movl 4(%esp),%ecx 128 movzbl CPUVAR(ILEVEL),%edx 129 cmpl %edx,%ecx /* new level is lower? */ 130 jae 1f 131 pushl %ebx 132 pushl %esi 133 pushl %edi 134 movl %ecx,%esi 135 movl %ecx,%edi 136 shll $24,%edi 137 0: 138 movl CPUVAR(IPENDING),%eax 139 testl %eax,CPUVAR(IUNMASK)(,%esi,8) /* deferred interrupts? */ 140 jnz 2f 141 movl CPUVAR(IPENDING)+4,%edx 142 testl %edx,CPUVAR(IUNMASK)+4(,%esi,8) 143 jnz 2f 144 movl %eax,%ebx 145 movl %edx,%ecx 146 andl $0x00ffffff,%ecx 147 orl %edi,%ecx 148 cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */ 149 jnz 0b 150 popl %edi 151 popl %esi 152 popl %ebx 153 1: 154 ret 155 2: 156 popl %edi 157 popl %esi 158 popl %ebx 159 160 /* The reference must be absolute, hence the indirect jump. */ 161 movl $Xspllower,%eax 162 jmp *%eax 163 164 .align 32, 0xCC 165 LABEL(cx8_spllower_end) 166 END(cx8_spllower) 167 168 /* 169 * void Xspllower(int s); 170 * 171 * Process pending interrupts. 172 * 173 * Important registers: 174 * ebx - cpl 175 * esi - address to resume loop at 176 * edi - scratch for Xsoftnet 177 * 178 * It is important that the bit scan instruction is bsr, it will get 179 * the highest 2 bits (currently the IPI and clock handlers) first, 180 * to avoid deadlocks where one CPU sends an IPI, another one is at 181 * splhigh() and defers it, lands in here via splx(), and handles 182 * a lower-prio one first, which needs to take the kernel lock --> 183 * the sending CPU will never see the that CPU accept the IPI 184 * (see pmap_tlb_shootnow). 185 */ 186 nop /* Don't get confused with cx8_spllower_end */ 187 188 IDTVEC(spllower) 189 pushl %ebp 190 movl %esp,%ebp 191 MCOUNT_ASM 192 pushl %ebx 193 pushl %esi 194 pushl %edi 195 movl 8(%ebp),%ebx 196 movl $.Lspllower_resume,%esi /* address to resume loop at */ 197 1: 198 /* 199 * Because of the way Xen interrupts work *%esi will in fact be called 200 * from Xdoreti via iret. So we have to always disable interrupts here 201 * for Xen. 202 */ 203 #ifndef XENPV 204 CLI(%eax) 205 #endif 206 .Lspllower_resume: 207 #ifdef XENPV 208 CLI(%eax) 209 #endif 210 #if defined(DEBUG) 211 #ifndef XENPV 212 pushf 213 popl %eax 214 testl $PSL_I,%eax 215 jnz .Lspllower_panic 216 #else 217 movl CPUVAR(VCPU),%eax 218 movb EVTCHN_UPCALL_MASK(%eax),%al 219 andb %al,%al 220 jz .Lspllower_panic 221 #endif /* XENPV */ 222 #endif /* defined(DEBUG) */ 223 movl %ebx,%eax /* get cpl */ 224 movl CPUVAR(IUNMASK)+4(,%eax,8),%eax 225 andl CPUVAR(IPENDING)+4,%eax /* any non-masked bits left? */ 226 jz 10f 227 bsrl %eax,%eax 228 btrl %eax,CPUVAR(IPENDING)+4 229 addl $32,%eax 230 movl CPUVAR(ISOURCES)(,%eax,4),%eax 231 jmp *IS_RECURSE(%eax) 232 10: 233 movl %ebx,%eax /* get cpl */ 234 movl CPUVAR(IUNMASK)(,%eax,8),%eax 235 andl CPUVAR(IPENDING),%eax /* any non-masked bits left? */ 236 jz 2f 237 bsrl %eax,%eax 238 btrl %eax,CPUVAR(IPENDING) 239 movl CPUVAR(ISOURCES)(,%eax,4),%eax 240 jmp *IS_RECURSE(%eax) 241 2: 242 movb %bl,CPUVAR(ILEVEL) 243 #ifdef XENPV 244 STIC(%eax) 245 jz 4f 246 call _C_LABEL(stipending) 247 testl %eax,%eax 248 jnz 1b 249 4: 250 #else 251 STI(%eax) 252 #endif 253 popl %edi 254 popl %esi 255 popl %ebx 256 leave 257 ret 258 #if defined(DEBUG) 259 .Lspllower_panic: 260 pushl $1f 261 call _C_LABEL(panic) 262 1: .asciz "SPLLOWER: INTERRUPT ENABLED" 263 #endif 264 IDTVEC_END(spllower) 265 266 /* 267 * Xdoreti: Handle return from interrupt after device handler finishes. 268 * 269 * Important registers: 270 * ebx - cpl to restore 271 * esi - address to resume loop at 272 * edi - scratch for Xsoftnet 273 * 274 * called with interrupt disabled. 275 */ 276 IDTVEC(doreti) 277 IDEPTH_DECR 278 popl %ebx /* get previous priority */ 279 .Ldoreti_resume_stic: 280 movl $.Ldoreti_resume,%esi /* address to resume loop at */ 281 .Ldoreti_resume: 282 283 #if defined(DEBUG) 284 #ifndef XENPV 285 pushf 286 popl %eax 287 testl $PSL_I,%eax 288 jnz .Ldoreti_panic 289 #else 290 movl CPUVAR(VCPU),%eax 291 movb EVTCHN_UPCALL_MASK(%eax),%al 292 andb %al,%al 293 jz .Ldoreti_panic 294 #endif /* XENPV */ 295 #endif /* defined(DEBUG) */ 296 297 movl %ebx,%eax 298 movl CPUVAR(IUNMASK)+4(,%eax,8),%eax 299 andl CPUVAR(IPENDING)+4,%eax 300 jz 10f 301 bsrl %eax,%eax /* slow, but not worth optimizing */ 302 btrl %eax,CPUVAR(IPENDING)+4 303 addl $32,%eax 304 movl CPUVAR(ISOURCES)(,%eax, 4),%eax 305 jmp *IS_RESUME(%eax) 306 10: 307 movl %ebx,%eax 308 movl CPUVAR(IUNMASK)(,%eax,8),%eax 309 andl CPUVAR(IPENDING),%eax 310 jz 2f 311 bsrl %eax,%eax /* slow, but not worth optimizing */ 312 btrl %eax,CPUVAR(IPENDING) 313 movl CPUVAR(ISOURCES)(,%eax, 4),%eax 314 jmp *IS_RESUME(%eax) 315 2: /* Check for ASTs on exit to user mode. */ 316 movb %bl,CPUVAR(ILEVEL) 317 5: 318 testb $CHK_UPL,TF_CS(%esp) 319 jnz doreti_checkast 320 jmp 6f 321 322 .type _C_LABEL(doreti_checkast), @function 323 LABEL(doreti_checkast) 324 CHECK_ASTPENDING(%eax) 325 jz 3f 326 CLEAR_ASTPENDING(%eax) 327 STI(%eax) 328 movl $T_ASTFLT,TF_TRAPNO(%esp) /* XXX undo later.. */ 329 /* Pushed T_ASTFLT into tf_trapno on entry. */ 330 pushl %esp 331 call _C_LABEL(trap) 332 addl $4,%esp 333 CLI(%eax) 334 jmp 5b 335 END(doreti_checkast) 336 337 3: 338 CHECK_DEFERRED_SWITCH 339 jnz 9f 340 HANDLE_DEFERRED_FPU 341 6: 342 #ifdef XENPV 343 STIC(%eax) 344 jz 4f 345 call _C_LABEL(stipending) 346 testl %eax,%eax 347 jz 4f 348 CLI(%eax) 349 jmp .Ldoreti_resume_stic 350 4: 351 #endif 352 INTRFASTEXIT 353 9: 354 STI(%eax) 355 call _C_LABEL(pmap_load) 356 CLI(%eax) 357 jmp doreti_checkast /* recheck ASTs */ 358 359 #if defined(DEBUG) 360 .Ldoreti_panic: 361 pushl $1f 362 call _C_LABEL(panic) 363 1: .asciz "DORETI: INTERRUPT ENABLED" 364 #endif 365 IDTVEC_END(doreti) 366 367 /* 368 * Xsoftintr() 369 * 370 * Switch to the LWP assigned to handle interrupts from the given 371 * source. We borrow the VM context from the interrupted LWP. 372 * 373 * On entry: 374 * 375 * %eax intrsource 376 * %esi address to return to 377 */ 378 IDTVEC(softintr) 379 pushl $_C_LABEL(softintr_ret) /* set up struct switchframe */ 380 pushl %ebx 381 pushl %esi 382 pushl %edi 383 movb $IPL_HIGH,CPUVAR(ILEVEL) 384 STI(%esi) 385 movl CPUVAR(CURLWP),%esi 386 movl IS_LWP(%eax),%edi /* switch to handler LWP */ 387 /* 388 * Simple MOV to set curlwp to softlwp. See below on ordering 389 * required to restore softlwp like cpu_switchto. 390 * 391 * 1. Don't need store-before-store barrier because x86 is TSO. 392 * 393 * 2. Don't need store-before-load barrier because when we 394 * enter a softint lwp, it can't be holding any mutexes, so 395 * it can't release any until after it has acquired them, so 396 * we need not participate in the protocol with 397 * mutex_vector_enter barriers here. 398 * 399 * Hence no need for XCHG or barriers around MOV. 400 */ 401 movl %edi,CPUVAR(CURLWP) 402 movl L_PCB(%edi),%edx 403 movl L_PCB(%esi),%ecx 404 movl %esp,PCB_ESP(%ecx) 405 movl %ebp,PCB_EBP(%ecx) 406 movl PCB_ESP0(%edx),%esp /* onto new stack */ 407 pushl IS_MAXLEVEL(%eax) /* ipl to run at */ 408 pushl %esi 409 call _C_LABEL(softint_dispatch)/* run handlers */ 410 addl $8,%esp 411 CLI(%ecx) 412 movl L_PCB(%esi),%ecx 413 movl PCB_ESP(%ecx),%esp 414 415 /* 416 * Use XCHG, not MOV, to coordinate mutex_exit on this CPU with 417 * mutex_vector_enter on another CPU. 418 * 419 * 1. Any prior mutex_exit by the softint must be visible to 420 * other CPUs before we restore curlwp on this one, 421 * requiring store-before-store ordering. 422 * 423 * (This is always guaranteed by the x86 memory model, TSO, 424 * but other architectures require a explicit barrier before 425 * the store to ci->ci_curlwp.) 426 * 427 * 2. Restoring curlwp must be visible on all other CPUs before 428 * any subsequent mutex_exit on this one can even test 429 * whether there might be waiters, requiring 430 * store-before-load ordering. 431 * 432 * (This is the only ordering x86 TSO ever requires any kind 433 * of barrier for -- in this case, we take advantage of the 434 * sequential consistency implied by XCHG to obviate the 435 * need for MFENCE or something.) 436 * 437 * See kern_mutex.c for details -- this is necessary for 438 * adaptive mutexes to detect whether the lwp is on the CPU in 439 * order to safely block without requiring atomic r/m/w in 440 * mutex_exit. See also cpu_switchto. 441 */ 442 xchgl %esi,CPUVAR(CURLWP) /* restore ci_curlwp */ 443 popl %edi /* unwind switchframe */ 444 popl %esi 445 addl $8,%esp 446 jmp *%esi /* back to splx/doreti */ 447 IDTVEC_END(softintr) 448 449 /* 450 * softintr_ret() 451 * 452 * Trampoline function that gets returned to by cpu_switchto() when 453 * an interrupt handler blocks. On entry: 454 * 455 * %eax prevlwp from cpu_switchto() 456 */ 457 ENTRY(softintr_ret) 458 incl CPUVAR(MTX_COUNT) /* re-adjust after mi_switch */ 459 CLI(%eax) 460 jmp *%esi /* back to splx/doreti */ 461 END(softintr_ret) 462 463 /* 464 * void softint_trigger(uintptr_t machdep); 465 * 466 * Software interrupt registration. 467 */ 468 ENTRY(softint_trigger) 469 movl 4(%esp),%eax 470 orl %eax,CPUVAR(IPENDING) /* atomic on local cpu */ 471 ret 472 END(softint_trigger) 473 474 /* 475 * Xrecurse_preempt() 476 * 477 * Handles preemption interrupts via Xspllower(). 478 */ 479 IDTVEC(recurse_preempt) 480 movb $IPL_PREEMPT,CPUVAR(ILEVEL) 481 STI(%eax) 482 pushl $0 483 call _C_LABEL(kpreempt) 484 addl $4,%esp 485 CLI(%eax) 486 jmp *%esi 487 IDTVEC_END(recurse_preempt) 488 489 /* 490 * Xresume_preempt() 491 * 492 * Handles preemption interrupts via Xdoreti(). 493 */ 494 IDTVEC(resume_preempt) 495 movb $IPL_PREEMPT,CPUVAR(ILEVEL) 496 STI(%eax) 497 testb $CHK_UPL,TF_CS(%esp) 498 jnz 1f 499 movl TF_EIP(%esp),%eax 500 pushl %eax 501 call _C_LABEL(kpreempt) /* from kernel */ 502 addl $4,%esp 503 CLI(%eax) 504 jmp *%esi 505 1: 506 call _C_LABEL(preempt) /* from user */ 507 CLI(%eax) 508 jmp *%esi 509 IDTVEC_END(resume_preempt) 510
Indexes created Mon Sep 29 03:10:08 GMT 2025