1 /* $NetBSD: frameasm.h,v 1.55 2022/07/30 14:11:00 riastradh Exp $ */ 2 3 #ifndef _AMD64_MACHINE_FRAMEASM_H 4 #define _AMD64_MACHINE_FRAMEASM_H 5 6 #ifdef _KERNEL_OPT 7 #include "opt_xen.h" 8 #include "opt_svs.h" 9 #include "opt_kcov.h" 10 #include "opt_kmsan.h" 11 #endif 12 13 /* 14 * Macros to define pushing/popping frames for interrupts, traps 15 * and system calls. Currently all the same; will diverge later. 16 */ 17 18 #ifdef XENPV 19 #define HYPERVISOR_iret hypercall_page + (__HYPERVISOR_iret * 32) 20 /* Xen do not need swapgs, done by hypervisor */ 21 #define swapgs 22 #define iretq pushq $0 ; jmp HYPERVISOR_iret 23 #define XEN_ONLY2(x,y) x,y 24 #define NOT_XEN(x) 25 26 #define CLI(temp_reg) \ 27 movq CPUVAR(VCPU),%r ## temp_reg ; \ 28 movb $1,EVTCHN_UPCALL_MASK(%r ## temp_reg); 29 30 #define STI(temp_reg) \ 31 movq CPUVAR(VCPU),%r ## temp_reg ; \ 32 movb $0,EVTCHN_UPCALL_MASK(%r ## temp_reg); 33 34 #define PUSHF(temp_reg) \ 35 movq CPUVAR(VCPU),%r ## temp_reg ; \ 36 movzbl EVTCHN_UPCALL_MASK(%r ## temp_reg), %e ## temp_reg; \ 37 pushq %r ## temp_reg 38 39 #define POPF \ 40 popq %rdi; \ 41 call _C_LABEL(xen_write_psl) 42 43 44 #else /* XENPV */ 45 #define XEN_ONLY2(x,y) 46 #define NOT_XEN(x) x 47 #define CLI(temp_reg) cli 48 #define STI(temp_reg) sti 49 #define PUSHF(temp_reg) pushf 50 #define POPL popl 51 #endif /* XENPV */ 52 53 #define HP_NAME_CLAC 1 54 #define HP_NAME_STAC 2 55 #define HP_NAME_NOLOCK 3 56 #define HP_NAME_RETFENCE 4 57 #define HP_NAME_SVS_ENTER 5 58 #define HP_NAME_SVS_LEAVE 6 59 #define HP_NAME_SVS_ENTER_ALT 7 60 #define HP_NAME_SVS_LEAVE_ALT 8 61 #define HP_NAME_IBRS_ENTER 9 62 #define HP_NAME_IBRS_LEAVE 10 63 #define HP_NAME_SVS_ENTER_NMI 11 64 #define HP_NAME_SVS_LEAVE_NMI 12 65 #define HP_NAME_MDS_LEAVE 13 66 67 #define HOTPATCH(name, size) \ 68 123: ; \ 69 .pushsection .rodata.hotpatch, "a" ; \ 70 .byte name ; \ 71 .byte size ; \ 72 .quad 123b ; \ 73 .popsection 74 75 #define SMAP_ENABLE \ 76 HOTPATCH(HP_NAME_CLAC, 3) ; \ 77 .byte 0x0F, 0x1F, 0x00 ; \ 78 79 #define SMAP_DISABLE \ 80 HOTPATCH(HP_NAME_STAC, 3) ; \ 81 .byte 0x0F, 0x1F, 0x00 ; \ 82 83 /* 84 * IBRS 85 */ 86 87 #define IBRS_ENTER_BYTES 12 88 #define IBRS_ENTER \ 89 HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \ 90 NOIBRS_ENTER 91 #define NOIBRS_ENTER \ 92 .byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \ 93 .fill (IBRS_ENTER_BYTES-2),1,0xCC 94 95 #define IBRS_LEAVE_BYTES 12 96 #define IBRS_LEAVE \ 97 HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \ 98 NOIBRS_LEAVE 99 #define NOIBRS_LEAVE \ 100 .byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \ 101 .fill (IBRS_LEAVE_BYTES-2),1,0xCC 102 103 /* 104 * MDS 105 */ 106 107 #define MDS_LEAVE_BYTES 10 108 #define MDS_LEAVE \ 109 HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \ 110 NOMDS_LEAVE 111 #define NOMDS_LEAVE \ 112 .byte 0xEB, (MDS_LEAVE_BYTES-2) /* jmp */ ; \ 113 .fill (MDS_LEAVE_BYTES-2),1,0xCC 114 115 #define SWAPGS NOT_XEN(swapgs) 116 117 /* 118 * These are used on interrupt or trap entry or exit. 119 */ 120 #define INTR_SAVE_GPRS \ 121 movq %rdi,TF_RDI(%rsp) ; \ 122 movq %rsi,TF_RSI(%rsp) ; \ 123 movq %rdx,TF_RDX(%rsp) ; \ 124 movq %rcx,TF_RCX(%rsp) ; \ 125 movq %r8,TF_R8(%rsp) ; \ 126 movq %r9,TF_R9(%rsp) ; \ 127 movq %r10,TF_R10(%rsp) ; \ 128 movq %r11,TF_R11(%rsp) ; \ 129 movq %r12,TF_R12(%rsp) ; \ 130 movq %r13,TF_R13(%rsp) ; \ 131 movq %r14,TF_R14(%rsp) ; \ 132 movq %r15,TF_R15(%rsp) ; \ 133 movq %rbp,TF_RBP(%rsp) ; \ 134 movq %rbx,TF_RBX(%rsp) ; \ 135 movq %rax,TF_RAX(%rsp) 136 137 #define INTR_RESTORE_GPRS \ 138 movq TF_RDI(%rsp),%rdi ; \ 139 movq TF_RSI(%rsp),%rsi ; \ 140 movq TF_RDX(%rsp),%rdx ; \ 141 movq TF_RCX(%rsp),%rcx ; \ 142 movq TF_R8(%rsp),%r8 ; \ 143 movq TF_R9(%rsp),%r9 ; \ 144 movq TF_R10(%rsp),%r10 ; \ 145 movq TF_R11(%rsp),%r11 ; \ 146 movq TF_R12(%rsp),%r12 ; \ 147 movq TF_R13(%rsp),%r13 ; \ 148 movq TF_R14(%rsp),%r14 ; \ 149 movq TF_R15(%rsp),%r15 ; \ 150 movq TF_RBP(%rsp),%rbp ; \ 151 movq TF_RBX(%rsp),%rbx ; \ 152 movq TF_RAX(%rsp),%rax 153 154 #define TEXT_USER_BEGIN .pushsection .text.user, "ax" 155 #define TEXT_USER_END .popsection 156 157 #ifdef SVS 158 159 /* XXX: put this somewhere else */ 160 #define SVS_UTLS 0xffffff0000000000 /* PMAP_PCPU_BASE */ 161 #define UTLS_KPDIRPA 0 162 #define UTLS_SCRATCH 8 163 #define UTLS_RSP0 16 164 165 #define SVS_ENTER_BYTES 22 166 #define NOSVS_ENTER \ 167 .byte 0xEB, (SVS_ENTER_BYTES-2) /* jmp */ ; \ 168 .fill (SVS_ENTER_BYTES-2),1,0xCC 169 #define SVS_ENTER \ 170 HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \ 171 NOSVS_ENTER 172 173 #define SVS_LEAVE_BYTES 21 174 #define NOSVS_LEAVE \ 175 .byte 0xEB, (SVS_LEAVE_BYTES-2) /* jmp */ ; \ 176 .fill (SVS_LEAVE_BYTES-2),1,0xCC 177 #define SVS_LEAVE \ 178 HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \ 179 NOSVS_LEAVE 180 181 #define SVS_ENTER_ALT_BYTES 23 182 #define NOSVS_ENTER_ALTSTACK \ 183 .byte 0xEB, (SVS_ENTER_ALT_BYTES-2) /* jmp */ ; \ 184 .fill (SVS_ENTER_ALT_BYTES-2),1,0xCC 185 #define SVS_ENTER_ALTSTACK \ 186 HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \ 187 NOSVS_ENTER_ALTSTACK 188 189 #define SVS_LEAVE_ALT_BYTES 22 190 #define NOSVS_LEAVE_ALTSTACK \ 191 .byte 0xEB, (SVS_LEAVE_ALT_BYTES-2) /* jmp */ ; \ 192 .fill (SVS_LEAVE_ALT_BYTES-2),1,0xCC 193 #define SVS_LEAVE_ALTSTACK \ 194 HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \ 195 NOSVS_LEAVE_ALTSTACK 196 197 #define SVS_ENTER_NMI_BYTES 22 198 #define NOSVS_ENTER_NMI \ 199 .byte 0xEB, (SVS_ENTER_NMI_BYTES-2) /* jmp */ ; \ 200 .fill (SVS_ENTER_NMI_BYTES-2),1,0xCC 201 #define SVS_ENTER_NMI \ 202 HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES) ; \ 203 NOSVS_ENTER_NMI 204 205 #define SVS_LEAVE_NMI_BYTES 11 206 #define NOSVS_LEAVE_NMI \ 207 .byte 0xEB, (SVS_LEAVE_NMI_BYTES-2) /* jmp */ ; \ 208 .fill (SVS_LEAVE_NMI_BYTES-2),1,0xCC 209 #define SVS_LEAVE_NMI \ 210 HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES) ; \ 211 NOSVS_LEAVE_NMI 212 213 #else 214 #define SVS_ENTER /* nothing */ 215 #define SVS_ENTER_NMI /* nothing */ 216 #define SVS_LEAVE /* nothing */ 217 #define SVS_LEAVE_NMI /* nothing */ 218 #define SVS_ENTER_ALTSTACK /* nothing */ 219 #define SVS_LEAVE_ALTSTACK /* nothing */ 220 #endif 221 222 #ifdef KMSAN 223 /* XXX this belongs somewhere else. */ 224 #define KMSAN_ENTER \ 225 movq %rsp,%rdi ; \ 226 movq $TF_REGSIZE+16+40,%rsi ; \ 227 xorq %rdx,%rdx ; \ 228 callq kmsan_mark ; \ 229 callq kmsan_intr_enter 230 #define KMSAN_LEAVE \ 231 pushq %rbp ; \ 232 movq %rsp,%rbp ; \ 233 callq kmsan_intr_leave ; \ 234 popq %rbp 235 #define KMSAN_INIT_ARG(sz) \ 236 pushq %rax ; \ 237 pushq %rcx ; \ 238 pushq %rdx ; \ 239 pushq %rsi ; \ 240 pushq %rdi ; \ 241 pushq %r8 ; \ 242 pushq %r9 ; \ 243 pushq %r10 ; \ 244 pushq %r11 ; \ 245 movq $sz,%rdi ; \ 246 callq _C_LABEL(kmsan_init_arg); \ 247 popq %r11 ; \ 248 popq %r10 ; \ 249 popq %r9 ; \ 250 popq %r8 ; \ 251 popq %rdi ; \ 252 popq %rsi ; \ 253 popq %rdx ; \ 254 popq %rcx ; \ 255 popq %rax 256 #define KMSAN_INIT_RET(sz) \ 257 pushq %rax ; \ 258 pushq %rcx ; \ 259 pushq %rdx ; \ 260 pushq %rsi ; \ 261 pushq %rdi ; \ 262 pushq %r8 ; \ 263 pushq %r9 ; \ 264 pushq %r10 ; \ 265 pushq %r11 ; \ 266 movq $sz,%rdi ; \ 267 callq _C_LABEL(kmsan_init_ret); \ 268 popq %r11 ; \ 269 popq %r10 ; \ 270 popq %r9 ; \ 271 popq %r8 ; \ 272 popq %rdi ; \ 273 popq %rsi ; \ 274 popq %rdx ; \ 275 popq %rcx ; \ 276 popq %rax 277 #else 278 #define KMSAN_ENTER /* nothing */ 279 #define KMSAN_LEAVE /* nothing */ 280 #define KMSAN_INIT_ARG(sz) /* nothing */ 281 #define KMSAN_INIT_RET(sz) /* nothing */ 282 #endif 283 284 #ifdef KCOV 285 #define KCOV_DISABLE \ 286 incl CPUVAR(IDEPTH) 287 #define KCOV_ENABLE \ 288 decl CPUVAR(IDEPTH) 289 #else 290 #define KCOV_DISABLE /* nothing */ 291 #define KCOV_ENABLE /* nothing */ 292 #endif 293 294 #define INTRENTRY \ 295 subq $TF_REGSIZE,%rsp ; \ 296 INTR_SAVE_GPRS ; \ 297 cld ; \ 298 SMAP_ENABLE ; \ 299 testb $SEL_UPL,TF_CS(%rsp) ; \ 300 je 98f ; \ 301 SWAPGS ; \ 302 IBRS_ENTER ; \ 303 SVS_ENTER ; \ 304 movw %gs,TF_GS(%rsp) ; \ 305 movw %fs,TF_FS(%rsp) ; \ 306 movw %es,TF_ES(%rsp) ; \ 307 movw %ds,TF_DS(%rsp) ; \ 308 98: KMSAN_ENTER 309 310 #define INTRFASTEXIT \ 311 jmp intrfastexit 312 313 #define INTR_RECURSE_HWFRAME \ 314 movq %rsp,%r10 ; \ 315 movl %ss,%r11d ; \ 316 pushq %r11 ; \ 317 pushq %r10 ; \ 318 pushfq ; \ 319 pushq $GSEL(GCODE_SEL,SEL_KPL); \ 320 /* XEN: We must fixup CS, as even kernel mode runs at CPL 3 */ \ 321 XEN_ONLY2(andb $0xfc,(%rsp);) \ 322 pushq %r13 ; 323 324 #define INTR_RECURSE_ENTRY \ 325 subq $TF_REGSIZE,%rsp ; \ 326 INTR_SAVE_GPRS ; \ 327 cld ; \ 328 KMSAN_ENTER 329 330 #define CHECK_DEFERRED_SWITCH \ 331 cmpl $0, CPUVAR(WANT_PMAPLOAD) 332 333 #define CHECK_ASTPENDING(reg) cmpl $0, L_MD_ASTPENDING(reg) 334 #define CLEAR_ASTPENDING(reg) movl $0, L_MD_ASTPENDING(reg) 335 336 /* 337 * If the FPU state is not in the CPU, restore it. Executed with interrupts 338 * disabled. 339 * 340 * %r14 is curlwp, must not be modified 341 * %rbx must not be modified 342 */ 343 #define HANDLE_DEFERRED_FPU \ 344 testl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \ 345 jnz 1f ; \ 346 call _C_LABEL(fpu_handle_deferred) ; \ 347 orl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \ 348 1: 349 350 #endif /* _AMD64_MACHINE_FRAMEASM_H */ 351
Indexes created Sun Sep 28 16:09:52 GMT 2025