1 /* $NetBSD: sig_fpu.c,v 1.3 2026/07/15 02:08:18 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2026 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __RCSID("$NetBSD: sig_fpu.c,v 1.3 2026/07/15 02:08:18 riastradh Exp $"); 31 32 #include "sig_fpu.h" 33 34 #include <sys/types.h> 35 36 #include <x86/cpu_extended_state.h> 37 38 #include <cpuid.h> 39 #include <inttypes.h> 40 #include <sched.h> 41 #include <signal.h> 42 #include <stdbool.h> 43 #include <stdio.h> 44 45 #include "h_macros.h" 46 47 #ifdef __x86_64__ 48 # define NXMMREGS 16 49 # define NYMMREGS 16 50 # define NZMMREGS 32 51 #else /* 32-bit */ 52 # define NXMMREGS 8 53 # define NYMMREGS 8 54 # define NZMMREGS 8 55 #endif 56 57 #define X87_CW_RC __BITS(11,10) /* rounding control (mode) */ 58 #define X87_CW_PC __BITS(9,8) /* precision control */ 59 #define X87_CW_SW_EXC __BITS(5,0) /* exception bits */ 60 61 struct cpuid { 62 uint32_t edx; 63 uint32_t ecx; 64 uint32_t ebx; 65 uint32_t eax; 66 }; 67 68 static struct cpuid 69 cpuid(uint32_t eax_fn, uint32_t ecx_idx) 70 { 71 struct cpuid c; 72 73 if (!__get_cpuid_count(eax_fn, ecx_idx, 74 &c.eax, &c.ebx, &c.ecx, &c.edx)) 75 memset(&c, 0, sizeof(c)); 76 77 return c; 78 } 79 80 __printflike(4,5) 81 __noinline 82 static void 83 hexdump(FILE *fp, const void *buf, size_t len, const char *title, ...) 84 { 85 va_list va; 86 const uint8_t *p = buf; 87 size_t i; 88 89 fprintf(fp, "# "); 90 va_start(va, title); 91 vfprintf(fp, title, va); 92 va_end(va); 93 fprintf(fp, "\n"); 94 for (i = 0; i < len; i++) { 95 if ((i % 8) == 0) 96 fprintf(fp, " "); 97 fprintf(fp, " %02hhx", p[i]); 98 if ((i % 16) == 15) 99 fprintf(fp, "\n"); 100 } 101 if (i % 16) 102 fprintf(fp, "\n"); 103 } 104 105 bool 106 x87_supported(void) 107 { 108 #ifdef __x86_64__ 109 return true; 110 #else /* 32-bit */ 111 return cpuid(0x00000001, 0).edx & __BIT(0); 112 #endif 113 } 114 115 int 116 test_x87(volatile bool *ready, const volatile bool *done) 117 { 118 struct save87 s, s1; 119 uint32_t rcpc_exc; 120 int error = 0; 121 122 /* 123 * Gather the current FPU state so we have some reasonable 124 * content for the weird stuff when we restore from it. 125 */ 126 asm("wait\n" 127 " fnsave %0" 128 : /*out*/ "=m"(s)); 129 130 /* 131 * Randomize the floating-point stack entries ST(0),...,ST(7). 132 */ 133 arc4random_buf(s.s87_ac, sizeof(s.s87_ac)); 134 135 /* 136 * Randomize rounding control, precision control, and a set of 137 * exception bits. We don't want to raise any unmasked 138 * exceptions, so we'll raise _and_ mask them. 139 * 140 * XXX Consider testing the condition code, exception status, 141 * and top of stack bits in the status word too. 142 */ 143 rcpc_exc = arc4random() & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC); 144 if (__SHIFTOUT(rcpc_exc, X87_CW_PC) == 1) /* reserved */ 145 rcpc_exc |= __SHIFTIN(3, X87_CW_PC); /* extended */ 146 s.s87_cw &= ~(X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC); 147 s.s87_cw |= rcpc_exc; 148 s.s87_sw &= ~X87_CW_SW_EXC; 149 s.s87_sw |= rcpc_exc & X87_CW_SW_EXC; 150 151 /* 152 * Load up the registers, report that we're ready, busy-wait 153 * with the registers still live -- so no subroutine calls -- 154 * until we're done, and then store the registers back to 155 * memory so we can check them. 156 */ 157 asm("\n" 158 " frstor %[s]\n" 159 " lock\n" 160 " orb $1,%[ready]\n" 161 "0: pause\n" 162 " cmpb $0,%[done]\n" 163 " lfence\n" 164 " je 0b\n" 165 " wait\n" 166 " fnsave %[s1]\n" 167 : /*out*/ [ready]"=m"(*ready), [s1]"=m"(s1) 168 : /*in*/ [done]"m"(*done), [s]"m"(s)); 169 170 /* 171 * If there are any mismatches between the before and after 172 * ST(i) registers, or the control word, or the status word, 173 * print them and fail. 174 */ 175 if (memcmp(s.s87_ac, s1.s87_ac, sizeof(s.s87_ac)) != 0) { 176 unsigned i; 177 178 for (i = 0; i < 8; i++) { 179 if (memcmp(&s.s87_ac[i], &s1.s87_ac[i], 180 sizeof(s.s87_ac[i])) == 0) 181 continue; 182 fprintf(stderr, "ST(%u)=0x%08x%016"PRIx64"," 183 " expected 0x%08x%016"PRIx64"\n", 184 i, 185 s1.s87_ac[i].f87_exp_sign, 186 s1.s87_ac[i].f87_mantissa, 187 s.s87_ac[i].f87_exp_sign, 188 s.s87_ac[i].f87_mantissa); 189 error |= 1 << i; 190 } 191 } 192 if ((s1.s87_cw & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC)) != 193 (s.s87_cw & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC))) { 194 fprintf(stderr, "cw=0x%04x, expected 0x%04x," 195 " diff 0x%04x\n", 196 s1.s87_cw, s.s87_cw, 197 (uint32_t)((s1.s87_cw ^ s.s87_cw) & 198 (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC))); 199 error |= 1 << 8; 200 } 201 if ((s1.s87_sw & X87_CW_SW_EXC) != 202 (s.s87_cw & X87_CW_SW_EXC)) { 203 fprintf(stderr, "sw=0x%04x, expected 0x%04x," 204 " diff 0x%04x\n", 205 s1.s87_sw, s.s87_sw, 206 (uint32_t)((s1.s87_sw ^ s.s87_sw) & 207 X87_CW_SW_EXC)); 208 error |= 1 << 9; 209 } 210 211 return error; 212 } 213 214 void 215 trash_x87(void) 216 { 217 struct save87 s; 218 uint32_t rcpc_exc; 219 220 /* 221 * Gather the current FPU state so we have some reasonable 222 * content for the weird stuff when we restore from it. 223 */ 224 asm("wait\n" 225 " fnsave %0" 226 : /*out*/ "=m"(s)); 227 228 /* 229 * Randomize the floating-point stack entries ST(0),...,ST(7). 230 */ 231 arc4random_buf(s.s87_ac, sizeof(s.s87_ac)); 232 233 /* 234 * Randomize rounding control, precision control, and a set of 235 * exception bits. We don't want to raise any unmasked 236 * exceptions, so we'll raise _and_ mask them. 237 * 238 * XXX Consider trashing the condition code, exception status, 239 * and top of stack bits in the status word too. 240 */ 241 rcpc_exc = arc4random() & (X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC); 242 s.s87_cw &= ~(X87_CW_RC | X87_CW_PC | X87_CW_SW_EXC); 243 s.s87_cw |= rcpc_exc; 244 s.s87_sw &= ~X87_CW_SW_EXC; 245 s.s87_sw |= rcpc_exc & X87_CW_SW_EXC; 246 247 /* 248 * Load the state into the FPU. 249 */ 250 asm volatile("frstor %0" :: /*in*/ "m"(s)); 251 } 252 253 #define MXCSR_MISALIGNEDEXC __BIT(17) /* misaligned exc fault */ 254 #define MXCSR_FTZ __BIT(15) /* flush to zero */ 255 #define MXCSR_RC __BITS(14,13) /* rounding control */ 256 #define MXCSR_EXCMASK __BITS(12,7) /* exceptions masked */ 257 #define MXCSR_DAZ __BIT(6) /* denormals are zero */ 258 #define MXCSR_EXCFLAG __BITS(5,0) /* exceptions raised */ 259 260 struct xmmregs { 261 struct { 262 uint8_t b[16]; 263 } __aligned(16) xmm[NXMMREGS]; 264 uint32_t mxcsr; 265 }; 266 267 bool 268 xmm_supported(void) 269 { 270 #ifdef __x86_64__ 271 return true; 272 #else /* 32-bit */ 273 return cpuid(0x00000001, 0).edx & __BIT(25); 274 #endif 275 } 276 277 __attribute__((target("sse"))) 278 int 279 test_xmm(volatile bool *ready, const volatile bool *done) 280 { 281 struct xmmregs before, after; 282 uint32_t exc; 283 unsigned i; 284 int error = 0; 285 286 /* 287 * Randomize the xmm register content. Randomize plausible 288 * bits for the mxcsr. 289 */ 290 arc4random_buf(before.xmm, sizeof(before.xmm)); 291 before.mxcsr = arc4random() & (MXCSR_FTZ | MXCSR_RC | MXCSR_DAZ); 292 exc = arc4random() & 0x3f; 293 before.mxcsr |= __SHIFTIN(exc, MXCSR_EXCMASK); 294 before.mxcsr |= __SHIFTIN(exc, MXCSR_EXCFLAG); 295 296 /* 297 * Load up the registers, report that we're ready, busy-wait 298 * with the registers still live -- so no subroutine calls -- 299 * until we're done, and then store the registers back to 300 * memory so we can check them. 301 */ 302 asm("\n" 303 " ldmxcsr %[mxcsr_before]\n" 304 " movdqa 0*16(%[xmm_before]),%%xmm0\n" 305 " movdqa 1*16(%[xmm_before]),%%xmm1\n" 306 " movdqa 2*16(%[xmm_before]),%%xmm2\n" 307 " movdqa 3*16(%[xmm_before]),%%xmm3\n" 308 " movdqa 4*16(%[xmm_before]),%%xmm4\n" 309 " movdqa 5*16(%[xmm_before]),%%xmm5\n" 310 " movdqa 6*16(%[xmm_before]),%%xmm6\n" 311 " movdqa 7*16(%[xmm_before]),%%xmm7\n" 312 #ifdef __x86_64__ 313 " movdqa 8*16(%[xmm_before]),%%xmm8\n" 314 " movdqa 9*16(%[xmm_before]),%%xmm9\n" 315 " movdqa 10*16(%[xmm_before]),%%xmm10\n" 316 " movdqa 11*16(%[xmm_before]),%%xmm11\n" 317 " movdqa 12*16(%[xmm_before]),%%xmm12\n" 318 " movdqa 13*16(%[xmm_before]),%%xmm13\n" 319 " movdqa 14*16(%[xmm_before]),%%xmm14\n" 320 " movdqa 15*16(%[xmm_before]),%%xmm15\n" 321 #endif 322 " lock\n" 323 " orb $1,%[ready]\n" 324 "0: pause\n" 325 " cmpb $0,%[done]\n" 326 " lfence\n" 327 " je 0b\n" 328 " movdqa %%xmm0,0*16(%[xmm_after])\n" 329 " movdqa %%xmm1,1*16(%[xmm_after])\n" 330 " movdqa %%xmm2,2*16(%[xmm_after])\n" 331 " movdqa %%xmm3,3*16(%[xmm_after])\n" 332 " movdqa %%xmm4,4*16(%[xmm_after])\n" 333 " movdqa %%xmm5,5*16(%[xmm_after])\n" 334 " movdqa %%xmm6,6*16(%[xmm_after])\n" 335 " movdqa %%xmm7,7*16(%[xmm_after])\n" 336 #ifdef __x86_64__ 337 " movdqa %%xmm8,8*16(%[xmm_after])\n" 338 " movdqa %%xmm9,9*16(%[xmm_after])\n" 339 " movdqa %%xmm10,10*16(%[xmm_after])\n" 340 " movdqa %%xmm11,11*16(%[xmm_after])\n" 341 " movdqa %%xmm12,12*16(%[xmm_after])\n" 342 " movdqa %%xmm13,13*16(%[xmm_after])\n" 343 " movdqa %%xmm14,14*16(%[xmm_after])\n" 344 " movdqa %%xmm15,15*16(%[xmm_after])\n" 345 #endif 346 " stmxcsr %[mxcsr_after]\n" 347 : /*out*/ [ready]"=m"(*ready), "=m"(after.xmm), 348 [mxcsr_after]"=m"(after.mxcsr) 349 : /*in*/ [done]"m"(*done), "m"(before.xmm), 350 [mxcsr_before]"m"(before.mxcsr), 351 [xmm_before]"r"(&before.xmm), [xmm_after]"r"(&after.xmm) 352 : /*clobber*/ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", 353 "xmm6", "xmm7" 354 #ifdef __x86_64__ 355 , "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", 356 "xmm13", "xmm14", "xmm15" 357 #endif 358 ); 359 360 /* 361 * If there are any mismatches between the before and after xmm 362 * registers, or the mxcsr, print them and fail. 363 */ 364 for (i = 0; i < __arraycount(before.xmm); i++) { 365 if (memcmp(&before.xmm[i], &after.xmm[i], 366 sizeof(before.xmm[i]))) { 367 fprintf(stderr, "xmm%u clobbered\n", i); 368 hexdump(stderr, &before.xmm[i], sizeof(before.xmm[i]), 369 "before"); 370 hexdump(stderr, &after.xmm[i], sizeof(after.xmm[i]), 371 "after"); 372 error |= 1 << i; 373 } 374 } 375 376 if (before.mxcsr != after.mxcsr) { 377 fprintf(stderr, "mxcsr clobbered:" 378 " before=0x%08"PRIx32", after=0x%08"PRIx32"\n", 379 before.mxcsr, after.mxcsr); 380 error |= 1 << 16; 381 } 382 383 return error; 384 } 385 386 __attribute__((target("sse"))) 387 void 388 trash_xmm(void) 389 { 390 struct xmmregs regs; 391 392 arc4random_buf(®s, sizeof(regs)); 393 394 asm("\n" 395 " movdqa 0*16(%[xmm]),%%xmm0\n" 396 " movdqa 1*16(%[xmm]),%%xmm1\n" 397 " movdqa 2*16(%[xmm]),%%xmm2\n" 398 " movdqa 3*16(%[xmm]),%%xmm3\n" 399 " movdqa 4*16(%[xmm]),%%xmm4\n" 400 " movdqa 5*16(%[xmm]),%%xmm5\n" 401 " movdqa 6*16(%[xmm]),%%xmm6\n" 402 " movdqa 7*16(%[xmm]),%%xmm7\n" 403 #ifdef __x86_64__ 404 " movdqa 8*16(%[xmm]),%%xmm8\n" 405 " movdqa 9*16(%[xmm]),%%xmm9\n" 406 " movdqa 10*16(%[xmm]),%%xmm10\n" 407 " movdqa 11*16(%[xmm]),%%xmm11\n" 408 " movdqa 12*16(%[xmm]),%%xmm12\n" 409 " movdqa 13*16(%[xmm]),%%xmm13\n" 410 " movdqa 14*16(%[xmm]),%%xmm14\n" 411 " movdqa 15*16(%[xmm]),%%xmm15\n" 412 #endif 413 : /*out*/ 414 : /*in*/ [xmm]"r"(regs.xmm), "m"(regs.xmm) 415 : /*clobber*/ "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", 416 "xmm6", "xmm7" 417 #ifdef __x86_64__ 418 , "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", 419 "xmm13", "xmm14", "xmm15" 420 #endif 421 ); 422 } 423 424 bool 425 ymm_supported(void) 426 { 427 return cpuid(0x00000001, 0).ecx & __BIT(28); 428 } 429 430 struct ymmregs { 431 struct { 432 uint8_t b[32]; 433 } __aligned(32) ymm[NYMMREGS]; 434 }; 435 436 __attribute__((target("avx"))) 437 int 438 test_ymm(volatile bool *ready, const volatile bool *done) 439 { 440 struct ymmregs before, after; 441 unsigned i; 442 int error = 0; 443 444 /* 445 * Randomize the ymm register content. 446 */ 447 arc4random_buf(&before, sizeof(before)); 448 449 /* 450 * Load up the registers, report that we're ready, busy-wait 451 * with the registers still live -- so no subroutine calls -- 452 * until we're done, and then store the registers back to 453 * memory so we can check them. 454 */ 455 asm("\n" 456 " vmovdqa 0*32(%[before_ymm]),%%ymm0\n" 457 " vmovdqa 1*32(%[before_ymm]),%%ymm1\n" 458 " vmovdqa 2*32(%[before_ymm]),%%ymm2\n" 459 " vmovdqa 3*32(%[before_ymm]),%%ymm3\n" 460 " vmovdqa 4*32(%[before_ymm]),%%ymm4\n" 461 " vmovdqa 5*32(%[before_ymm]),%%ymm5\n" 462 " vmovdqa 6*32(%[before_ymm]),%%ymm6\n" 463 " vmovdqa 7*32(%[before_ymm]),%%ymm7\n" 464 #ifdef __x86_64__ 465 " vmovdqa 8*32(%[before_ymm]),%%ymm8\n" 466 " vmovdqa 9*32(%[before_ymm]),%%ymm9\n" 467 " vmovdqa 10*32(%[before_ymm]),%%ymm10\n" 468 " vmovdqa 11*32(%[before_ymm]),%%ymm11\n" 469 " vmovdqa 12*32(%[before_ymm]),%%ymm12\n" 470 " vmovdqa 13*32(%[before_ymm]),%%ymm13\n" 471 " vmovdqa 14*32(%[before_ymm]),%%ymm14\n" 472 " vmovdqa 15*32(%[before_ymm]),%%ymm15\n" 473 #endif 474 " lock\n" 475 " orb $1,%[ready]\n" 476 "0: pause\n" 477 " cmpb $0,%[done]\n" 478 " lfence\n" 479 " je 0b\n" 480 " vmovdqa %%ymm0,0*32(%[after_ymm])\n" 481 " vmovdqa %%ymm1,1*32(%[after_ymm])\n" 482 " vmovdqa %%ymm2,2*32(%[after_ymm])\n" 483 " vmovdqa %%ymm3,3*32(%[after_ymm])\n" 484 " vmovdqa %%ymm4,4*32(%[after_ymm])\n" 485 " vmovdqa %%ymm5,5*32(%[after_ymm])\n" 486 " vmovdqa %%ymm6,6*32(%[after_ymm])\n" 487 " vmovdqa %%ymm7,7*32(%[after_ymm])\n" 488 #ifdef __x86_64__ 489 " vmovdqa %%ymm8,8*32(%[after_ymm])\n" 490 " vmovdqa %%ymm9,9*32(%[after_ymm])\n" 491 " vmovdqa %%ymm10,10*32(%[after_ymm])\n" 492 " vmovdqa %%ymm11,11*32(%[after_ymm])\n" 493 " vmovdqa %%ymm12,12*32(%[after_ymm])\n" 494 " vmovdqa %%ymm13,13*32(%[after_ymm])\n" 495 " vmovdqa %%ymm14,14*32(%[after_ymm])\n" 496 " vmovdqa %%ymm15,15*32(%[after_ymm])\n" 497 #endif 498 : /*out*/ [ready]"=m"(*ready), "=m"(after.ymm) 499 : /*in*/ [done]"m"(*done), "m"(before.ymm), 500 [before_ymm]"r"(before.ymm), [after_ymm]"r"(after.ymm) 501 : /*clobber*/ "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", 502 "ymm6", "ymm7" 503 #ifdef __x86_64__ 504 , "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", 505 "ymm13", "ymm14", "ymm15" 506 #endif 507 ); 508 509 for (i = 0; i < __arraycount(before.ymm); i++) { 510 if (memcmp(&before.ymm[i], &after.ymm[i], 511 sizeof(before.ymm[i]))) { 512 fprintf(stderr, "ymm%u clobbered\n", i); 513 hexdump(stderr, &before.ymm[i], sizeof(before.ymm[i]), 514 "before"); 515 hexdump(stderr, &after.ymm[i], sizeof(after.ymm[i]), 516 "after"); 517 error |= 1 << i; 518 } 519 } 520 521 return error; 522 } 523 524 __attribute__((target("avx"))) 525 void 526 trash_ymm(void) 527 { 528 struct ymmregs regs; 529 530 arc4random_buf(®s, sizeof(regs)); 531 532 asm("\n" 533 " vmovdqa 0*32(%[ymm]),%%ymm0\n" 534 " vmovdqa 1*32(%[ymm]),%%ymm1\n" 535 " vmovdqa 2*32(%[ymm]),%%ymm2\n" 536 " vmovdqa 3*32(%[ymm]),%%ymm3\n" 537 " vmovdqa 4*32(%[ymm]),%%ymm4\n" 538 " vmovdqa 5*32(%[ymm]),%%ymm5\n" 539 " vmovdqa 6*32(%[ymm]),%%ymm6\n" 540 " vmovdqa 7*32(%[ymm]),%%ymm7\n" 541 #ifdef __x86_64__ 542 " vmovdqa 8*32(%[ymm]),%%ymm8\n" 543 " vmovdqa 9*32(%[ymm]),%%ymm9\n" 544 " vmovdqa 10*32(%[ymm]),%%ymm10\n" 545 " vmovdqa 11*32(%[ymm]),%%ymm11\n" 546 " vmovdqa 12*32(%[ymm]),%%ymm12\n" 547 " vmovdqa 13*32(%[ymm]),%%ymm13\n" 548 " vmovdqa 14*32(%[ymm]),%%ymm14\n" 549 " vmovdqa 15*32(%[ymm]),%%ymm15\n" 550 #endif 551 : /*out*/ 552 : /*in*/ "m"(regs.ymm), [ymm]"r"(regs.ymm) 553 : /*clobber*/ "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", 554 "ymm6", "ymm7" 555 #ifdef __x86_64__ 556 , "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", 557 "ymm13", "ymm14", "ymm15" 558 #endif 559 ); 560 } 561 562 bool 563 zmm_supported(void) 564 { 565 return cpuid(0x00000007, 0).ebx & __BIT(16); 566 } 567 568 struct zmmregs { 569 struct { 570 uint8_t b[64]; 571 } __aligned(64) zmm[NZMMREGS]; 572 }; 573 574 __attribute__((target("avx512f"))) 575 int 576 test_zmm(volatile bool *ready, const volatile bool *done) 577 { 578 struct zmmregs before, after; 579 unsigned i; 580 int error = 0; 581 582 /* 583 * Randomize the zmm register content. 584 */ 585 arc4random_buf(&before, sizeof(before)); 586 587 /* 588 * Load up the registers, report that we're ready, busy-wait 589 * with the registers still live -- so no subroutine calls -- 590 * until we're done, and then store the registers back to 591 * memory so we can check them. 592 */ 593 asm("\n" 594 " vmovdqa64 0*64(%[before_zmm]),%%zmm0\n" 595 " vmovdqa64 1*64(%[before_zmm]),%%zmm1\n" 596 " vmovdqa64 2*64(%[before_zmm]),%%zmm2\n" 597 " vmovdqa64 3*64(%[before_zmm]),%%zmm3\n" 598 " vmovdqa64 4*64(%[before_zmm]),%%zmm4\n" 599 " vmovdqa64 5*64(%[before_zmm]),%%zmm5\n" 600 " vmovdqa64 6*64(%[before_zmm]),%%zmm6\n" 601 " vmovdqa64 7*64(%[before_zmm]),%%zmm7\n" 602 #ifdef __x86_64__ 603 " vmovdqa64 8*64(%[before_zmm]),%%zmm8\n" 604 " vmovdqa64 9*64(%[before_zmm]),%%zmm9\n" 605 " vmovdqa64 10*64(%[before_zmm]),%%zmm10\n" 606 " vmovdqa64 11*64(%[before_zmm]),%%zmm11\n" 607 " vmovdqa64 12*64(%[before_zmm]),%%zmm12\n" 608 " vmovdqa64 13*64(%[before_zmm]),%%zmm13\n" 609 " vmovdqa64 14*64(%[before_zmm]),%%zmm14\n" 610 " vmovdqa64 15*64(%[before_zmm]),%%zmm15\n" 611 " vmovdqa64 16*64(%[before_zmm]),%%zmm16\n" 612 " vmovdqa64 17*64(%[before_zmm]),%%zmm17\n" 613 " vmovdqa64 18*64(%[before_zmm]),%%zmm18\n" 614 " vmovdqa64 19*64(%[before_zmm]),%%zmm19\n" 615 " vmovdqa64 20*64(%[before_zmm]),%%zmm20\n" 616 " vmovdqa64 21*64(%[before_zmm]),%%zmm21\n" 617 " vmovdqa64 22*64(%[before_zmm]),%%zmm22\n" 618 " vmovdqa64 23*64(%[before_zmm]),%%zmm23\n" 619 " vmovdqa64 24*64(%[before_zmm]),%%zmm24\n" 620 " vmovdqa64 25*64(%[before_zmm]),%%zmm25\n" 621 " vmovdqa64 26*64(%[before_zmm]),%%zmm26\n" 622 " vmovdqa64 27*64(%[before_zmm]),%%zmm27\n" 623 " vmovdqa64 28*64(%[before_zmm]),%%zmm28\n" 624 " vmovdqa64 29*64(%[before_zmm]),%%zmm29\n" 625 " vmovdqa64 30*64(%[before_zmm]),%%zmm30\n" 626 " vmovdqa64 31*64(%[before_zmm]),%%zmm31\n" 627 #endif 628 " lock\n" 629 " orb $1,%[ready]\n" 630 "0: pause\n" 631 " cmpb $0,%[done]\n" 632 " lfence\n" 633 " je 0b\n" 634 " vmovdqa64 %%zmm0,0*64(%[after_zmm])\n" 635 " vmovdqa64 %%zmm1,1*64(%[after_zmm])\n" 636 " vmovdqa64 %%zmm2,2*64(%[after_zmm])\n" 637 " vmovdqa64 %%zmm3,3*64(%[after_zmm])\n" 638 " vmovdqa64 %%zmm4,4*64(%[after_zmm])\n" 639 " vmovdqa64 %%zmm5,5*64(%[after_zmm])\n" 640 " vmovdqa64 %%zmm6,6*64(%[after_zmm])\n" 641 " vmovdqa64 %%zmm7,7*64(%[after_zmm])\n" 642 #ifdef __x86_64__ 643 " vmovdqa64 %%zmm8,8*64(%[after_zmm])\n" 644 " vmovdqa64 %%zmm9,9*64(%[after_zmm])\n" 645 " vmovdqa64 %%zmm10,10*64(%[after_zmm])\n" 646 " vmovdqa64 %%zmm11,11*64(%[after_zmm])\n" 647 " vmovdqa64 %%zmm12,12*64(%[after_zmm])\n" 648 " vmovdqa64 %%zmm13,13*64(%[after_zmm])\n" 649 " vmovdqa64 %%zmm14,14*64(%[after_zmm])\n" 650 " vmovdqa64 %%zmm15,15*64(%[after_zmm])\n" 651 " vmovdqa64 %%zmm16,16*64(%[after_zmm])\n" 652 " vmovdqa64 %%zmm17,17*64(%[after_zmm])\n" 653 " vmovdqa64 %%zmm18,18*64(%[after_zmm])\n" 654 " vmovdqa64 %%zmm19,19*64(%[after_zmm])\n" 655 " vmovdqa64 %%zmm20,20*64(%[after_zmm])\n" 656 " vmovdqa64 %%zmm21,21*64(%[after_zmm])\n" 657 " vmovdqa64 %%zmm22,22*64(%[after_zmm])\n" 658 " vmovdqa64 %%zmm23,23*64(%[after_zmm])\n" 659 " vmovdqa64 %%zmm24,24*64(%[after_zmm])\n" 660 " vmovdqa64 %%zmm25,25*64(%[after_zmm])\n" 661 " vmovdqa64 %%zmm26,26*64(%[after_zmm])\n" 662 " vmovdqa64 %%zmm27,27*64(%[after_zmm])\n" 663 " vmovdqa64 %%zmm28,28*64(%[after_zmm])\n" 664 " vmovdqa64 %%zmm29,29*64(%[after_zmm])\n" 665 " vmovdqa64 %%zmm30,30*64(%[after_zmm])\n" 666 " vmovdqa64 %%zmm31,31*64(%[after_zmm])\n" 667 #endif 668 : /*out*/ [ready]"=m"(*ready), "=m"(after.zmm) 669 : /*in*/ [done]"m"(*done), "m"(before.zmm), 670 [before_zmm]"r"(before.zmm), [after_zmm]"r"(after.zmm) 671 : /*clobber*/ "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", 672 "zmm6", "zmm7" 673 #ifdef __x86_64__ 674 , "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", 675 "zmm13", "zmm14", "zmm15", "zmm16", "zmm17", 676 "zmm18", "zmm19", "zmm20", "zmm21", "zmm22", 677 "zmm23", "zmm24", "zmm25", "zmm26", "zmm27", 678 "zmm28", "zmm29", "zmm30", "zmm31" 679 #endif 680 ); 681 682 for (i = 0; i < __arraycount(before.zmm); i++) { 683 if (memcmp(&before.zmm[i], &after.zmm[i], 684 sizeof(before.zmm[i]))) { 685 fprintf(stderr, "zmm%u clobbered\n", i); 686 hexdump(stderr, &before.zmm[i], sizeof(before.zmm[i]), 687 "before"); 688 hexdump(stderr, &after.zmm[i], sizeof(after.zmm[i]), 689 "after"); 690 error |= 1 << i; 691 } 692 } 693 694 return error; 695 } 696 697 __attribute__((target("avx512f"))) 698 void 699 trash_zmm(void) 700 { 701 struct zmmregs regs; 702 703 arc4random_buf(®s, sizeof(regs)); 704 705 asm("\n" 706 " vmovdqa64 0*64(%[zmm]),%%zmm0\n" 707 " vmovdqa64 1*64(%[zmm]),%%zmm1\n" 708 " vmovdqa64 2*64(%[zmm]),%%zmm2\n" 709 " vmovdqa64 3*64(%[zmm]),%%zmm3\n" 710 " vmovdqa64 4*64(%[zmm]),%%zmm4\n" 711 " vmovdqa64 5*64(%[zmm]),%%zmm5\n" 712 " vmovdqa64 6*64(%[zmm]),%%zmm6\n" 713 " vmovdqa64 7*64(%[zmm]),%%zmm7\n" 714 #ifdef __x86_64__ 715 " vmovdqa64 8*64(%[zmm]),%%zmm8\n" 716 " vmovdqa64 9*64(%[zmm]),%%zmm9\n" 717 " vmovdqa64 10*64(%[zmm]),%%zmm10\n" 718 " vmovdqa64 11*64(%[zmm]),%%zmm11\n" 719 " vmovdqa64 12*64(%[zmm]),%%zmm12\n" 720 " vmovdqa64 13*64(%[zmm]),%%zmm13\n" 721 " vmovdqa64 14*64(%[zmm]),%%zmm14\n" 722 " vmovdqa64 15*64(%[zmm]),%%zmm15\n" 723 " vmovdqa64 16*64(%[zmm]),%%zmm16\n" 724 " vmovdqa64 17*64(%[zmm]),%%zmm17\n" 725 " vmovdqa64 18*64(%[zmm]),%%zmm18\n" 726 " vmovdqa64 19*64(%[zmm]),%%zmm19\n" 727 " vmovdqa64 20*64(%[zmm]),%%zmm20\n" 728 " vmovdqa64 21*64(%[zmm]),%%zmm21\n" 729 " vmovdqa64 22*64(%[zmm]),%%zmm22\n" 730 " vmovdqa64 23*64(%[zmm]),%%zmm23\n" 731 " vmovdqa64 24*64(%[zmm]),%%zmm24\n" 732 " vmovdqa64 25*64(%[zmm]),%%zmm25\n" 733 " vmovdqa64 26*64(%[zmm]),%%zmm26\n" 734 " vmovdqa64 27*64(%[zmm]),%%zmm27\n" 735 " vmovdqa64 28*64(%[zmm]),%%zmm28\n" 736 " vmovdqa64 29*64(%[zmm]),%%zmm29\n" 737 " vmovdqa64 30*64(%[zmm]),%%zmm30\n" 738 " vmovdqa64 31*64(%[zmm]),%%zmm31\n" 739 #endif 740 : /*out*/ 741 : /*in*/ "m"(regs.zmm), [zmm]"r"(regs.zmm) 742 : /*clobber*/ "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", 743 "zmm6", "zmm7" 744 #ifdef __x86_64__ 745 , "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", 746 "zmm13", "zmm14", "zmm15", "zmm16", "zmm17", 747 "zmm18", "zmm19", "zmm20", "zmm21", "zmm22", 748 "zmm23", "zmm24", "zmm25", "zmm26", "zmm27", 749 "zmm28", "zmm29", "zmm30", "zmm31" 750 #endif 751 ); 752 } 753
Indexes created Thu Jul 16 00:26:27 UTC 2026