subr_msan.c revision 1.16
1 /* $NetBSD: subr_msan.c,v 1.16 2021/09/07 16:56:13 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KMSAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.16 2021/09/07 16:56:13 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/kprintf.h> 42 #include <sys/kmem.h> 43 #include <sys/mbuf.h> 44 #include <sys/buf.h> 45 #include <sys/cpu.h> 46 #include <sys/msan.h> 47 48 static void kmsan_printf(const char *, ...); 49 50 void kmsan_init_arg(size_t); 51 void kmsan_init_ret(size_t); 52 53 #ifdef KMSAN_PANIC 54 #define REPORT panic 55 #else 56 #define REPORT kmsan_printf 57 #endif 58 59 /* -------------------------------------------------------------------------- */ 60 61 /* 62 * Part of the compiler ABI. 63 */ 64 65 typedef uint32_t msan_orig_t; 66 67 typedef struct { 68 uint8_t *shad; 69 msan_orig_t *orig; 70 } msan_meta_t; 71 72 #define MSAN_PARAM_SIZE 800 73 #define MSAN_RETVAL_SIZE 800 74 typedef struct { 75 uint8_t param[MSAN_PARAM_SIZE]; 76 uint8_t retval[MSAN_RETVAL_SIZE]; 77 uint8_t _va_arg[MSAN_PARAM_SIZE]; 78 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 79 uint64_t va_arg_overflow_size; 80 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 81 msan_orig_t retval_origin; 82 msan_orig_t origin; 83 } msan_tls_t; 84 85 /* -------------------------------------------------------------------------- */ 86 87 /* The MD code. */ 88 #include <machine/msan.h> 89 90 /* -------------------------------------------------------------------------- */ 91 92 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 93 #define MSAN_NCONTEXT 16 94 95 typedef struct { 96 size_t ctx; 97 msan_tls_t tls[MSAN_NCONTEXT]; 98 } msan_lwp_t; 99 100 static msan_tls_t dummy_tls; 101 102 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 103 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 104 static msan_lwp_t msan_lwp0; 105 static bool kmsan_enabled __read_mostly; 106 107 /* -------------------------------------------------------------------------- */ 108 109 static bool kmsan_reporting = false; 110 111 static inline void 112 kmsan_printf(const char *fmt, ...) 113 { 114 va_list ap; 115 116 va_start(ap, fmt); 117 kprintf(fmt, TOCONS, NULL, NULL, ap); 118 va_end(ap); 119 } 120 121 static inline const char * 122 kmsan_orig_name(int type) 123 { 124 switch (type) { 125 case KMSAN_TYPE_STACK: 126 return "Stack"; 127 case KMSAN_TYPE_KMEM: 128 return "Kmem"; 129 case KMSAN_TYPE_MALLOC: 130 return "Malloc"; 131 case KMSAN_TYPE_POOL: 132 return "Pool"; 133 case KMSAN_TYPE_UVM: 134 return "Uvm"; 135 default: 136 return "Unknown"; 137 } 138 } 139 140 /* 141 * The format of the string is: "----var@function". Parse it to display a nice 142 * warning. 143 */ 144 static void 145 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 146 { 147 const char *mod, *sym; 148 extern int db_active; 149 msan_orig_t *orig; 150 const char *typename; 151 char *var, *fn; 152 uintptr_t ptr; 153 char buf[128]; 154 int type; 155 156 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 157 return; 158 159 kmsan_reporting = true; 160 __insn_barrier(); 161 162 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 163 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 164 165 if (*orig == 0) { 166 REPORT("MSan: Uninitialized Memory In %s At Offset " 167 "%zu\n", hook, off); 168 goto out; 169 } 170 171 kmsan_md_orig_decode(*orig, &type, &ptr); 172 typename = kmsan_orig_name(type); 173 174 if (kmsan_md_is_pc(ptr)) { 175 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 176 REPORT("MSan: Uninitialized %s Memory In %s " 177 "At Offset %zu/%zu, IP %p\n", typename, hook, off, 178 size, (void *)ptr); 179 } else { 180 REPORT("MSan: Uninitialized %s Memory In %s " 181 "At Offset %zu/%zu, From %s()\n", typename, hook, 182 off, size, sym); 183 } 184 } else { 185 var = (char *)ptr + 4; 186 strlcpy(buf, var, sizeof(buf)); 187 var = buf; 188 fn = __builtin_strchr(buf, '@'); 189 *fn++ = '\0'; 190 REPORT("MSan: Uninitialized %s Memory In %s At Offset " 191 "%zu, Variable '%s' From %s()\n", typename, hook, off, 192 var, fn); 193 } 194 195 out: 196 kmsan_md_unwind(); 197 __insn_barrier(); 198 kmsan_reporting = false; 199 } 200 201 static void 202 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 203 { 204 const char *mod, *sym; 205 extern int db_active; 206 const char *typename; 207 char *var, *fn; 208 uintptr_t ptr; 209 char buf[128]; 210 int type; 211 212 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 213 return; 214 215 kmsan_reporting = true; 216 __insn_barrier(); 217 218 if (orig == 0) { 219 REPORT("MSan: Uninitialized Variable In %p\n", 220 (void *)pc); 221 goto out; 222 } 223 224 kmsan_md_orig_decode(orig, &type, &ptr); 225 typename = kmsan_orig_name(type); 226 227 if (kmsan_md_is_pc(ptr)) { 228 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 229 REPORT("MSan: Uninitialized %s Memory, " 230 "Origin %x\n", typename, orig); 231 } else { 232 REPORT("MSan: Uninitialized %s Memory " 233 "From %s()\n", typename, sym); 234 } 235 } else { 236 var = (char *)ptr + 4; 237 strlcpy(buf, var, sizeof(buf)); 238 var = buf; 239 fn = __builtin_strchr(buf, '@'); 240 *fn++ = '\0'; 241 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 242 var, fn); 243 } 244 245 out: 246 kmsan_md_unwind(); 247 __insn_barrier(); 248 kmsan_reporting = false; 249 } 250 251 /* -------------------------------------------------------------------------- */ 252 253 static inline msan_meta_t 254 kmsan_meta_get(void *addr, size_t size) 255 { 256 msan_meta_t ret; 257 258 if (__predict_false(!kmsan_enabled)) { 259 ret.shad = msan_dummy_shad; 260 ret.orig = (msan_orig_t *)msan_dummy_orig; 261 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 262 ret.shad = msan_dummy_shad; 263 ret.orig = (msan_orig_t *)msan_dummy_orig; 264 } else { 265 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 266 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 267 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 268 } 269 270 return ret; 271 } 272 273 static inline void 274 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 275 { 276 msan_orig_t *orig; 277 size_t i; 278 279 if (__predict_false(!kmsan_enabled)) 280 return; 281 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 282 return; 283 284 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 285 size += ((uintptr_t)orig & 0x3); 286 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 287 288 for (i = 0; i < size; i += 4) { 289 orig[i / 4] = o; 290 } 291 } 292 293 static inline void 294 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 295 { 296 uint8_t *shad; 297 298 if (__predict_false(!kmsan_enabled)) 299 return; 300 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 301 return; 302 303 shad = kmsan_md_addr_to_shad(addr); 304 __builtin_memset(shad, c, size); 305 } 306 307 static inline void 308 kmsan_meta_copy(void *dst, const void *src, size_t size) 309 { 310 uint8_t *orig_src, *orig_dst; 311 uint8_t *shad_src, *shad_dst; 312 msan_orig_t *_src, *_dst; 313 size_t i; 314 315 if (__predict_false(!kmsan_enabled)) 316 return; 317 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 318 return; 319 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 320 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 321 return; 322 } 323 324 shad_src = kmsan_md_addr_to_shad(src); 325 shad_dst = kmsan_md_addr_to_shad(dst); 326 __builtin_memmove(shad_dst, shad_src, size); 327 328 orig_src = kmsan_md_addr_to_orig(src); 329 orig_dst = kmsan_md_addr_to_orig(dst); 330 for (i = 0; i < size; i++) { 331 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 332 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 333 *_dst = *_src; 334 orig_src++; 335 orig_dst++; 336 } 337 } 338 339 static inline void 340 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 341 { 342 uint8_t *shad; 343 size_t i; 344 345 if (__predict_false(!kmsan_enabled)) 346 return; 347 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 348 return; 349 350 shad = kmsan_md_addr_to_shad(addr); 351 for (i = 0; i < size; i++) { 352 if (__predict_true(shad[i] == 0)) 353 continue; 354 kmsan_report_hook((const char *)addr + i, size, i, hook); 355 break; 356 } 357 } 358 359 void 360 kmsan_init_arg(size_t n) 361 { 362 msan_lwp_t *lwp; 363 uint8_t *arg; 364 365 if (__predict_false(!kmsan_enabled)) 366 return; 367 lwp = curlwp->l_kmsan; 368 arg = lwp->tls[lwp->ctx].param; 369 __builtin_memset(arg, 0, n); 370 } 371 372 void 373 kmsan_init_ret(size_t n) 374 { 375 msan_lwp_t *lwp; 376 uint8_t *arg; 377 378 if (__predict_false(!kmsan_enabled)) 379 return; 380 lwp = curlwp->l_kmsan; 381 arg = lwp->tls[lwp->ctx].retval; 382 __builtin_memset(arg, 0, n); 383 } 384 385 static void 386 kmsan_check_arg(size_t size, const char *hook) 387 { 388 msan_lwp_t *lwp; 389 uint8_t *arg; 390 size_t i; 391 392 if (__predict_false(!kmsan_enabled)) 393 return; 394 lwp = curlwp->l_kmsan; 395 arg = lwp->tls[lwp->ctx].param; 396 397 for (i = 0; i < size; i++) { 398 if (__predict_true(arg[i] == 0)) 399 continue; 400 kmsan_report_hook((const char *)arg + i, size, i, hook); 401 break; 402 } 403 } 404 405 void 406 kmsan_lwp_alloc(struct lwp *l) 407 { 408 msan_lwp_t *lwp; 409 410 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 411 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 412 lwp->ctx = 1; 413 414 l->l_kmsan = lwp; 415 } 416 417 void 418 kmsan_lwp_free(struct lwp *l) 419 { 420 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 421 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 422 } 423 424 void kmsan_intr_enter(void); 425 void kmsan_intr_leave(void); 426 void kmsan_softint(struct lwp *); 427 428 void 429 kmsan_intr_enter(void) 430 { 431 msan_lwp_t *lwp; 432 433 if (__predict_false(!kmsan_enabled)) 434 return; 435 lwp = curlwp->l_kmsan; 436 437 lwp->ctx++; 438 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 439 kmsan_enabled = false; 440 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 441 } 442 443 kmsan_init_arg(sizeof(void *)); 444 } 445 446 void 447 kmsan_intr_leave(void) 448 { 449 msan_lwp_t *lwp; 450 451 if (__predict_false(!kmsan_enabled)) 452 return; 453 lwp = curlwp->l_kmsan; 454 455 if (__predict_false(lwp->ctx == 0)) { 456 kmsan_enabled = false; 457 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 458 } 459 lwp->ctx--; 460 } 461 462 void 463 kmsan_softint(struct lwp *l) 464 { 465 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 466 } 467 468 /* -------------------------------------------------------------------------- */ 469 470 void 471 kmsan_shadow_map(void *addr, size_t size) 472 { 473 size_t npages, i; 474 vaddr_t va; 475 476 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 477 KASSERT(size % PAGE_SIZE == 0); 478 479 npages = size / PAGE_SIZE; 480 481 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 482 for (i = 0; i < npages; i++) { 483 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 484 } 485 486 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 487 for (i = 0; i < npages; i++) { 488 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 489 } 490 } 491 492 void 493 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 494 { 495 msan_orig_t orig; 496 497 orig = kmsan_md_orig_encode(type, pc); 498 kmsan_origin_fill(addr, orig, size); 499 } 500 501 void 502 kmsan_mark(void *addr, size_t size, uint8_t c) 503 { 504 kmsan_shadow_fill(addr, c, size); 505 } 506 507 void 508 kmsan_check_mbuf(void *buf) 509 { 510 struct mbuf *m = buf; 511 512 do { 513 kmsan_shadow_check(mtod(m, void *), m->m_len, "MbufChain"); 514 } while ((m = m->m_next) != NULL); 515 } 516 517 void 518 kmsan_check_buf(void *buf) 519 { 520 buf_t *bp = buf; 521 522 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()"); 523 } 524 525 void 526 kmsan_init(void *stack) 527 { 528 /* MD initialization. */ 529 kmsan_md_init(); 530 531 /* Map the stack. */ 532 kmsan_shadow_map(stack, USPACE); 533 534 /* Initialize the TLS for curlwp. */ 535 msan_lwp0.ctx = 1; 536 curlwp->l_kmsan = &msan_lwp0; 537 538 /* Now officially enabled. */ 539 kmsan_enabled = true; 540 } 541 542 /* -------------------------------------------------------------------------- */ 543 544 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 545 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 546 547 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 548 { 549 return kmsan_meta_get(addr, size); 550 } 551 552 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 553 { 554 return kmsan_meta_get(addr, size); 555 } 556 557 #define MSAN_META_FUNC(size) \ 558 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 559 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 560 { \ 561 return kmsan_meta_get(addr, size); \ 562 } \ 563 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 564 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 565 { \ 566 return kmsan_meta_get(addr, size); \ 567 } 568 569 MSAN_META_FUNC(1) 570 MSAN_META_FUNC(2) 571 MSAN_META_FUNC(4) 572 MSAN_META_FUNC(8) 573 574 void __msan_instrument_asm_store(void *, size_t); 575 msan_orig_t __msan_chain_origin(msan_orig_t); 576 void __msan_poison_alloca(void *, uint64_t, char *); 577 void __msan_unpoison_alloca(void *, uint64_t); 578 void __msan_warning(msan_orig_t); 579 msan_tls_t *__msan_get_context_state(void); 580 581 void __msan_instrument_asm_store(void *addr, size_t size) 582 { 583 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 584 } 585 586 msan_orig_t __msan_chain_origin(msan_orig_t origin) 587 { 588 return origin; 589 } 590 591 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 592 { 593 msan_orig_t orig; 594 595 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 596 kmsan_origin_fill(addr, orig, size); 597 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 598 } 599 600 void __msan_unpoison_alloca(void *addr, uint64_t size) 601 { 602 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 603 } 604 605 void __msan_warning(msan_orig_t origin) 606 { 607 if (__predict_false(!kmsan_enabled)) 608 return; 609 kmsan_report_inline(origin, __RET_ADDR); 610 } 611 612 msan_tls_t *__msan_get_context_state(void) 613 { 614 msan_lwp_t *lwp; 615 616 if (__predict_false(!kmsan_enabled)) 617 return &dummy_tls; 618 lwp = curlwp->l_kmsan; 619 620 return &lwp->tls[lwp->ctx]; 621 } 622 623 /* -------------------------------------------------------------------------- */ 624 625 /* 626 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 627 * initialized areas properly. 628 */ 629 630 void *kmsan_memcpy(void *dst, const void *src, size_t len) 631 { 632 /* No kmsan_check_arg, because inlined. */ 633 kmsan_init_ret(sizeof(void *)); 634 if (__predict_true(len != 0)) { 635 kmsan_meta_copy(dst, src, len); 636 } 637 return __builtin_memcpy(dst, src, len); 638 } 639 640 int 641 kmsan_memcmp(const void *b1, const void *b2, size_t len) 642 { 643 const uint8_t *_b1 = b1, *_b2 = b2; 644 size_t i; 645 646 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 647 "memcmp():args"); 648 kmsan_init_ret(sizeof(int)); 649 650 for (i = 0; i < len; i++) { 651 if (*_b1 != *_b2) { 652 kmsan_shadow_check(b1, i + 1, "memcmp():arg1"); 653 kmsan_shadow_check(b2, i + 1, "memcmp():arg2"); 654 return *_b1 - *_b2; 655 } 656 _b1++, _b2++; 657 } 658 659 return 0; 660 } 661 662 void *kmsan_memset(void *dst, int c, size_t len) 663 { 664 /* No kmsan_check_arg, because inlined. */ 665 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 666 kmsan_init_ret(sizeof(void *)); 667 return __builtin_memset(dst, c, len); 668 } 669 670 void *kmsan_memmove(void *dst, const void *src, size_t len) 671 { 672 /* No kmsan_check_arg, because inlined. */ 673 if (__predict_true(len != 0)) { 674 kmsan_meta_copy(dst, src, len); 675 } 676 kmsan_init_ret(sizeof(void *)); 677 return __builtin_memmove(dst, src, len); 678 } 679 680 __strong_alias(__msan_memcpy, kmsan_memcpy) 681 __strong_alias(__msan_memset, kmsan_memset) 682 __strong_alias(__msan_memmove, kmsan_memmove) 683 684 char * 685 kmsan_strcpy(char *dst, const char *src) 686 { 687 const char *_src = src; 688 char *_dst = dst; 689 size_t len = 0; 690 691 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 692 693 while (1) { 694 len++; 695 *dst = *src; 696 if (*src == '\0') 697 break; 698 src++, dst++; 699 } 700 701 kmsan_shadow_check(_src, len, "strcpy():arg2"); 702 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 703 kmsan_init_ret(sizeof(char *)); 704 return _dst; 705 } 706 707 int 708 kmsan_strcmp(const char *s1, const char *s2) 709 { 710 const char *_s1 = s1, *_s2 = s2; 711 size_t len = 0; 712 713 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 714 kmsan_init_ret(sizeof(int)); 715 716 while (1) { 717 len++; 718 if (*s1 != *s2) 719 break; 720 if (*s1 == '\0') { 721 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 722 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 723 return 0; 724 } 725 s1++, s2++; 726 } 727 728 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 729 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 730 731 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 732 } 733 734 size_t 735 kmsan_strlen(const char *str) 736 { 737 const char *s; 738 739 kmsan_check_arg(sizeof(str), "strlen():args"); 740 741 s = str; 742 while (1) { 743 if (*s == '\0') 744 break; 745 s++; 746 } 747 748 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1"); 749 kmsan_init_ret(sizeof(size_t)); 750 return (s - str); 751 } 752 753 char * 754 kmsan_strcat(char *dst, const char *src) 755 { 756 size_t ldst, lsrc; 757 char *ret; 758 759 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args"); 760 761 ldst = __builtin_strlen(dst); 762 lsrc = __builtin_strlen(src); 763 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1"); 764 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2"); 765 ret = __builtin_strcat(dst, src); 766 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1); 767 768 kmsan_init_ret(sizeof(char *)); 769 return ret; 770 } 771 772 char * 773 kmsan_strchr(const char *s, int c) 774 { 775 char *ret; 776 777 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args"); 778 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1"); 779 ret = __builtin_strchr(s, c); 780 781 kmsan_init_ret(sizeof(char *)); 782 return ret; 783 } 784 785 char * 786 kmsan_strrchr(const char *s, int c) 787 { 788 char *ret; 789 790 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args"); 791 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1"); 792 ret = __builtin_strrchr(s, c); 793 794 kmsan_init_ret(sizeof(char *)); 795 return ret; 796 } 797 798 #undef kcopy 799 #undef copyin 800 #undef copyout 801 #undef copyinstr 802 #undef copyoutstr 803 804 int kmsan_kcopy(const void *, void *, size_t); 805 int kmsan_copyin(const void *, void *, size_t); 806 int kmsan_copyout(const void *, void *, size_t); 807 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 808 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 809 810 int kcopy(const void *, void *, size_t); 811 int copyin(const void *, void *, size_t); 812 int copyout(const void *, void *, size_t); 813 int copyinstr(const void *, void *, size_t, size_t *); 814 int copyoutstr(const void *, void *, size_t, size_t *); 815 816 int 817 kmsan_kcopy(const void *src, void *dst, size_t len) 818 { 819 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), 820 "kcopy():args"); 821 if (__predict_true(len != 0)) { 822 kmsan_meta_copy(dst, src, len); 823 } 824 kmsan_init_ret(sizeof(int)); 825 return kcopy(src, dst, len); 826 } 827 828 int 829 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 830 { 831 int ret; 832 833 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 834 "copyin():args"); 835 ret = copyin(uaddr, kaddr, len); 836 if (ret == 0) 837 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 838 kmsan_init_ret(sizeof(int)); 839 840 return ret; 841 } 842 843 int 844 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 845 { 846 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 847 "copyout():args"); 848 kmsan_shadow_check(kaddr, len, "copyout():arg1"); 849 kmsan_init_ret(sizeof(int)); 850 return copyout(kaddr, uaddr, len); 851 } 852 853 int 854 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 855 { 856 size_t _done; 857 int ret; 858 859 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 860 sizeof(len) + sizeof(done), "copyinstr():args"); 861 ret = copyinstr(uaddr, kaddr, len, &_done); 862 if (ret == 0 || ret == ENAMETOOLONG) 863 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 864 if (done != NULL) { 865 *done = _done; 866 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 867 } 868 kmsan_init_ret(sizeof(int)); 869 870 return ret; 871 } 872 873 int 874 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 875 { 876 size_t _done; 877 int ret; 878 879 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 880 sizeof(len) + sizeof(done), "copyoutstr():args"); 881 ret = copyoutstr(kaddr, uaddr, len, &_done); 882 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1"); 883 if (done != NULL) { 884 *done = _done; 885 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 886 } 887 kmsan_init_ret(sizeof(int)); 888 889 return ret; 890 } 891 892 /* -------------------------------------------------------------------------- */ 893 894 #undef _ucas_32 895 #undef _ucas_32_mp 896 #undef _ucas_64 897 #undef _ucas_64_mp 898 #undef _ufetch_8 899 #undef _ufetch_16 900 #undef _ufetch_32 901 #undef _ufetch_64 902 #undef _ustore_8 903 #undef _ustore_16 904 #undef _ustore_32 905 #undef _ustore_64 906 907 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 908 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 909 int 910 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 911 uint32_t *ret) 912 { 913 int _ret; 914 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 915 sizeof(new) + sizeof(ret), "ucas_32():args"); 916 _ret = _ucas_32(uaddr, old, new, ret); 917 if (_ret == 0) 918 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 919 kmsan_init_ret(sizeof(int)); 920 return _ret; 921 } 922 923 #ifdef __HAVE_UCAS_MP 924 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 925 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 926 int 927 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 928 uint32_t *ret) 929 { 930 int _ret; 931 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 932 sizeof(new) + sizeof(ret), "ucas_32_mp():args"); 933 _ret = _ucas_32_mp(uaddr, old, new, ret); 934 if (_ret == 0) 935 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 936 kmsan_init_ret(sizeof(int)); 937 return _ret; 938 } 939 #endif 940 941 #ifdef _LP64 942 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 943 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 944 int 945 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 946 uint64_t *ret) 947 { 948 int _ret; 949 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 950 sizeof(new) + sizeof(ret), "ucas_64():args"); 951 _ret = _ucas_64(uaddr, old, new, ret); 952 if (_ret == 0) 953 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 954 kmsan_init_ret(sizeof(int)); 955 return _ret; 956 } 957 958 #ifdef __HAVE_UCAS_MP 959 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 960 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 961 int 962 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 963 uint64_t *ret) 964 { 965 int _ret; 966 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 967 sizeof(new) + sizeof(ret), "ucas_64_mp():args"); 968 _ret = _ucas_64_mp(uaddr, old, new, ret); 969 if (_ret == 0) 970 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 971 kmsan_init_ret(sizeof(int)); 972 return _ret; 973 } 974 #endif 975 #endif 976 977 int _ufetch_8(const uint8_t *, uint8_t *); 978 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 979 int 980 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 981 { 982 int _ret; 983 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args"); 984 _ret = _ufetch_8(uaddr, valp); 985 if (_ret == 0) 986 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 987 kmsan_init_ret(sizeof(int)); 988 return _ret; 989 } 990 991 int _ufetch_16(const uint16_t *, uint16_t *); 992 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 993 int 994 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 995 { 996 int _ret; 997 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args"); 998 _ret = _ufetch_16(uaddr, valp); 999 if (_ret == 0) 1000 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1001 kmsan_init_ret(sizeof(int)); 1002 return _ret; 1003 } 1004 1005 int _ufetch_32(const uint32_t *, uint32_t *); 1006 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 1007 int 1008 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 1009 { 1010 int _ret; 1011 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args"); 1012 _ret = _ufetch_32(uaddr, valp); 1013 if (_ret == 0) 1014 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1015 kmsan_init_ret(sizeof(int)); 1016 return _ret; 1017 } 1018 1019 #ifdef _LP64 1020 int _ufetch_64(const uint64_t *, uint64_t *); 1021 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1022 int 1023 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1024 { 1025 int _ret; 1026 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args"); 1027 _ret = _ufetch_64(uaddr, valp); 1028 if (_ret == 0) 1029 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1030 kmsan_init_ret(sizeof(int)); 1031 return _ret; 1032 } 1033 #endif 1034 1035 int _ustore_8(uint8_t *, uint8_t); 1036 int kmsan__ustore_8(uint8_t *, uint8_t); 1037 int 1038 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1039 { 1040 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args"); 1041 kmsan_init_ret(sizeof(int)); 1042 return _ustore_8(uaddr, val); 1043 } 1044 1045 int _ustore_16(uint16_t *, uint16_t); 1046 int kmsan__ustore_16(uint16_t *, uint16_t); 1047 int 1048 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1049 { 1050 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args"); 1051 kmsan_init_ret(sizeof(int)); 1052 return _ustore_16(uaddr, val); 1053 } 1054 1055 int _ustore_32(uint32_t *, uint32_t); 1056 int kmsan__ustore_32(uint32_t *, uint32_t); 1057 int 1058 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1059 { 1060 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args"); 1061 kmsan_init_ret(sizeof(int)); 1062 return _ustore_32(uaddr, val); 1063 } 1064 1065 #ifdef _LP64 1066 int _ustore_64(uint64_t *, uint64_t); 1067 int kmsan__ustore_64(uint64_t *, uint64_t); 1068 int 1069 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1070 { 1071 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args"); 1072 kmsan_init_ret(sizeof(int)); 1073 return _ustore_64(uaddr, val); 1074 } 1075 #endif 1076 1077 /* -------------------------------------------------------------------------- */ 1078 1079 #undef atomic_add_32 1080 #undef atomic_add_int 1081 #undef atomic_add_long 1082 #undef atomic_add_ptr 1083 #undef atomic_add_64 1084 #undef atomic_add_32_nv 1085 #undef atomic_add_int_nv 1086 #undef atomic_add_long_nv 1087 #undef atomic_add_ptr_nv 1088 #undef atomic_add_64_nv 1089 #undef atomic_and_32 1090 #undef atomic_and_uint 1091 #undef atomic_and_ulong 1092 #undef atomic_and_64 1093 #undef atomic_and_32_nv 1094 #undef atomic_and_uint_nv 1095 #undef atomic_and_ulong_nv 1096 #undef atomic_and_64_nv 1097 #undef atomic_or_32 1098 #undef atomic_or_uint 1099 #undef atomic_or_ulong 1100 #undef atomic_or_64 1101 #undef atomic_or_32_nv 1102 #undef atomic_or_uint_nv 1103 #undef atomic_or_ulong_nv 1104 #undef atomic_or_64_nv 1105 #undef atomic_cas_32 1106 #undef atomic_cas_uint 1107 #undef atomic_cas_ulong 1108 #undef atomic_cas_ptr 1109 #undef atomic_cas_64 1110 #undef atomic_cas_32_ni 1111 #undef atomic_cas_uint_ni 1112 #undef atomic_cas_ulong_ni 1113 #undef atomic_cas_ptr_ni 1114 #undef atomic_cas_64_ni 1115 #undef atomic_swap_32 1116 #undef atomic_swap_uint 1117 #undef atomic_swap_ulong 1118 #undef atomic_swap_ptr 1119 #undef atomic_swap_64 1120 #undef atomic_dec_32 1121 #undef atomic_dec_uint 1122 #undef atomic_dec_ulong 1123 #undef atomic_dec_ptr 1124 #undef atomic_dec_64 1125 #undef atomic_dec_32_nv 1126 #undef atomic_dec_uint_nv 1127 #undef atomic_dec_ulong_nv 1128 #undef atomic_dec_ptr_nv 1129 #undef atomic_dec_64_nv 1130 #undef atomic_inc_32 1131 #undef atomic_inc_uint 1132 #undef atomic_inc_ulong 1133 #undef atomic_inc_ptr 1134 #undef atomic_inc_64 1135 #undef atomic_inc_32_nv 1136 #undef atomic_inc_uint_nv 1137 #undef atomic_inc_ulong_nv 1138 #undef atomic_inc_ptr_nv 1139 #undef atomic_inc_64_nv 1140 1141 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1142 void atomic_add_##name(volatile targ1 *, targ2); \ 1143 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1144 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1145 { \ 1146 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1147 "atomic_add_" #name "():args"); \ 1148 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1149 "atomic_add_" #name "():arg1"); \ 1150 atomic_add_##name(ptr, val); \ 1151 } \ 1152 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1153 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1154 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1155 { \ 1156 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1157 "atomic_add_" #name "_nv():args"); \ 1158 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1159 "atomic_add_" #name "_nv():arg1"); \ 1160 kmsan_init_ret(sizeof(tret)); \ 1161 return atomic_add_##name##_nv(ptr, val); \ 1162 } 1163 1164 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1165 void atomic_and_##name(volatile targ1 *, targ2); \ 1166 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1167 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1168 { \ 1169 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1170 "atomic_and_" #name "():args"); \ 1171 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1172 "atomic_and_" #name "():arg1"); \ 1173 atomic_and_##name(ptr, val); \ 1174 } \ 1175 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1176 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1177 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1178 { \ 1179 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1180 "atomic_and_" #name "_nv():args"); \ 1181 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1182 "atomic_and_" #name "_nv():arg1"); \ 1183 kmsan_init_ret(sizeof(tret)); \ 1184 return atomic_and_##name##_nv(ptr, val); \ 1185 } 1186 1187 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1188 void atomic_or_##name(volatile targ1 *, targ2); \ 1189 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1190 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1191 { \ 1192 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1193 "atomic_or_" #name "():args"); \ 1194 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1195 "atomic_or_" #name "():arg1"); \ 1196 atomic_or_##name(ptr, val); \ 1197 } \ 1198 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1199 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1200 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1201 { \ 1202 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1203 "atomic_or_" #name "_nv():args"); \ 1204 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1205 "atomic_or_" #name "_nv():arg1"); \ 1206 kmsan_init_ret(sizeof(tret)); \ 1207 return atomic_or_##name##_nv(ptr, val); \ 1208 } 1209 1210 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1211 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1212 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1213 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1214 { \ 1215 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1216 "atomic_cas_" #name "():args"); \ 1217 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1218 "atomic_cas_" #name "():arg1"); \ 1219 kmsan_init_ret(sizeof(tret)); \ 1220 return atomic_cas_##name(ptr, exp, new); \ 1221 } \ 1222 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1223 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1224 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1225 { \ 1226 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1227 "atomic_cas_" #name "_ni():args"); \ 1228 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1229 "atomic_cas_" #name "_ni():arg1"); \ 1230 kmsan_init_ret(sizeof(tret)); \ 1231 return atomic_cas_##name##_ni(ptr, exp, new); \ 1232 } 1233 1234 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1235 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1236 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1237 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1238 { \ 1239 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1240 "atomic_swap_" #name "():args"); \ 1241 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1242 "atomic_swap_" #name "():arg1"); \ 1243 kmsan_init_ret(sizeof(tret)); \ 1244 return atomic_swap_##name(ptr, val); \ 1245 } 1246 1247 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1248 void atomic_dec_##name(volatile targ1 *); \ 1249 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1250 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1251 { \ 1252 kmsan_check_arg(sizeof(ptr), \ 1253 "atomic_dec_" #name "():args"); \ 1254 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1255 "atomic_dec_" #name "():arg1"); \ 1256 atomic_dec_##name(ptr); \ 1257 } \ 1258 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1259 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1260 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1261 { \ 1262 kmsan_check_arg(sizeof(ptr), \ 1263 "atomic_dec_" #name "_nv():args"); \ 1264 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1265 "atomic_dec_" #name "_nv():arg1"); \ 1266 kmsan_init_ret(sizeof(tret)); \ 1267 return atomic_dec_##name##_nv(ptr); \ 1268 } 1269 1270 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1271 void atomic_inc_##name(volatile targ1 *); \ 1272 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1273 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1274 { \ 1275 kmsan_check_arg(sizeof(ptr), \ 1276 "atomic_inc_" #name "():args"); \ 1277 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1278 "atomic_inc_" #name "():arg1"); \ 1279 atomic_inc_##name(ptr); \ 1280 } \ 1281 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1282 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1283 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1284 { \ 1285 kmsan_check_arg(sizeof(ptr), \ 1286 "atomic_inc_" #name "_nv():args"); \ 1287 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1288 "atomic_inc_" #name "_nv():arg1"); \ 1289 kmsan_init_ret(sizeof(tret)); \ 1290 return atomic_inc_##name##_nv(ptr); \ 1291 } 1292 1293 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1294 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1295 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1296 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1297 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1298 1299 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1300 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1301 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1302 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1303 1304 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1305 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1306 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1307 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1308 1309 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1310 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1311 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1312 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1313 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1314 1315 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1316 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1317 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1318 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1319 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1320 1321 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1322 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1323 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1324 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1325 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1326 1327 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1328 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1329 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1330 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1331 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1332 1333 /* -------------------------------------------------------------------------- */ 1334 1335 #include <sys/bus.h> 1336 1337 #undef bus_space_read_multi_1 1338 #undef bus_space_read_multi_2 1339 #undef bus_space_read_multi_4 1340 #undef bus_space_read_multi_8 1341 #undef bus_space_read_multi_stream_1 1342 #undef bus_space_read_multi_stream_2 1343 #undef bus_space_read_multi_stream_4 1344 #undef bus_space_read_multi_stream_8 1345 #undef bus_space_read_region_1 1346 #undef bus_space_read_region_2 1347 #undef bus_space_read_region_4 1348 #undef bus_space_read_region_8 1349 #undef bus_space_read_region_stream_1 1350 #undef bus_space_read_region_stream_2 1351 #undef bus_space_read_region_stream_4 1352 #undef bus_space_read_region_stream_8 1353 1354 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1355 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1356 bus_size_t, uint##bits##_t *, bus_size_t); \ 1357 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1358 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1359 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1360 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1361 bus_size_t count) \ 1362 { \ 1363 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1364 sizeof(uint##bits##_t) * count); \ 1365 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1366 } \ 1367 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1368 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1369 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1370 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1371 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1372 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1373 bus_size_t count) \ 1374 { \ 1375 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1376 sizeof(uint##bits##_t) * count); \ 1377 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1378 } \ 1379 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1380 bus_size_t, uint##bits##_t *, bus_size_t); \ 1381 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1382 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1383 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1384 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1385 bus_size_t count) \ 1386 { \ 1387 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1388 sizeof(uint##bits##_t) * count); \ 1389 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1390 } \ 1391 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1392 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1393 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1394 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1395 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1396 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1397 bus_size_t count) \ 1398 { \ 1399 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1400 sizeof(uint##bits##_t) * count); \ 1401 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1402 } 1403 1404 MSAN_BUS_READ_FUNC(1, 8) 1405 MSAN_BUS_READ_FUNC(2, 16) 1406 MSAN_BUS_READ_FUNC(4, 32) 1407 MSAN_BUS_READ_FUNC(8, 64) 1408 1409 #undef bus_space_write_multi_1 1410 #undef bus_space_write_multi_2 1411 #undef bus_space_write_multi_4 1412 #undef bus_space_write_multi_8 1413 #undef bus_space_write_multi_stream_1 1414 #undef bus_space_write_multi_stream_2 1415 #undef bus_space_write_multi_stream_4 1416 #undef bus_space_write_multi_stream_8 1417 #undef bus_space_write_region_1 1418 #undef bus_space_write_region_2 1419 #undef bus_space_write_region_4 1420 #undef bus_space_write_region_8 1421 #undef bus_space_write_region_stream_1 1422 #undef bus_space_write_region_stream_2 1423 #undef bus_space_write_region_stream_4 1424 #undef bus_space_write_region_stream_8 1425 1426 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1427 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1428 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1429 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1430 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1431 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1432 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1433 bus_size_t count) \ 1434 { \ 1435 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1436 "bus_space_write()"); \ 1437 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1438 } \ 1439 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1440 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1441 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1442 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1443 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1444 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1445 bus_size_t count) \ 1446 { \ 1447 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1448 "bus_space_write()"); \ 1449 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1450 } \ 1451 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1452 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1453 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1454 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1455 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1456 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1457 bus_size_t count) \ 1458 { \ 1459 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1460 "bus_space_write()"); \ 1461 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1462 } \ 1463 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1464 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1465 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1466 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1467 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1468 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1469 bus_size_t count) \ 1470 { \ 1471 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1472 "bus_space_write()"); \ 1473 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1474 } 1475 1476 MSAN_BUS_WRITE_FUNC(1, 8) 1477 MSAN_BUS_WRITE_FUNC(2, 16) 1478 MSAN_BUS_WRITE_FUNC(4, 32) 1479 MSAN_BUS_WRITE_FUNC(8, 64) 1480 1481 /* -------------------------------------------------------------------------- */ 1482 1483 #include <sys/mbuf.h> 1484 1485 static void 1486 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1487 bool init, uintptr_t pc) 1488 { 1489 if (init) { 1490 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1491 } else { 1492 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp"); 1493 } 1494 } 1495 1496 static void 1497 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1498 bool init, uintptr_t pc) 1499 { 1500 bus_addr_t minlen; 1501 1502 for (; m != NULL && len != 0; m = m->m_next) { 1503 if (offset >= m->m_len) { 1504 offset -= m->m_len; 1505 continue; 1506 } 1507 1508 minlen = MIN(len, m->m_len - offset); 1509 1510 if (init) { 1511 kmsan_shadow_fill(mtod(m, char *) + offset, 1512 KMSAN_STATE_INITED, minlen); 1513 } else { 1514 kmsan_shadow_check(mtod(m, char *) + offset, 1515 minlen, "MbufDmaSyncOp"); 1516 } 1517 1518 offset = 0; 1519 len -= minlen; 1520 } 1521 } 1522 1523 static void 1524 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1525 bool init, uintptr_t pc) 1526 { 1527 bus_size_t minlen, resid; 1528 struct iovec *iov; 1529 int i; 1530 1531 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1532 return; 1533 1534 resid = uio->uio_resid; 1535 iov = uio->uio_iov; 1536 1537 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1538 minlen = MIN(resid, iov[i].iov_len); 1539 1540 if (init) { 1541 kmsan_shadow_fill(iov[i].iov_base, 1542 KMSAN_STATE_INITED, minlen); 1543 } else { 1544 kmsan_shadow_check(iov[i].iov_base, minlen, 1545 "UioDmaSyncOp"); 1546 } 1547 1548 resid -= minlen; 1549 } 1550 } 1551 1552 void 1553 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1554 { 1555 bool init; 1556 1557 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1558 return; 1559 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1560 1561 switch (map->dm_buftype) { 1562 case KMSAN_DMA_LINEAR: 1563 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1564 __RET_ADDR); 1565 break; 1566 case KMSAN_DMA_MBUF: 1567 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1568 __RET_ADDR); 1569 break; 1570 case KMSAN_DMA_UIO: 1571 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1572 __RET_ADDR); 1573 break; 1574 case KMSAN_DMA_RAW: 1575 break; 1576 default: 1577 panic("%s: impossible", __func__); 1578 } 1579 } 1580 1581 void 1582 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1583 { 1584 map->dm_buf = buf; 1585 map->dm_buflen = buflen; 1586 map->dm_buftype = type; 1587 } 1588
Indexes created Sat Sep 20 22:09:52 GMT 2025