subr_msan.c revision 1.18
1 /* $NetBSD: subr_msan.c,v 1.18 2022/10/26 23:38:09 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KMSAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.18 2022/10/26 23:38:09 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/kprintf.h> 42 #include <sys/kmem.h> 43 #include <sys/mbuf.h> 44 #include <sys/buf.h> 45 #include <sys/cpu.h> 46 #include <sys/msan.h> 47 48 #include <ddb/db_active.h> 49 50 static void kmsan_printf(const char *, ...); 51 52 void kmsan_init_arg(size_t); 53 void kmsan_init_ret(size_t); 54 55 #ifdef KMSAN_PANIC 56 #define REPORT panic 57 #else 58 #define REPORT kmsan_printf 59 #endif 60 61 /* -------------------------------------------------------------------------- */ 62 63 /* 64 * Part of the compiler ABI. 65 */ 66 67 typedef uint32_t msan_orig_t; 68 69 typedef struct { 70 uint8_t *shad; 71 msan_orig_t *orig; 72 } msan_meta_t; 73 74 #define MSAN_PARAM_SIZE 800 75 #define MSAN_RETVAL_SIZE 800 76 typedef struct { 77 uint8_t param[MSAN_PARAM_SIZE]; 78 uint8_t retval[MSAN_RETVAL_SIZE]; 79 uint8_t _va_arg[MSAN_PARAM_SIZE]; 80 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 81 uint64_t va_arg_overflow_size; 82 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 83 msan_orig_t retval_origin; 84 msan_orig_t origin; 85 } msan_tls_t; 86 87 /* -------------------------------------------------------------------------- */ 88 89 /* The MD code. */ 90 #include <machine/msan.h> 91 92 /* -------------------------------------------------------------------------- */ 93 94 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 95 #define MSAN_NCONTEXT 16 96 97 typedef struct { 98 size_t ctx; 99 msan_tls_t tls[MSAN_NCONTEXT]; 100 } msan_lwp_t; 101 102 static msan_tls_t dummy_tls; 103 104 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 105 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 106 static msan_lwp_t msan_lwp0; 107 static bool kmsan_enabled __read_mostly; 108 109 /* -------------------------------------------------------------------------- */ 110 111 static bool kmsan_reporting = false; 112 113 static inline void 114 kmsan_printf(const char *fmt, ...) 115 { 116 va_list ap; 117 118 va_start(ap, fmt); 119 kprintf(fmt, TOCONS, NULL, NULL, ap); 120 va_end(ap); 121 } 122 123 static inline const char * 124 kmsan_orig_name(int type) 125 { 126 switch (type) { 127 case KMSAN_TYPE_STACK: 128 return "Stack"; 129 case KMSAN_TYPE_KMEM: 130 return "Kmem"; 131 case KMSAN_TYPE_MALLOC: 132 return "Malloc"; 133 case KMSAN_TYPE_POOL: 134 return "Pool"; 135 case KMSAN_TYPE_UVM: 136 return "Uvm"; 137 default: 138 return "Unknown"; 139 } 140 } 141 142 /* 143 * The format of the string is: "----var@function". Parse it to display a nice 144 * warning. 145 */ 146 static void 147 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 148 { 149 const char *mod, *sym; 150 msan_orig_t *orig; 151 const char *typename; 152 char *var, *fn; 153 uintptr_t ptr; 154 char buf[128]; 155 int type; 156 int s; 157 158 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 159 return; 160 161 kmsan_reporting = true; 162 __insn_barrier(); 163 164 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 165 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 166 167 if (*orig == 0) { 168 REPORT("MSan: Uninitialized Memory In %s At Offset " 169 "%zu\n", hook, off); 170 goto out; 171 } 172 173 kmsan_md_orig_decode(*orig, &type, &ptr); 174 typename = kmsan_orig_name(type); 175 176 if (kmsan_md_is_pc(ptr)) { 177 s = pserialize_read_enter(); 178 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 179 REPORT("MSan: Uninitialized %s Memory In %s " 180 "At Offset %zu/%zu, IP %p\n", typename, hook, off, 181 size, (void *)ptr); 182 } else { 183 REPORT("MSan: Uninitialized %s Memory In %s " 184 "At Offset %zu/%zu, From %s()\n", typename, hook, 185 off, size, sym); 186 } 187 pserialize_read_exit(s); 188 } else { 189 var = (char *)ptr + 4; 190 strlcpy(buf, var, sizeof(buf)); 191 var = buf; 192 fn = __builtin_strchr(buf, '@'); 193 *fn++ = '\0'; 194 REPORT("MSan: Uninitialized %s Memory In %s At Offset " 195 "%zu, Variable '%s' From %s()\n", typename, hook, off, 196 var, fn); 197 } 198 199 out: 200 kmsan_md_unwind(); 201 __insn_barrier(); 202 kmsan_reporting = false; 203 } 204 205 static void 206 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 207 { 208 const char *mod, *sym; 209 const char *typename; 210 char *var, *fn; 211 uintptr_t ptr; 212 char buf[128]; 213 int type; 214 int s; 215 216 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 217 return; 218 219 kmsan_reporting = true; 220 __insn_barrier(); 221 222 if (orig == 0) { 223 REPORT("MSan: Uninitialized Variable In %p\n", 224 (void *)pc); 225 goto out; 226 } 227 228 kmsan_md_orig_decode(orig, &type, &ptr); 229 typename = kmsan_orig_name(type); 230 231 if (kmsan_md_is_pc(ptr)) { 232 s = pserialize_read_enter(); 233 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 234 REPORT("MSan: Uninitialized %s Memory, " 235 "Origin %x\n", typename, orig); 236 } else { 237 REPORT("MSan: Uninitialized %s Memory " 238 "From %s()\n", typename, sym); 239 } 240 pserialize_read_exit(s); 241 } else { 242 var = (char *)ptr + 4; 243 strlcpy(buf, var, sizeof(buf)); 244 var = buf; 245 fn = __builtin_strchr(buf, '@'); 246 *fn++ = '\0'; 247 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 248 var, fn); 249 } 250 251 out: 252 kmsan_md_unwind(); 253 __insn_barrier(); 254 kmsan_reporting = false; 255 } 256 257 /* -------------------------------------------------------------------------- */ 258 259 static inline msan_meta_t 260 kmsan_meta_get(void *addr, size_t size) 261 { 262 msan_meta_t ret; 263 264 if (__predict_false(!kmsan_enabled)) { 265 ret.shad = msan_dummy_shad; 266 ret.orig = (msan_orig_t *)msan_dummy_orig; 267 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 268 ret.shad = msan_dummy_shad; 269 ret.orig = (msan_orig_t *)msan_dummy_orig; 270 } else { 271 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 272 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 273 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 274 } 275 276 return ret; 277 } 278 279 static inline void 280 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 281 { 282 msan_orig_t *orig; 283 size_t i; 284 285 if (__predict_false(!kmsan_enabled)) 286 return; 287 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 288 return; 289 290 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 291 size += ((uintptr_t)orig & 0x3); 292 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 293 294 for (i = 0; i < size; i += 4) { 295 orig[i / 4] = o; 296 } 297 } 298 299 static inline void 300 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 301 { 302 uint8_t *shad; 303 304 if (__predict_false(!kmsan_enabled)) 305 return; 306 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 307 return; 308 309 shad = kmsan_md_addr_to_shad(addr); 310 __builtin_memset(shad, c, size); 311 } 312 313 static inline void 314 kmsan_meta_copy(void *dst, const void *src, size_t size) 315 { 316 uint8_t *orig_src, *orig_dst; 317 uint8_t *shad_src, *shad_dst; 318 msan_orig_t *_src, *_dst; 319 size_t i; 320 321 if (__predict_false(!kmsan_enabled)) 322 return; 323 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 324 return; 325 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 326 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 327 return; 328 } 329 330 shad_src = kmsan_md_addr_to_shad(src); 331 shad_dst = kmsan_md_addr_to_shad(dst); 332 __builtin_memmove(shad_dst, shad_src, size); 333 334 orig_src = kmsan_md_addr_to_orig(src); 335 orig_dst = kmsan_md_addr_to_orig(dst); 336 for (i = 0; i < size; i++) { 337 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 338 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 339 *_dst = *_src; 340 orig_src++; 341 orig_dst++; 342 } 343 } 344 345 static inline void 346 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 347 { 348 uint8_t *shad; 349 size_t i; 350 351 if (__predict_false(!kmsan_enabled)) 352 return; 353 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 354 return; 355 356 shad = kmsan_md_addr_to_shad(addr); 357 for (i = 0; i < size; i++) { 358 if (__predict_true(shad[i] == 0)) 359 continue; 360 kmsan_report_hook((const char *)addr + i, size, i, hook); 361 break; 362 } 363 } 364 365 void 366 kmsan_init_arg(size_t n) 367 { 368 msan_lwp_t *lwp; 369 uint8_t *arg; 370 371 if (__predict_false(!kmsan_enabled)) 372 return; 373 lwp = curlwp->l_kmsan; 374 arg = lwp->tls[lwp->ctx].param; 375 __builtin_memset(arg, 0, n); 376 } 377 378 void 379 kmsan_init_ret(size_t n) 380 { 381 msan_lwp_t *lwp; 382 uint8_t *arg; 383 384 if (__predict_false(!kmsan_enabled)) 385 return; 386 lwp = curlwp->l_kmsan; 387 arg = lwp->tls[lwp->ctx].retval; 388 __builtin_memset(arg, 0, n); 389 } 390 391 static void 392 kmsan_check_arg(size_t size, const char *hook) 393 { 394 msan_lwp_t *lwp; 395 uint8_t *arg; 396 size_t i; 397 398 if (__predict_false(!kmsan_enabled)) 399 return; 400 lwp = curlwp->l_kmsan; 401 arg = lwp->tls[lwp->ctx].param; 402 403 for (i = 0; i < size; i++) { 404 if (__predict_true(arg[i] == 0)) 405 continue; 406 kmsan_report_hook((const char *)arg + i, size, i, hook); 407 break; 408 } 409 } 410 411 void 412 kmsan_lwp_alloc(struct lwp *l) 413 { 414 msan_lwp_t *lwp; 415 416 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 417 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 418 lwp->ctx = 1; 419 420 l->l_kmsan = lwp; 421 } 422 423 void 424 kmsan_lwp_free(struct lwp *l) 425 { 426 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 427 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 428 } 429 430 void kmsan_intr_enter(void); 431 void kmsan_intr_leave(void); 432 void kmsan_softint(struct lwp *); 433 434 void 435 kmsan_intr_enter(void) 436 { 437 msan_lwp_t *lwp; 438 439 if (__predict_false(!kmsan_enabled)) 440 return; 441 lwp = curlwp->l_kmsan; 442 443 lwp->ctx++; 444 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 445 kmsan_enabled = false; 446 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 447 } 448 449 kmsan_init_arg(sizeof(void *)); 450 } 451 452 void 453 kmsan_intr_leave(void) 454 { 455 msan_lwp_t *lwp; 456 457 if (__predict_false(!kmsan_enabled)) 458 return; 459 lwp = curlwp->l_kmsan; 460 461 if (__predict_false(lwp->ctx == 0)) { 462 kmsan_enabled = false; 463 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 464 } 465 lwp->ctx--; 466 } 467 468 void 469 kmsan_softint(struct lwp *l) 470 { 471 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 472 } 473 474 /* -------------------------------------------------------------------------- */ 475 476 void 477 kmsan_shadow_map(void *addr, size_t size) 478 { 479 size_t npages, i; 480 vaddr_t va; 481 482 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 483 KASSERT(size % PAGE_SIZE == 0); 484 485 npages = size / PAGE_SIZE; 486 487 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 488 for (i = 0; i < npages; i++) { 489 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 490 } 491 492 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 493 for (i = 0; i < npages; i++) { 494 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 495 } 496 } 497 498 void 499 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 500 { 501 msan_orig_t orig; 502 503 orig = kmsan_md_orig_encode(type, pc); 504 kmsan_origin_fill(addr, orig, size); 505 } 506 507 void 508 kmsan_mark(void *addr, size_t size, uint8_t c) 509 { 510 kmsan_shadow_fill(addr, c, size); 511 } 512 513 void 514 kmsan_check_mbuf(void *buf) 515 { 516 struct mbuf *m = buf; 517 518 do { 519 kmsan_shadow_check(mtod(m, void *), m->m_len, "MbufChain"); 520 } while ((m = m->m_next) != NULL); 521 } 522 523 void 524 kmsan_check_buf(void *buf) 525 { 526 buf_t *bp = buf; 527 528 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()"); 529 } 530 531 void 532 kmsan_init(void *stack) 533 { 534 /* MD initialization. */ 535 kmsan_md_init(); 536 537 /* Map the stack. */ 538 kmsan_shadow_map(stack, USPACE); 539 540 /* Initialize the TLS for curlwp. */ 541 msan_lwp0.ctx = 1; 542 curlwp->l_kmsan = &msan_lwp0; 543 544 /* Now officially enabled. */ 545 kmsan_enabled = true; 546 } 547 548 /* -------------------------------------------------------------------------- */ 549 550 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 551 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 552 553 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 554 { 555 return kmsan_meta_get(addr, size); 556 } 557 558 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 559 { 560 return kmsan_meta_get(addr, size); 561 } 562 563 #define MSAN_META_FUNC(size) \ 564 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 565 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 566 { \ 567 return kmsan_meta_get(addr, size); \ 568 } \ 569 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 570 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 571 { \ 572 return kmsan_meta_get(addr, size); \ 573 } 574 575 MSAN_META_FUNC(1) 576 MSAN_META_FUNC(2) 577 MSAN_META_FUNC(4) 578 MSAN_META_FUNC(8) 579 580 void __msan_instrument_asm_store(void *, size_t); 581 msan_orig_t __msan_chain_origin(msan_orig_t); 582 void __msan_poison_alloca(void *, uint64_t, char *); 583 void __msan_unpoison_alloca(void *, uint64_t); 584 void __msan_warning(msan_orig_t); 585 msan_tls_t *__msan_get_context_state(void); 586 587 void __msan_instrument_asm_store(void *addr, size_t size) 588 { 589 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 590 } 591 592 msan_orig_t __msan_chain_origin(msan_orig_t origin) 593 { 594 return origin; 595 } 596 597 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 598 { 599 msan_orig_t orig; 600 601 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 602 kmsan_origin_fill(addr, orig, size); 603 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 604 } 605 606 void __msan_unpoison_alloca(void *addr, uint64_t size) 607 { 608 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 609 } 610 611 void __msan_warning(msan_orig_t origin) 612 { 613 if (__predict_false(!kmsan_enabled)) 614 return; 615 kmsan_report_inline(origin, __RET_ADDR); 616 } 617 618 msan_tls_t *__msan_get_context_state(void) 619 { 620 msan_lwp_t *lwp; 621 622 if (__predict_false(!kmsan_enabled)) 623 return &dummy_tls; 624 lwp = curlwp->l_kmsan; 625 626 return &lwp->tls[lwp->ctx]; 627 } 628 629 /* -------------------------------------------------------------------------- */ 630 631 /* 632 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 633 * initialized areas properly. 634 */ 635 636 void *kmsan_memcpy(void *dst, const void *src, size_t len) 637 { 638 /* No kmsan_check_arg, because inlined. */ 639 kmsan_init_ret(sizeof(void *)); 640 if (__predict_true(len != 0)) { 641 kmsan_meta_copy(dst, src, len); 642 } 643 return __builtin_memcpy(dst, src, len); 644 } 645 646 int 647 kmsan_memcmp(const void *b1, const void *b2, size_t len) 648 { 649 const uint8_t *_b1 = b1, *_b2 = b2; 650 size_t i; 651 652 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 653 "memcmp():args"); 654 kmsan_init_ret(sizeof(int)); 655 656 for (i = 0; i < len; i++) { 657 if (*_b1 != *_b2) { 658 kmsan_shadow_check(b1, i + 1, "memcmp():arg1"); 659 kmsan_shadow_check(b2, i + 1, "memcmp():arg2"); 660 return *_b1 - *_b2; 661 } 662 _b1++, _b2++; 663 } 664 665 return 0; 666 } 667 668 void *kmsan_memset(void *dst, int c, size_t len) 669 { 670 /* No kmsan_check_arg, because inlined. */ 671 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 672 kmsan_init_ret(sizeof(void *)); 673 return __builtin_memset(dst, c, len); 674 } 675 676 void *kmsan_memmove(void *dst, const void *src, size_t len) 677 { 678 /* No kmsan_check_arg, because inlined. */ 679 if (__predict_true(len != 0)) { 680 kmsan_meta_copy(dst, src, len); 681 } 682 kmsan_init_ret(sizeof(void *)); 683 return __builtin_memmove(dst, src, len); 684 } 685 686 __strong_alias(__msan_memcpy, kmsan_memcpy) 687 __strong_alias(__msan_memset, kmsan_memset) 688 __strong_alias(__msan_memmove, kmsan_memmove) 689 690 char * 691 kmsan_strcpy(char *dst, const char *src) 692 { 693 const char *_src = src; 694 char *_dst = dst; 695 size_t len = 0; 696 697 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 698 699 while (1) { 700 len++; 701 *dst = *src; 702 if (*src == '\0') 703 break; 704 src++, dst++; 705 } 706 707 kmsan_shadow_check(_src, len, "strcpy():arg2"); 708 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 709 kmsan_init_ret(sizeof(char *)); 710 return _dst; 711 } 712 713 int 714 kmsan_strcmp(const char *s1, const char *s2) 715 { 716 const char *_s1 = s1, *_s2 = s2; 717 size_t len = 0; 718 719 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 720 kmsan_init_ret(sizeof(int)); 721 722 while (1) { 723 len++; 724 if (*s1 != *s2) 725 break; 726 if (*s1 == '\0') { 727 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 728 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 729 return 0; 730 } 731 s1++, s2++; 732 } 733 734 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 735 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 736 737 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 738 } 739 740 size_t 741 kmsan_strlen(const char *str) 742 { 743 const char *s; 744 745 kmsan_check_arg(sizeof(str), "strlen():args"); 746 747 s = str; 748 while (1) { 749 if (*s == '\0') 750 break; 751 s++; 752 } 753 754 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1"); 755 kmsan_init_ret(sizeof(size_t)); 756 return (s - str); 757 } 758 759 char * 760 kmsan_strcat(char *dst, const char *src) 761 { 762 size_t ldst, lsrc; 763 char *ret; 764 765 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args"); 766 767 ldst = __builtin_strlen(dst); 768 lsrc = __builtin_strlen(src); 769 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1"); 770 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2"); 771 ret = __builtin_strcat(dst, src); 772 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1); 773 774 kmsan_init_ret(sizeof(char *)); 775 return ret; 776 } 777 778 char * 779 kmsan_strchr(const char *s, int c) 780 { 781 char *ret; 782 783 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args"); 784 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1"); 785 ret = __builtin_strchr(s, c); 786 787 kmsan_init_ret(sizeof(char *)); 788 return ret; 789 } 790 791 char * 792 kmsan_strrchr(const char *s, int c) 793 { 794 char *ret; 795 796 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args"); 797 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1"); 798 ret = __builtin_strrchr(s, c); 799 800 kmsan_init_ret(sizeof(char *)); 801 return ret; 802 } 803 804 #undef kcopy 805 #undef copyin 806 #undef copyout 807 #undef copyinstr 808 #undef copyoutstr 809 810 int kmsan_kcopy(const void *, void *, size_t); 811 int kmsan_copyin(const void *, void *, size_t); 812 int kmsan_copyout(const void *, void *, size_t); 813 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 814 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 815 816 int kcopy(const void *, void *, size_t); 817 int copyin(const void *, void *, size_t); 818 int copyout(const void *, void *, size_t); 819 int copyinstr(const void *, void *, size_t, size_t *); 820 int copyoutstr(const void *, void *, size_t, size_t *); 821 822 int 823 kmsan_kcopy(const void *src, void *dst, size_t len) 824 { 825 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), 826 "kcopy():args"); 827 if (__predict_true(len != 0)) { 828 kmsan_meta_copy(dst, src, len); 829 } 830 kmsan_init_ret(sizeof(int)); 831 return kcopy(src, dst, len); 832 } 833 834 int 835 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 836 { 837 int ret; 838 839 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 840 "copyin():args"); 841 ret = copyin(uaddr, kaddr, len); 842 if (ret == 0) 843 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 844 kmsan_init_ret(sizeof(int)); 845 846 return ret; 847 } 848 849 int 850 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 851 { 852 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 853 "copyout():args"); 854 kmsan_shadow_check(kaddr, len, "copyout():arg1"); 855 kmsan_init_ret(sizeof(int)); 856 return copyout(kaddr, uaddr, len); 857 } 858 859 int 860 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 861 { 862 size_t _done; 863 int ret; 864 865 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 866 sizeof(len) + sizeof(done), "copyinstr():args"); 867 ret = copyinstr(uaddr, kaddr, len, &_done); 868 if (ret == 0 || ret == ENAMETOOLONG) 869 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 870 if (done != NULL) { 871 *done = _done; 872 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 873 } 874 kmsan_init_ret(sizeof(int)); 875 876 return ret; 877 } 878 879 int 880 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 881 { 882 size_t _done; 883 int ret; 884 885 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 886 sizeof(len) + sizeof(done), "copyoutstr():args"); 887 ret = copyoutstr(kaddr, uaddr, len, &_done); 888 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1"); 889 if (done != NULL) { 890 *done = _done; 891 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 892 } 893 kmsan_init_ret(sizeof(int)); 894 895 return ret; 896 } 897 898 /* -------------------------------------------------------------------------- */ 899 900 #undef _ucas_32 901 #undef _ucas_32_mp 902 #undef _ucas_64 903 #undef _ucas_64_mp 904 #undef _ufetch_8 905 #undef _ufetch_16 906 #undef _ufetch_32 907 #undef _ufetch_64 908 #undef _ustore_8 909 #undef _ustore_16 910 #undef _ustore_32 911 #undef _ustore_64 912 913 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 914 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 915 int 916 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 917 uint32_t *ret) 918 { 919 int _ret; 920 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 921 sizeof(new) + sizeof(ret), "ucas_32():args"); 922 _ret = _ucas_32(uaddr, old, new, ret); 923 if (_ret == 0) 924 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 925 kmsan_init_ret(sizeof(int)); 926 return _ret; 927 } 928 929 #ifdef __HAVE_UCAS_MP 930 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 931 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 932 int 933 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 934 uint32_t *ret) 935 { 936 int _ret; 937 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 938 sizeof(new) + sizeof(ret), "ucas_32_mp():args"); 939 _ret = _ucas_32_mp(uaddr, old, new, ret); 940 if (_ret == 0) 941 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 942 kmsan_init_ret(sizeof(int)); 943 return _ret; 944 } 945 #endif 946 947 #ifdef _LP64 948 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 949 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 950 int 951 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 952 uint64_t *ret) 953 { 954 int _ret; 955 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 956 sizeof(new) + sizeof(ret), "ucas_64():args"); 957 _ret = _ucas_64(uaddr, old, new, ret); 958 if (_ret == 0) 959 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 960 kmsan_init_ret(sizeof(int)); 961 return _ret; 962 } 963 964 #ifdef __HAVE_UCAS_MP 965 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 966 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 967 int 968 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 969 uint64_t *ret) 970 { 971 int _ret; 972 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 973 sizeof(new) + sizeof(ret), "ucas_64_mp():args"); 974 _ret = _ucas_64_mp(uaddr, old, new, ret); 975 if (_ret == 0) 976 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 977 kmsan_init_ret(sizeof(int)); 978 return _ret; 979 } 980 #endif 981 #endif 982 983 int _ufetch_8(const uint8_t *, uint8_t *); 984 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 985 int 986 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 987 { 988 int _ret; 989 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args"); 990 _ret = _ufetch_8(uaddr, valp); 991 if (_ret == 0) 992 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 993 kmsan_init_ret(sizeof(int)); 994 return _ret; 995 } 996 997 int _ufetch_16(const uint16_t *, uint16_t *); 998 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 999 int 1000 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 1001 { 1002 int _ret; 1003 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args"); 1004 _ret = _ufetch_16(uaddr, valp); 1005 if (_ret == 0) 1006 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1007 kmsan_init_ret(sizeof(int)); 1008 return _ret; 1009 } 1010 1011 int _ufetch_32(const uint32_t *, uint32_t *); 1012 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 1013 int 1014 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 1015 { 1016 int _ret; 1017 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args"); 1018 _ret = _ufetch_32(uaddr, valp); 1019 if (_ret == 0) 1020 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1021 kmsan_init_ret(sizeof(int)); 1022 return _ret; 1023 } 1024 1025 #ifdef _LP64 1026 int _ufetch_64(const uint64_t *, uint64_t *); 1027 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1028 int 1029 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1030 { 1031 int _ret; 1032 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args"); 1033 _ret = _ufetch_64(uaddr, valp); 1034 if (_ret == 0) 1035 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1036 kmsan_init_ret(sizeof(int)); 1037 return _ret; 1038 } 1039 #endif 1040 1041 int _ustore_8(uint8_t *, uint8_t); 1042 int kmsan__ustore_8(uint8_t *, uint8_t); 1043 int 1044 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1045 { 1046 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args"); 1047 kmsan_init_ret(sizeof(int)); 1048 return _ustore_8(uaddr, val); 1049 } 1050 1051 int _ustore_16(uint16_t *, uint16_t); 1052 int kmsan__ustore_16(uint16_t *, uint16_t); 1053 int 1054 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1055 { 1056 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args"); 1057 kmsan_init_ret(sizeof(int)); 1058 return _ustore_16(uaddr, val); 1059 } 1060 1061 int _ustore_32(uint32_t *, uint32_t); 1062 int kmsan__ustore_32(uint32_t *, uint32_t); 1063 int 1064 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1065 { 1066 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args"); 1067 kmsan_init_ret(sizeof(int)); 1068 return _ustore_32(uaddr, val); 1069 } 1070 1071 #ifdef _LP64 1072 int _ustore_64(uint64_t *, uint64_t); 1073 int kmsan__ustore_64(uint64_t *, uint64_t); 1074 int 1075 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1076 { 1077 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args"); 1078 kmsan_init_ret(sizeof(int)); 1079 return _ustore_64(uaddr, val); 1080 } 1081 #endif 1082 1083 /* -------------------------------------------------------------------------- */ 1084 1085 #undef atomic_add_32 1086 #undef atomic_add_int 1087 #undef atomic_add_long 1088 #undef atomic_add_ptr 1089 #undef atomic_add_64 1090 #undef atomic_add_32_nv 1091 #undef atomic_add_int_nv 1092 #undef atomic_add_long_nv 1093 #undef atomic_add_ptr_nv 1094 #undef atomic_add_64_nv 1095 #undef atomic_and_32 1096 #undef atomic_and_uint 1097 #undef atomic_and_ulong 1098 #undef atomic_and_64 1099 #undef atomic_and_32_nv 1100 #undef atomic_and_uint_nv 1101 #undef atomic_and_ulong_nv 1102 #undef atomic_and_64_nv 1103 #undef atomic_or_32 1104 #undef atomic_or_uint 1105 #undef atomic_or_ulong 1106 #undef atomic_or_64 1107 #undef atomic_or_32_nv 1108 #undef atomic_or_uint_nv 1109 #undef atomic_or_ulong_nv 1110 #undef atomic_or_64_nv 1111 #undef atomic_cas_32 1112 #undef atomic_cas_uint 1113 #undef atomic_cas_ulong 1114 #undef atomic_cas_ptr 1115 #undef atomic_cas_64 1116 #undef atomic_cas_32_ni 1117 #undef atomic_cas_uint_ni 1118 #undef atomic_cas_ulong_ni 1119 #undef atomic_cas_ptr_ni 1120 #undef atomic_cas_64_ni 1121 #undef atomic_swap_32 1122 #undef atomic_swap_uint 1123 #undef atomic_swap_ulong 1124 #undef atomic_swap_ptr 1125 #undef atomic_swap_64 1126 #undef atomic_dec_32 1127 #undef atomic_dec_uint 1128 #undef atomic_dec_ulong 1129 #undef atomic_dec_ptr 1130 #undef atomic_dec_64 1131 #undef atomic_dec_32_nv 1132 #undef atomic_dec_uint_nv 1133 #undef atomic_dec_ulong_nv 1134 #undef atomic_dec_ptr_nv 1135 #undef atomic_dec_64_nv 1136 #undef atomic_inc_32 1137 #undef atomic_inc_uint 1138 #undef atomic_inc_ulong 1139 #undef atomic_inc_ptr 1140 #undef atomic_inc_64 1141 #undef atomic_inc_32_nv 1142 #undef atomic_inc_uint_nv 1143 #undef atomic_inc_ulong_nv 1144 #undef atomic_inc_ptr_nv 1145 #undef atomic_inc_64_nv 1146 1147 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1148 void atomic_add_##name(volatile targ1 *, targ2); \ 1149 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1150 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1151 { \ 1152 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1153 "atomic_add_" #name "():args"); \ 1154 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1155 "atomic_add_" #name "():arg1"); \ 1156 atomic_add_##name(ptr, val); \ 1157 } \ 1158 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1159 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1160 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1161 { \ 1162 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1163 "atomic_add_" #name "_nv():args"); \ 1164 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1165 "atomic_add_" #name "_nv():arg1"); \ 1166 kmsan_init_ret(sizeof(tret)); \ 1167 return atomic_add_##name##_nv(ptr, val); \ 1168 } 1169 1170 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1171 void atomic_and_##name(volatile targ1 *, targ2); \ 1172 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1173 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1174 { \ 1175 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1176 "atomic_and_" #name "():args"); \ 1177 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1178 "atomic_and_" #name "():arg1"); \ 1179 atomic_and_##name(ptr, val); \ 1180 } \ 1181 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1182 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1183 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1184 { \ 1185 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1186 "atomic_and_" #name "_nv():args"); \ 1187 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1188 "atomic_and_" #name "_nv():arg1"); \ 1189 kmsan_init_ret(sizeof(tret)); \ 1190 return atomic_and_##name##_nv(ptr, val); \ 1191 } 1192 1193 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1194 void atomic_or_##name(volatile targ1 *, targ2); \ 1195 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1196 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1197 { \ 1198 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1199 "atomic_or_" #name "():args"); \ 1200 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1201 "atomic_or_" #name "():arg1"); \ 1202 atomic_or_##name(ptr, val); \ 1203 } \ 1204 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1205 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1206 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1207 { \ 1208 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1209 "atomic_or_" #name "_nv():args"); \ 1210 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1211 "atomic_or_" #name "_nv():arg1"); \ 1212 kmsan_init_ret(sizeof(tret)); \ 1213 return atomic_or_##name##_nv(ptr, val); \ 1214 } 1215 1216 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1217 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1218 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1219 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1220 { \ 1221 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1222 "atomic_cas_" #name "():args"); \ 1223 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1224 "atomic_cas_" #name "():arg1"); \ 1225 kmsan_init_ret(sizeof(tret)); \ 1226 return atomic_cas_##name(ptr, exp, new); \ 1227 } \ 1228 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1229 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1230 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1231 { \ 1232 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1233 "atomic_cas_" #name "_ni():args"); \ 1234 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1235 "atomic_cas_" #name "_ni():arg1"); \ 1236 kmsan_init_ret(sizeof(tret)); \ 1237 return atomic_cas_##name##_ni(ptr, exp, new); \ 1238 } 1239 1240 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1241 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1242 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1243 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1244 { \ 1245 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1246 "atomic_swap_" #name "():args"); \ 1247 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1248 "atomic_swap_" #name "():arg1"); \ 1249 kmsan_init_ret(sizeof(tret)); \ 1250 return atomic_swap_##name(ptr, val); \ 1251 } 1252 1253 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1254 void atomic_dec_##name(volatile targ1 *); \ 1255 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1256 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1257 { \ 1258 kmsan_check_arg(sizeof(ptr), \ 1259 "atomic_dec_" #name "():args"); \ 1260 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1261 "atomic_dec_" #name "():arg1"); \ 1262 atomic_dec_##name(ptr); \ 1263 } \ 1264 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1265 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1266 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1267 { \ 1268 kmsan_check_arg(sizeof(ptr), \ 1269 "atomic_dec_" #name "_nv():args"); \ 1270 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1271 "atomic_dec_" #name "_nv():arg1"); \ 1272 kmsan_init_ret(sizeof(tret)); \ 1273 return atomic_dec_##name##_nv(ptr); \ 1274 } 1275 1276 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1277 void atomic_inc_##name(volatile targ1 *); \ 1278 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1279 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1280 { \ 1281 kmsan_check_arg(sizeof(ptr), \ 1282 "atomic_inc_" #name "():args"); \ 1283 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1284 "atomic_inc_" #name "():arg1"); \ 1285 atomic_inc_##name(ptr); \ 1286 } \ 1287 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1288 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1289 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1290 { \ 1291 kmsan_check_arg(sizeof(ptr), \ 1292 "atomic_inc_" #name "_nv():args"); \ 1293 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1294 "atomic_inc_" #name "_nv():arg1"); \ 1295 kmsan_init_ret(sizeof(tret)); \ 1296 return atomic_inc_##name##_nv(ptr); \ 1297 } 1298 1299 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1300 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1301 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1302 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1303 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1304 1305 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1306 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1307 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1308 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1309 1310 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1311 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1312 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1313 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1314 1315 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1316 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1317 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1318 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1319 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1320 1321 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1322 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1323 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1324 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1325 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1326 1327 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1328 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1329 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1330 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1331 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1332 1333 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1334 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1335 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1336 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1337 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1338 1339 /* -------------------------------------------------------------------------- */ 1340 1341 #include <sys/bus.h> 1342 1343 #undef bus_space_read_multi_1 1344 #undef bus_space_read_multi_2 1345 #undef bus_space_read_multi_4 1346 #undef bus_space_read_multi_8 1347 #undef bus_space_read_multi_stream_1 1348 #undef bus_space_read_multi_stream_2 1349 #undef bus_space_read_multi_stream_4 1350 #undef bus_space_read_multi_stream_8 1351 #undef bus_space_read_region_1 1352 #undef bus_space_read_region_2 1353 #undef bus_space_read_region_4 1354 #undef bus_space_read_region_8 1355 #undef bus_space_read_region_stream_1 1356 #undef bus_space_read_region_stream_2 1357 #undef bus_space_read_region_stream_4 1358 #undef bus_space_read_region_stream_8 1359 1360 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1361 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1362 bus_size_t, uint##bits##_t *, bus_size_t); \ 1363 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1364 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1365 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1366 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1367 bus_size_t count) \ 1368 { \ 1369 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1370 sizeof(uint##bits##_t) * count); \ 1371 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1372 } \ 1373 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1374 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1375 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1376 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1377 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1378 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1379 bus_size_t count) \ 1380 { \ 1381 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1382 sizeof(uint##bits##_t) * count); \ 1383 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1384 } \ 1385 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1386 bus_size_t, uint##bits##_t *, bus_size_t); \ 1387 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1388 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1389 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1390 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1391 bus_size_t count) \ 1392 { \ 1393 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1394 sizeof(uint##bits##_t) * count); \ 1395 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1396 } \ 1397 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1398 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1399 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1400 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1401 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1402 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1403 bus_size_t count) \ 1404 { \ 1405 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1406 sizeof(uint##bits##_t) * count); \ 1407 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1408 } 1409 1410 MSAN_BUS_READ_FUNC(1, 8) 1411 MSAN_BUS_READ_FUNC(2, 16) 1412 MSAN_BUS_READ_FUNC(4, 32) 1413 MSAN_BUS_READ_FUNC(8, 64) 1414 1415 #undef bus_space_write_multi_1 1416 #undef bus_space_write_multi_2 1417 #undef bus_space_write_multi_4 1418 #undef bus_space_write_multi_8 1419 #undef bus_space_write_multi_stream_1 1420 #undef bus_space_write_multi_stream_2 1421 #undef bus_space_write_multi_stream_4 1422 #undef bus_space_write_multi_stream_8 1423 #undef bus_space_write_region_1 1424 #undef bus_space_write_region_2 1425 #undef bus_space_write_region_4 1426 #undef bus_space_write_region_8 1427 #undef bus_space_write_region_stream_1 1428 #undef bus_space_write_region_stream_2 1429 #undef bus_space_write_region_stream_4 1430 #undef bus_space_write_region_stream_8 1431 1432 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1433 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1434 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1435 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1436 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1437 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1438 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1439 bus_size_t count) \ 1440 { \ 1441 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1442 "bus_space_write()"); \ 1443 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1444 } \ 1445 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1446 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1447 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1448 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1449 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1450 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1451 bus_size_t count) \ 1452 { \ 1453 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1454 "bus_space_write()"); \ 1455 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1456 } \ 1457 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1458 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1459 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1460 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1461 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1462 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1463 bus_size_t count) \ 1464 { \ 1465 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1466 "bus_space_write()"); \ 1467 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1468 } \ 1469 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1470 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1471 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1472 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1473 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1474 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1475 bus_size_t count) \ 1476 { \ 1477 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1478 "bus_space_write()"); \ 1479 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1480 } 1481 1482 MSAN_BUS_WRITE_FUNC(1, 8) 1483 MSAN_BUS_WRITE_FUNC(2, 16) 1484 MSAN_BUS_WRITE_FUNC(4, 32) 1485 MSAN_BUS_WRITE_FUNC(8, 64) 1486 1487 /* -------------------------------------------------------------------------- */ 1488 1489 #include <sys/mbuf.h> 1490 1491 static void 1492 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1493 bool init, uintptr_t pc) 1494 { 1495 if (init) { 1496 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1497 } else { 1498 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp"); 1499 } 1500 } 1501 1502 static void 1503 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1504 bool init, uintptr_t pc) 1505 { 1506 bus_addr_t minlen; 1507 1508 for (; m != NULL && len != 0; m = m->m_next) { 1509 if (offset >= m->m_len) { 1510 offset -= m->m_len; 1511 continue; 1512 } 1513 1514 minlen = MIN(len, m->m_len - offset); 1515 1516 if (init) { 1517 kmsan_shadow_fill(mtod(m, char *) + offset, 1518 KMSAN_STATE_INITED, minlen); 1519 } else { 1520 kmsan_shadow_check(mtod(m, char *) + offset, 1521 minlen, "MbufDmaSyncOp"); 1522 } 1523 1524 offset = 0; 1525 len -= minlen; 1526 } 1527 } 1528 1529 static void 1530 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1531 bool init, uintptr_t pc) 1532 { 1533 bus_size_t minlen, resid; 1534 struct iovec *iov; 1535 int i; 1536 1537 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1538 return; 1539 1540 resid = uio->uio_resid; 1541 iov = uio->uio_iov; 1542 1543 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1544 minlen = MIN(resid, iov[i].iov_len); 1545 1546 if (init) { 1547 kmsan_shadow_fill(iov[i].iov_base, 1548 KMSAN_STATE_INITED, minlen); 1549 } else { 1550 kmsan_shadow_check(iov[i].iov_base, minlen, 1551 "UioDmaSyncOp"); 1552 } 1553 1554 resid -= minlen; 1555 } 1556 } 1557 1558 void 1559 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1560 { 1561 bool init; 1562 1563 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1564 return; 1565 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1566 1567 switch (map->dm_buftype) { 1568 case KMSAN_DMA_LINEAR: 1569 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1570 __RET_ADDR); 1571 break; 1572 case KMSAN_DMA_MBUF: 1573 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1574 __RET_ADDR); 1575 break; 1576 case KMSAN_DMA_UIO: 1577 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1578 __RET_ADDR); 1579 break; 1580 case KMSAN_DMA_RAW: 1581 break; 1582 default: 1583 panic("%s: impossible", __func__); 1584 } 1585 } 1586 1587 void 1588 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1589 { 1590 map->dm_buf = buf; 1591 map->dm_buflen = buflen; 1592 map->dm_buftype = type; 1593 } 1594
Indexes created Sat Sep 20 22:09:52 GMT 2025