subr_msan.c revision 1.5.2.1
1 /* $NetBSD: subr_msan.c,v 1.5.2.1 2020/01/25 22:38:51 ad Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.5.2.1 2020/01/25 22:38:51 ad Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/kprintf.h> 43 #include <sys/kmem.h> 44 #include <sys/mbuf.h> 45 #include <sys/buf.h> 46 #include <sys/cpu.h> 47 #include <sys/msan.h> 48 49 #include <uvm/uvm.h> 50 51 static void kmsan_printf(const char *, ...); 52 53 #ifdef KMSAN_PANIC 54 #define REPORT panic 55 #else 56 #define REPORT kmsan_printf 57 #endif 58 59 /* -------------------------------------------------------------------------- */ 60 61 /* 62 * Part of the compiler ABI. 63 */ 64 65 typedef uint32_t msan_orig_t; 66 67 typedef struct { 68 uint8_t *shad; 69 msan_orig_t *orig; 70 } msan_meta_t; 71 72 #define MSAN_PARAM_SIZE 800 73 #define MSAN_RETVAL_SIZE 800 74 typedef struct { 75 uint8_t param[MSAN_PARAM_SIZE]; 76 uint8_t retval[MSAN_RETVAL_SIZE]; 77 uint8_t _va_arg[MSAN_PARAM_SIZE]; 78 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 79 uint64_t va_arg_overflow_size; 80 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 81 msan_orig_t retval_origin; 82 msan_orig_t origin; 83 } msan_tls_t; 84 85 /* -------------------------------------------------------------------------- */ 86 87 /* The MD code. */ 88 #include <machine/msan.h> 89 90 /* -------------------------------------------------------------------------- */ 91 92 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 93 #define MSAN_NCONTEXT 16 94 95 typedef struct { 96 size_t ctx; 97 msan_tls_t tls[MSAN_NCONTEXT]; 98 } msan_lwp_t; 99 100 static msan_tls_t dummy_tls; 101 102 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 103 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 104 static msan_lwp_t msan_lwp0; 105 static bool kmsan_enabled __read_mostly; 106 107 /* -------------------------------------------------------------------------- */ 108 109 static bool kmsan_reporting = false; 110 111 static inline void 112 kmsan_printf(const char *fmt, ...) 113 { 114 va_list ap; 115 116 va_start(ap, fmt); 117 kprintf(fmt, TOCONS, NULL, NULL, ap); 118 va_end(ap); 119 } 120 121 static inline const char * 122 kmsan_orig_name(int type) 123 { 124 switch (type) { 125 case KMSAN_TYPE_STACK: 126 return "Stack"; 127 case KMSAN_TYPE_KMEM: 128 return "Kmem"; 129 case KMSAN_TYPE_MALLOC: 130 return "Malloc"; 131 case KMSAN_TYPE_POOL: 132 return "Pool"; 133 case KMSAN_TYPE_UVM: 134 return "Uvm"; 135 default: 136 return "Unknown"; 137 } 138 } 139 140 /* 141 * The format of the string is: "----var@function". Parse it to display a nice 142 * warning. 143 */ 144 static void 145 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 146 { 147 const char *mod, *sym; 148 extern int db_active; 149 msan_orig_t *orig; 150 const char *typename; 151 char *var, *fn; 152 uintptr_t ptr; 153 char buf[128]; 154 int type; 155 156 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 157 return; 158 159 kmsan_reporting = true; 160 __insn_barrier(); 161 162 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 163 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 164 165 if (*orig == 0) { 166 REPORT("MSan: Uninitialized Memory In %s() At Offset " 167 "%zu\n", hook, off); 168 goto out; 169 } 170 171 kmsan_md_orig_decode(*orig, &type, &ptr); 172 typename = kmsan_orig_name(type); 173 174 if (kmsan_md_is_pc(ptr)) { 175 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 176 REPORT("MSan: Uninitialized %s Memory In %s() " 177 "At Offset %zu, IP %p\n", typename, hook, off, 178 (void *)ptr); 179 } else { 180 REPORT("MSan: Uninitialized %s Memory In %s() " 181 "At Offset %zu, From %s()\n", typename, hook, off, 182 sym); 183 } 184 } else { 185 var = (char *)ptr + 4; 186 strlcpy(buf, var, sizeof(buf)); 187 var = buf; 188 fn = __builtin_strchr(buf, '@'); 189 *fn++ = '\0'; 190 REPORT("MSan: Uninitialized %s Memory In %s() At Offset " 191 "%zu, Variable '%s' From %s()\n", typename, hook, off, 192 var, fn); 193 } 194 195 out: 196 kmsan_md_unwind(); 197 __insn_barrier(); 198 kmsan_reporting = false; 199 } 200 201 static void 202 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 203 { 204 const char *mod, *sym; 205 extern int db_active; 206 const char *typename; 207 char *var, *fn; 208 uintptr_t ptr; 209 char buf[128]; 210 int type; 211 212 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 213 return; 214 215 kmsan_reporting = true; 216 __insn_barrier(); 217 218 if (orig == 0) { 219 REPORT("MSan: Uninitialized Variable In %p\n", 220 (void *)pc); 221 goto out; 222 } 223 224 kmsan_md_orig_decode(orig, &type, &ptr); 225 typename = kmsan_orig_name(type); 226 227 if (kmsan_md_is_pc(ptr)) { 228 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 229 REPORT("MSan: Uninitialized %s Memory, " 230 "Origin %x\n", typename, orig); 231 } else { 232 REPORT("MSan: Uninitialized %s Memory " 233 "From %s()\n", typename, sym); 234 } 235 } else { 236 var = (char *)ptr + 4; 237 strlcpy(buf, var, sizeof(buf)); 238 var = buf; 239 fn = __builtin_strchr(buf, '@'); 240 *fn++ = '\0'; 241 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 242 var, fn); 243 } 244 245 out: 246 kmsan_md_unwind(); 247 __insn_barrier(); 248 kmsan_reporting = false; 249 } 250 251 /* -------------------------------------------------------------------------- */ 252 253 static inline msan_meta_t 254 kmsan_meta_get(void *addr, size_t size) 255 { 256 msan_meta_t ret; 257 258 if (__predict_false(!kmsan_enabled)) { 259 ret.shad = msan_dummy_shad; 260 ret.orig = (msan_orig_t *)msan_dummy_orig; 261 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 262 ret.shad = msan_dummy_shad; 263 ret.orig = (msan_orig_t *)msan_dummy_orig; 264 } else { 265 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 266 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 267 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 268 } 269 270 return ret; 271 } 272 273 static inline void 274 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 275 { 276 msan_orig_t *orig; 277 size_t i; 278 279 if (__predict_false(!kmsan_enabled)) 280 return; 281 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 282 return; 283 284 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 285 size += ((uintptr_t)orig & 0x3); 286 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 287 288 for (i = 0; i < size; i += 4) { 289 orig[i / 4] = o; 290 } 291 } 292 293 static inline void 294 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 295 { 296 uint8_t *shad; 297 298 if (__predict_false(!kmsan_enabled)) 299 return; 300 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 301 return; 302 303 shad = kmsan_md_addr_to_shad(addr); 304 __builtin_memset(shad, c, size); 305 } 306 307 static inline void 308 kmsan_meta_copy(void *dst, const void *src, size_t size) 309 { 310 uint8_t *orig_src, *orig_dst; 311 uint8_t *shad_src, *shad_dst; 312 msan_orig_t *_src, *_dst; 313 size_t i; 314 315 if (__predict_false(!kmsan_enabled)) 316 return; 317 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 318 return; 319 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 320 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 321 return; 322 } 323 324 shad_src = kmsan_md_addr_to_shad(src); 325 shad_dst = kmsan_md_addr_to_shad(dst); 326 __builtin_memmove(shad_dst, shad_src, size); 327 328 orig_src = kmsan_md_addr_to_orig(src); 329 orig_dst = kmsan_md_addr_to_orig(dst); 330 for (i = 0; i < size; i++) { 331 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 332 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 333 *_dst = *_src; 334 orig_src++; 335 orig_dst++; 336 } 337 } 338 339 static inline void 340 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 341 { 342 uint8_t *shad; 343 size_t i; 344 345 if (__predict_false(!kmsan_enabled)) 346 return; 347 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 348 return; 349 350 shad = kmsan_md_addr_to_shad(addr); 351 for (i = 0; i < size; i++) { 352 if (__predict_true(shad[i] == 0)) 353 continue; 354 kmsan_report_hook((const char *)addr + i, size, i, hook); 355 break; 356 } 357 } 358 359 void kmsan_init_arg(size_t); 360 void kmsan_init_ret(size_t); 361 362 void 363 kmsan_init_arg(size_t n) 364 { 365 msan_lwp_t *lwp; 366 uint8_t *arg; 367 368 if (__predict_false(!kmsan_enabled)) 369 return; 370 lwp = curlwp->l_kmsan; 371 arg = lwp->tls[lwp->ctx].param; 372 __builtin_memset(arg, 0, n); 373 } 374 375 void 376 kmsan_init_ret(size_t n) 377 { 378 msan_lwp_t *lwp; 379 uint8_t *arg; 380 381 if (__predict_false(!kmsan_enabled)) 382 return; 383 lwp = curlwp->l_kmsan; 384 arg = lwp->tls[lwp->ctx].retval; 385 __builtin_memset(arg, 0, n); 386 } 387 388 static void 389 kmsan_check_arg(size_t size, const char *hook) 390 { 391 msan_lwp_t *lwp; 392 uint8_t *arg; 393 size_t i; 394 395 if (__predict_false(!kmsan_enabled)) 396 return; 397 lwp = curlwp->l_kmsan; 398 arg = lwp->tls[lwp->ctx].param; 399 400 for (i = 0; i < size; i++) { 401 if (__predict_true(arg[i] == 0)) 402 continue; 403 kmsan_report_hook((const char *)arg + i, size, i, hook); 404 break; 405 } 406 } 407 408 void 409 kmsan_lwp_alloc(struct lwp *l) 410 { 411 msan_lwp_t *lwp; 412 413 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 414 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 415 lwp->ctx = 1; 416 417 l->l_kmsan = lwp; 418 } 419 420 void 421 kmsan_lwp_free(struct lwp *l) 422 { 423 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 424 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 425 } 426 427 void kmsan_intr_enter(void); 428 void kmsan_intr_leave(void); 429 void kmsan_softint(struct lwp *); 430 431 void 432 kmsan_intr_enter(void) 433 { 434 msan_lwp_t *lwp; 435 436 if (__predict_false(!kmsan_enabled)) 437 return; 438 lwp = curlwp->l_kmsan; 439 440 lwp->ctx++; 441 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 442 kmsan_enabled = false; 443 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 444 } 445 446 kmsan_init_arg(sizeof(void *)); 447 } 448 449 void 450 kmsan_intr_leave(void) 451 { 452 msan_lwp_t *lwp; 453 454 if (__predict_false(!kmsan_enabled)) 455 return; 456 lwp = curlwp->l_kmsan; 457 458 if (__predict_false(lwp->ctx == 0)) { 459 kmsan_enabled = false; 460 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 461 } 462 lwp->ctx--; 463 } 464 465 void 466 kmsan_softint(struct lwp *l) 467 { 468 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 469 } 470 471 /* -------------------------------------------------------------------------- */ 472 473 void 474 kmsan_shadow_map(void *addr, size_t size) 475 { 476 size_t npages, i; 477 vaddr_t va; 478 479 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 480 KASSERT(size % PAGE_SIZE == 0); 481 482 npages = size / PAGE_SIZE; 483 484 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 485 for (i = 0; i < npages; i++) { 486 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 487 } 488 489 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 490 for (i = 0; i < npages; i++) { 491 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 492 } 493 } 494 495 void 496 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 497 { 498 msan_orig_t orig; 499 500 orig = kmsan_md_orig_encode(type, pc); 501 kmsan_origin_fill(addr, orig, size); 502 } 503 504 void 505 kmsan_mark(void *addr, size_t size, uint8_t c) 506 { 507 kmsan_shadow_fill(addr, c, size); 508 } 509 510 void 511 kmsan_check_mbuf(void *buf) 512 { 513 struct mbuf *m = buf; 514 515 do { 516 kmsan_shadow_check(mtod(m, void *), m->m_len, "if_transmit"); 517 } while ((m = m->m_next) != NULL); 518 } 519 520 void 521 kmsan_check_buf(void *buf) 522 { 523 buf_t *bp = buf; 524 525 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite"); 526 } 527 528 void 529 kmsan_init(void *stack) 530 { 531 /* MD initialization. */ 532 kmsan_md_init(); 533 534 /* Map the stack. */ 535 kmsan_shadow_map(stack, USPACE); 536 537 /* Initialize the TLS for curlwp. */ 538 msan_lwp0.ctx = 1; 539 curlwp->l_kmsan = &msan_lwp0; 540 541 /* Now officially enabled. */ 542 kmsan_enabled = true; 543 } 544 545 /* -------------------------------------------------------------------------- */ 546 547 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 548 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 549 550 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 551 { 552 return kmsan_meta_get(addr, size); 553 } 554 555 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 556 { 557 return kmsan_meta_get(addr, size); 558 } 559 560 #define MSAN_META_FUNC(size) \ 561 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 562 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 563 { \ 564 return kmsan_meta_get(addr, size); \ 565 } \ 566 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 567 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 568 { \ 569 return kmsan_meta_get(addr, size); \ 570 } 571 572 MSAN_META_FUNC(1) 573 MSAN_META_FUNC(2) 574 MSAN_META_FUNC(4) 575 MSAN_META_FUNC(8) 576 577 void __msan_instrument_asm_store(void *, size_t); 578 msan_orig_t __msan_chain_origin(msan_orig_t); 579 void __msan_poison_alloca(void *, uint64_t, char *); 580 void __msan_unpoison_alloca(void *, uint64_t); 581 void __msan_warning(msan_orig_t); 582 msan_tls_t *__msan_get_context_state(void); 583 584 void __msan_instrument_asm_store(void *addr, size_t size) 585 { 586 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 587 } 588 589 msan_orig_t __msan_chain_origin(msan_orig_t origin) 590 { 591 return origin; 592 } 593 594 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 595 { 596 msan_orig_t orig; 597 598 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 599 kmsan_origin_fill(addr, orig, size); 600 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 601 } 602 603 void __msan_unpoison_alloca(void *addr, uint64_t size) 604 { 605 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 606 } 607 608 void __msan_warning(msan_orig_t origin) 609 { 610 if (__predict_false(!kmsan_enabled)) 611 return; 612 kmsan_report_inline(origin, __RET_ADDR); 613 } 614 615 msan_tls_t *__msan_get_context_state(void) 616 { 617 msan_lwp_t *lwp; 618 619 if (__predict_false(!kmsan_enabled)) 620 return &dummy_tls; 621 lwp = curlwp->l_kmsan; 622 623 return &lwp->tls[lwp->ctx]; 624 } 625 626 /* -------------------------------------------------------------------------- */ 627 628 /* 629 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 630 * initialized areas properly. 631 */ 632 633 void *kmsan_memcpy(void *dst, const void *src, size_t len) 634 { 635 /* No kmsan_check_arg, because inlined. */ 636 kmsan_init_ret(sizeof(void *)); 637 if (__predict_true(len != 0)) { 638 kmsan_meta_copy(dst, src, len); 639 } 640 return __builtin_memcpy(dst, src, len); 641 } 642 643 int 644 kmsan_memcmp(const void *b1, const void *b2, size_t len) 645 { 646 const uint8_t *_b1 = b1, *_b2 = b2; 647 size_t i; 648 649 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), "memcmp"); 650 kmsan_init_ret(sizeof(int)); 651 652 for (i = 0; i < len; i++) { 653 if (*_b1 != *_b2) { 654 kmsan_shadow_check(b1, i + 1, "memcmp"); 655 kmsan_shadow_check(b2, i + 1, "memcmp"); 656 return *_b1 - *_b2; 657 } 658 _b1++, _b2++; 659 } 660 661 return 0; 662 } 663 664 void *kmsan_memset(void *dst, int c, size_t len) 665 { 666 /* No kmsan_check_arg, because inlined. */ 667 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 668 kmsan_init_ret(sizeof(void *)); 669 return __builtin_memset(dst, c, len); 670 } 671 672 void *kmsan_memmove(void *dst, const void *src, size_t len) 673 { 674 /* No kmsan_check_arg, because inlined. */ 675 if (__predict_true(len != 0)) { 676 kmsan_meta_copy(dst, src, len); 677 } 678 kmsan_init_ret(sizeof(void *)); 679 return __builtin_memmove(dst, src, len); 680 } 681 682 __strong_alias(__msan_memcpy, kmsan_memcpy) 683 __strong_alias(__msan_memset, kmsan_memset) 684 __strong_alias(__msan_memmove, kmsan_memmove) 685 686 char * 687 kmsan_strcpy(char *dst, const char *src) 688 { 689 const char *_src = src; 690 char *_dst = dst; 691 size_t len = 0; 692 693 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy"); 694 695 while (1) { 696 len++; 697 *dst = *src; 698 if (*src == '\0') 699 break; 700 src++, dst++; 701 } 702 703 kmsan_shadow_check(_src, len, "strcpy"); 704 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 705 kmsan_init_ret(sizeof(char *)); 706 return _dst; 707 } 708 709 int 710 kmsan_strcmp(const char *s1, const char *s2) 711 { 712 const char *_s1 = s1, *_s2 = s2; 713 size_t len = 0; 714 715 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp"); 716 kmsan_init_ret(sizeof(int)); 717 718 while (1) { 719 len++; 720 if (*s1 != *s2) 721 break; 722 if (*s1 == '\0') { 723 kmsan_shadow_check(_s1, len, "strcmp"); 724 kmsan_shadow_check(_s2, len, "strcmp"); 725 return 0; 726 } 727 s1++, s2++; 728 } 729 730 kmsan_shadow_check(_s1, len, "strcmp"); 731 kmsan_shadow_check(_s2, len, "strcmp"); 732 733 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 734 } 735 736 size_t 737 kmsan_strlen(const char *str) 738 { 739 const char *s; 740 741 kmsan_check_arg(sizeof(str), "strlen"); 742 743 s = str; 744 while (1) { 745 if (*s == '\0') 746 break; 747 s++; 748 } 749 750 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen"); 751 kmsan_init_ret(sizeof(size_t)); 752 return (s - str); 753 } 754 755 char * 756 kmsan_strcat(char *dst, const char *src) 757 { 758 size_t ldst, lsrc; 759 char *ret; 760 761 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat"); 762 763 ldst = __builtin_strlen(dst); 764 lsrc = __builtin_strlen(src); 765 kmsan_shadow_check(dst, ldst + 1, "strcat"); 766 kmsan_shadow_check(src, lsrc + 1, "strcat"); 767 ret = __builtin_strcat(dst, src); 768 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1); 769 770 kmsan_init_ret(sizeof(char *)); 771 return ret; 772 } 773 774 char * 775 kmsan_strchr(const char *s, int c) 776 { 777 char *ret; 778 779 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr"); 780 kmsan_shadow_check(s, __builtin_strlen(s), "strchr"); 781 ret = __builtin_strchr(s, c); 782 783 kmsan_init_ret(sizeof(char *)); 784 return ret; 785 } 786 787 char * 788 kmsan_strrchr(const char *s, int c) 789 { 790 char *ret; 791 792 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr"); 793 kmsan_shadow_check(s, __builtin_strlen(s), "strrchr"); 794 ret = __builtin_strrchr(s, c); 795 796 kmsan_init_ret(sizeof(char *)); 797 return ret; 798 } 799 800 #undef kcopy 801 #undef copystr 802 #undef copyin 803 #undef copyout 804 #undef copyinstr 805 #undef copyoutstr 806 807 int kmsan_kcopy(const void *, void *, size_t); 808 int kmsan_copystr(const void *, void *, size_t, size_t *); 809 int kmsan_copyin(const void *, void *, size_t); 810 int kmsan_copyout(const void *, void *, size_t); 811 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 812 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 813 814 int kcopy(const void *, void *, size_t); 815 int copystr(const void *, void *, size_t, size_t *); 816 int copyin(const void *, void *, size_t); 817 int copyout(const void *, void *, size_t); 818 int copyinstr(const void *, void *, size_t, size_t *); 819 int copyoutstr(const void *, void *, size_t, size_t *); 820 821 int 822 kmsan_kcopy(const void *src, void *dst, size_t len) 823 { 824 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), "kcopy"); 825 if (__predict_true(len != 0)) { 826 kmsan_meta_copy(dst, src, len); 827 } 828 kmsan_init_ret(sizeof(int)); 829 return kcopy(src, dst, len); 830 } 831 832 int 833 kmsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 834 { 835 size_t _done; 836 int ret; 837 838 kmsan_check_arg(sizeof(kfaddr) + sizeof(kdaddr) + 839 sizeof(len) + sizeof(done), "copystr"); 840 ret = copystr(kfaddr, kdaddr, len, &_done); 841 if (ret == 0) 842 kmsan_meta_copy(kdaddr, kfaddr, _done); 843 if (done != NULL) { 844 *done = _done; 845 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 846 } 847 kmsan_init_ret(sizeof(int)); 848 849 return ret; 850 } 851 852 int 853 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 854 { 855 int ret; 856 857 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), "copyin"); 858 ret = copyin(uaddr, kaddr, len); 859 if (ret == 0) 860 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 861 kmsan_init_ret(sizeof(int)); 862 863 return ret; 864 } 865 866 int 867 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 868 { 869 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), "copyout"); 870 kmsan_shadow_check(kaddr, len, "copyout"); 871 kmsan_init_ret(sizeof(int)); 872 return copyout(kaddr, uaddr, len); 873 } 874 875 int 876 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 877 { 878 size_t _done; 879 int ret; 880 881 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 882 sizeof(len) + sizeof(done), "copyinstr"); 883 ret = copyinstr(uaddr, kaddr, len, &_done); 884 if (ret == 0) 885 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 886 if (done != NULL) { 887 *done = _done; 888 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 889 } 890 kmsan_init_ret(sizeof(int)); 891 892 return ret; 893 } 894 895 int 896 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 897 { 898 size_t _done; 899 int ret; 900 901 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 902 sizeof(len) + sizeof(done), "copyoutstr"); 903 ret = copyoutstr(kaddr, uaddr, len, &_done); 904 kmsan_shadow_check(kaddr, _done, "copyoutstr"); 905 if (done != NULL) { 906 *done = _done; 907 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 908 } 909 kmsan_init_ret(sizeof(int)); 910 911 return ret; 912 } 913 914 /* -------------------------------------------------------------------------- */ 915 916 #undef _ucas_32 917 #undef _ucas_32_mp 918 #undef _ucas_64 919 #undef _ucas_64_mp 920 #undef _ufetch_8 921 #undef _ufetch_16 922 #undef _ufetch_32 923 #undef _ufetch_64 924 #undef _ustore_8 925 #undef _ustore_16 926 #undef _ustore_32 927 #undef _ustore_64 928 929 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 930 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 931 int 932 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 933 uint32_t *ret) 934 { 935 int _ret; 936 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 937 sizeof(new) + sizeof(ret), "ucas_32"); 938 _ret = _ucas_32(uaddr, old, new, ret); 939 if (_ret == 0) 940 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 941 kmsan_init_ret(sizeof(int)); 942 return _ret; 943 } 944 945 #ifdef __HAVE_UCAS_MP 946 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 947 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 948 int 949 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 950 uint32_t *ret) 951 { 952 int _ret; 953 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 954 sizeof(new) + sizeof(ret), "ucas_32_mp"); 955 _ret = _ucas_32_mp(uaddr, old, new, ret); 956 if (_ret == 0) 957 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 958 kmsan_init_ret(sizeof(int)); 959 return _ret; 960 } 961 #endif 962 963 #ifdef _LP64 964 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 965 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 966 int 967 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 968 uint64_t *ret) 969 { 970 int _ret; 971 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 972 sizeof(new) + sizeof(ret), "ucas_64"); 973 _ret = _ucas_64(uaddr, old, new, ret); 974 if (_ret == 0) 975 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 976 kmsan_init_ret(sizeof(int)); 977 return _ret; 978 } 979 980 #ifdef __HAVE_UCAS_MP 981 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 982 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 983 int 984 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 985 uint64_t *ret) 986 { 987 int _ret; 988 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 989 sizeof(new) + sizeof(ret), "ucas_64_mp"); 990 _ret = _ucas_64_mp(uaddr, old, new, ret); 991 if (_ret == 0) 992 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 993 kmsan_init_ret(sizeof(int)); 994 return _ret; 995 } 996 #endif 997 #endif 998 999 int _ufetch_8(const uint8_t *, uint8_t *); 1000 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 1001 int 1002 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 1003 { 1004 int _ret; 1005 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8"); 1006 _ret = _ufetch_8(uaddr, valp); 1007 if (_ret == 0) 1008 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1009 kmsan_init_ret(sizeof(int)); 1010 return _ret; 1011 } 1012 1013 int _ufetch_16(const uint16_t *, uint16_t *); 1014 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 1015 int 1016 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 1017 { 1018 int _ret; 1019 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16"); 1020 _ret = _ufetch_16(uaddr, valp); 1021 if (_ret == 0) 1022 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1023 kmsan_init_ret(sizeof(int)); 1024 return _ret; 1025 } 1026 1027 int _ufetch_32(const uint32_t *, uint32_t *); 1028 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 1029 int 1030 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 1031 { 1032 int _ret; 1033 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32"); 1034 _ret = _ufetch_32(uaddr, valp); 1035 if (_ret == 0) 1036 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1037 kmsan_init_ret(sizeof(int)); 1038 return _ret; 1039 } 1040 1041 #ifdef _LP64 1042 int _ufetch_64(const uint64_t *, uint64_t *); 1043 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1044 int 1045 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1046 { 1047 int _ret; 1048 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64"); 1049 _ret = _ufetch_64(uaddr, valp); 1050 if (_ret == 0) 1051 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1052 kmsan_init_ret(sizeof(int)); 1053 return _ret; 1054 } 1055 #endif 1056 1057 int _ustore_8(uint8_t *, uint8_t); 1058 int kmsan__ustore_8(uint8_t *, uint8_t); 1059 int 1060 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1061 { 1062 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8"); 1063 kmsan_init_ret(sizeof(int)); 1064 return _ustore_8(uaddr, val); 1065 } 1066 1067 int _ustore_16(uint16_t *, uint16_t); 1068 int kmsan__ustore_16(uint16_t *, uint16_t); 1069 int 1070 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1071 { 1072 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16"); 1073 kmsan_init_ret(sizeof(int)); 1074 return _ustore_16(uaddr, val); 1075 } 1076 1077 int _ustore_32(uint32_t *, uint32_t); 1078 int kmsan__ustore_32(uint32_t *, uint32_t); 1079 int 1080 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1081 { 1082 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32"); 1083 kmsan_init_ret(sizeof(int)); 1084 return _ustore_32(uaddr, val); 1085 } 1086 1087 #ifdef _LP64 1088 int _ustore_64(uint64_t *, uint64_t); 1089 int kmsan__ustore_64(uint64_t *, uint64_t); 1090 int 1091 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1092 { 1093 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64"); 1094 kmsan_init_ret(sizeof(int)); 1095 return _ustore_64(uaddr, val); 1096 } 1097 #endif 1098 1099 /* -------------------------------------------------------------------------- */ 1100 1101 #undef atomic_add_32 1102 #undef atomic_add_int 1103 #undef atomic_add_long 1104 #undef atomic_add_ptr 1105 #undef atomic_add_64 1106 #undef atomic_add_32_nv 1107 #undef atomic_add_int_nv 1108 #undef atomic_add_long_nv 1109 #undef atomic_add_ptr_nv 1110 #undef atomic_add_64_nv 1111 #undef atomic_and_32 1112 #undef atomic_and_uint 1113 #undef atomic_and_ulong 1114 #undef atomic_and_64 1115 #undef atomic_and_32_nv 1116 #undef atomic_and_uint_nv 1117 #undef atomic_and_ulong_nv 1118 #undef atomic_and_64_nv 1119 #undef atomic_or_32 1120 #undef atomic_or_uint 1121 #undef atomic_or_ulong 1122 #undef atomic_or_64 1123 #undef atomic_or_32_nv 1124 #undef atomic_or_uint_nv 1125 #undef atomic_or_ulong_nv 1126 #undef atomic_or_64_nv 1127 #undef atomic_cas_32 1128 #undef atomic_cas_uint 1129 #undef atomic_cas_ulong 1130 #undef atomic_cas_ptr 1131 #undef atomic_cas_64 1132 #undef atomic_cas_32_ni 1133 #undef atomic_cas_uint_ni 1134 #undef atomic_cas_ulong_ni 1135 #undef atomic_cas_ptr_ni 1136 #undef atomic_cas_64_ni 1137 #undef atomic_swap_32 1138 #undef atomic_swap_uint 1139 #undef atomic_swap_ulong 1140 #undef atomic_swap_ptr 1141 #undef atomic_swap_64 1142 #undef atomic_dec_32 1143 #undef atomic_dec_uint 1144 #undef atomic_dec_ulong 1145 #undef atomic_dec_ptr 1146 #undef atomic_dec_64 1147 #undef atomic_dec_32_nv 1148 #undef atomic_dec_uint_nv 1149 #undef atomic_dec_ulong_nv 1150 #undef atomic_dec_ptr_nv 1151 #undef atomic_dec_64_nv 1152 #undef atomic_inc_32 1153 #undef atomic_inc_uint 1154 #undef atomic_inc_ulong 1155 #undef atomic_inc_ptr 1156 #undef atomic_inc_64 1157 #undef atomic_inc_32_nv 1158 #undef atomic_inc_uint_nv 1159 #undef atomic_inc_ulong_nv 1160 #undef atomic_inc_ptr_nv 1161 #undef atomic_inc_64_nv 1162 1163 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1164 void atomic_add_##name(volatile targ1 *, targ2); \ 1165 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1166 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1167 { \ 1168 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1169 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1170 __func__); \ 1171 atomic_add_##name(ptr, val); \ 1172 } \ 1173 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1174 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1175 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1176 { \ 1177 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1178 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1179 __func__); \ 1180 kmsan_init_ret(sizeof(tret)); \ 1181 return atomic_add_##name##_nv(ptr, val); \ 1182 } 1183 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1184 void atomic_and_##name(volatile targ1 *, targ2); \ 1185 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1186 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1187 { \ 1188 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1189 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1190 __func__); \ 1191 atomic_and_##name(ptr, val); \ 1192 } \ 1193 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1194 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1195 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1196 { \ 1197 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1198 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1199 __func__); \ 1200 kmsan_init_ret(sizeof(tret)); \ 1201 return atomic_and_##name##_nv(ptr, val); \ 1202 } 1203 1204 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1205 void atomic_or_##name(volatile targ1 *, targ2); \ 1206 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1207 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1208 { \ 1209 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1210 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1211 __func__); \ 1212 atomic_or_##name(ptr, val); \ 1213 } \ 1214 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1215 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1216 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1217 { \ 1218 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1219 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1220 __func__); \ 1221 kmsan_init_ret(sizeof(tret)); \ 1222 return atomic_or_##name##_nv(ptr, val); \ 1223 } 1224 1225 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1226 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1227 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1228 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1229 { \ 1230 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1231 __func__); \ 1232 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1233 __func__); \ 1234 kmsan_init_ret(sizeof(tret)); \ 1235 return atomic_cas_##name(ptr, exp, new); \ 1236 } \ 1237 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1238 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1239 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1240 { \ 1241 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1242 __func__); \ 1243 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1244 __func__); \ 1245 kmsan_init_ret(sizeof(tret)); \ 1246 return atomic_cas_##name##_ni(ptr, exp, new); \ 1247 } 1248 1249 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1250 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1251 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1252 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1253 { \ 1254 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1255 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1256 __func__); \ 1257 kmsan_init_ret(sizeof(tret)); \ 1258 return atomic_swap_##name(ptr, val); \ 1259 } 1260 1261 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1262 void atomic_dec_##name(volatile targ1 *); \ 1263 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1264 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1265 { \ 1266 kmsan_check_arg(sizeof(ptr), __func__); \ 1267 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1268 __func__); \ 1269 atomic_dec_##name(ptr); \ 1270 } \ 1271 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1272 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1273 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1274 { \ 1275 kmsan_check_arg(sizeof(ptr), __func__); \ 1276 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1277 __func__); \ 1278 kmsan_init_ret(sizeof(tret)); \ 1279 return atomic_dec_##name##_nv(ptr); \ 1280 } 1281 1282 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1283 void atomic_inc_##name(volatile targ1 *); \ 1284 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1285 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1286 { \ 1287 kmsan_check_arg(sizeof(ptr), __func__); \ 1288 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1289 __func__); \ 1290 atomic_inc_##name(ptr); \ 1291 } \ 1292 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1293 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1294 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1295 { \ 1296 kmsan_check_arg(sizeof(ptr), __func__); \ 1297 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1298 __func__); \ 1299 kmsan_init_ret(sizeof(tret)); \ 1300 return atomic_inc_##name##_nv(ptr); \ 1301 } 1302 1303 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1304 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1305 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1306 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1307 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1308 1309 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1310 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1311 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1312 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1313 1314 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1315 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1316 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1317 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1318 1319 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1320 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1321 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1322 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1323 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1324 1325 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1326 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1327 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1328 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1329 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1330 1331 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1332 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1333 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1334 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1335 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1336 1337 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1338 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1339 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1340 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1341 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1342 1343 /* -------------------------------------------------------------------------- */ 1344 1345 #include <sys/bus.h> 1346 1347 #undef bus_space_read_multi_1 1348 #undef bus_space_read_multi_2 1349 #undef bus_space_read_multi_4 1350 #undef bus_space_read_multi_8 1351 #undef bus_space_read_multi_stream_1 1352 #undef bus_space_read_multi_stream_2 1353 #undef bus_space_read_multi_stream_4 1354 #undef bus_space_read_multi_stream_8 1355 #undef bus_space_read_region_1 1356 #undef bus_space_read_region_2 1357 #undef bus_space_read_region_4 1358 #undef bus_space_read_region_8 1359 #undef bus_space_read_region_stream_1 1360 #undef bus_space_read_region_stream_2 1361 #undef bus_space_read_region_stream_4 1362 #undef bus_space_read_region_stream_8 1363 1364 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1365 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1366 bus_size_t, uint##bits##_t *, bus_size_t); \ 1367 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1368 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1369 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1370 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1371 bus_size_t count) \ 1372 { \ 1373 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1374 sizeof(uint##bits##_t) * count); \ 1375 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1376 } \ 1377 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1378 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1379 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1380 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1381 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1382 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1383 bus_size_t count) \ 1384 { \ 1385 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1386 sizeof(uint##bits##_t) * count); \ 1387 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1388 } \ 1389 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1390 bus_size_t, uint##bits##_t *, bus_size_t); \ 1391 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1392 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1393 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1394 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1395 bus_size_t count) \ 1396 { \ 1397 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1398 sizeof(uint##bits##_t) * count); \ 1399 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1400 } \ 1401 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1402 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1403 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1404 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1405 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1406 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1407 bus_size_t count) \ 1408 { \ 1409 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1410 sizeof(uint##bits##_t) * count); \ 1411 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1412 } 1413 1414 MSAN_BUS_READ_FUNC(1, 8) 1415 MSAN_BUS_READ_FUNC(2, 16) 1416 MSAN_BUS_READ_FUNC(4, 32) 1417 MSAN_BUS_READ_FUNC(8, 64) 1418 1419 #undef bus_space_write_multi_1 1420 #undef bus_space_write_multi_2 1421 #undef bus_space_write_multi_4 1422 #undef bus_space_write_multi_8 1423 #undef bus_space_write_multi_stream_1 1424 #undef bus_space_write_multi_stream_2 1425 #undef bus_space_write_multi_stream_4 1426 #undef bus_space_write_multi_stream_8 1427 #undef bus_space_write_region_1 1428 #undef bus_space_write_region_2 1429 #undef bus_space_write_region_4 1430 #undef bus_space_write_region_8 1431 #undef bus_space_write_region_stream_1 1432 #undef bus_space_write_region_stream_2 1433 #undef bus_space_write_region_stream_4 1434 #undef bus_space_write_region_stream_8 1435 1436 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1437 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1438 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1439 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1440 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1441 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1442 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1443 bus_size_t count) \ 1444 { \ 1445 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1446 "bus_space_write"); \ 1447 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1448 } \ 1449 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1450 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1451 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1452 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1453 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1454 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1455 bus_size_t count) \ 1456 { \ 1457 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1458 "bus_space_write"); \ 1459 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1460 } \ 1461 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1462 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1463 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1464 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1465 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1466 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1467 bus_size_t count) \ 1468 { \ 1469 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1470 "bus_space_write"); \ 1471 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1472 } \ 1473 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1474 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1475 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1476 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1477 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1478 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1479 bus_size_t count) \ 1480 { \ 1481 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1482 "bus_space_write"); \ 1483 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1484 } 1485 1486 MSAN_BUS_WRITE_FUNC(1, 8) 1487 MSAN_BUS_WRITE_FUNC(2, 16) 1488 MSAN_BUS_WRITE_FUNC(4, 32) 1489 MSAN_BUS_WRITE_FUNC(8, 64) 1490 1491 /* -------------------------------------------------------------------------- */ 1492 1493 #include <sys/mbuf.h> 1494 1495 static void 1496 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1497 bool init, uintptr_t pc) 1498 { 1499 if (init) { 1500 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1501 } else { 1502 kmsan_shadow_check(buf + offset, len, "dma_sync_linear"); 1503 } 1504 } 1505 1506 static void 1507 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1508 bool init, uintptr_t pc) 1509 { 1510 bus_addr_t minlen; 1511 1512 for (; m != NULL && len != 0; m = m->m_next) { 1513 if (offset >= m->m_len) { 1514 offset -= m->m_len; 1515 continue; 1516 } 1517 1518 minlen = MIN(len, m->m_len - offset); 1519 1520 if (init) { 1521 kmsan_shadow_fill(mtod(m, char *) + offset, 1522 KMSAN_STATE_INITED, minlen); 1523 } else { 1524 kmsan_shadow_check(mtod(m, char *) + offset, 1525 minlen, "dma_sync_mbuf"); 1526 } 1527 1528 offset = 0; 1529 len -= minlen; 1530 } 1531 } 1532 1533 static void 1534 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1535 bool init, uintptr_t pc) 1536 { 1537 bus_size_t minlen, resid; 1538 struct iovec *iov; 1539 int i; 1540 1541 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1542 return; 1543 1544 resid = uio->uio_resid; 1545 iov = uio->uio_iov; 1546 1547 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1548 minlen = MIN(resid, iov[i].iov_len); 1549 1550 if (init) { 1551 kmsan_shadow_fill(iov[i].iov_base, 1552 KMSAN_STATE_INITED, minlen); 1553 } else { 1554 kmsan_shadow_check(iov[i].iov_base, minlen, 1555 "dma_sync_uio"); 1556 } 1557 1558 resid -= minlen; 1559 } 1560 } 1561 1562 void 1563 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1564 { 1565 bool init; 1566 1567 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1568 return; 1569 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1570 1571 switch (map->dm_buftype) { 1572 case KMSAN_DMA_LINEAR: 1573 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1574 __RET_ADDR); 1575 break; 1576 case KMSAN_DMA_MBUF: 1577 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1578 __RET_ADDR); 1579 break; 1580 case KMSAN_DMA_UIO: 1581 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1582 __RET_ADDR); 1583 break; 1584 case KMSAN_DMA_RAW: 1585 break; 1586 default: 1587 panic("%s: impossible", __func__); 1588 } 1589 } 1590 1591 void 1592 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1593 { 1594 map->dm_buf = buf; 1595 map->dm_buflen = buflen; 1596 map->dm_buftype = type; 1597 } 1598
Indexes created Sat Sep 20 22:09:52 GMT 2025