subr_msan.c revision 1.13
1 /* $NetBSD: subr_msan.c,v 1.13 2020/09/05 16:30:12 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.13 2020/09/05 16:30:12 riastradh Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/kprintf.h> 43 #include <sys/kmem.h> 44 #include <sys/mbuf.h> 45 #include <sys/buf.h> 46 #include <sys/cpu.h> 47 #include <sys/msan.h> 48 49 static void kmsan_printf(const char *, ...); 50 51 void kmsan_init_arg(size_t); 52 void kmsan_init_ret(size_t); 53 54 #ifdef KMSAN_PANIC 55 #define REPORT panic 56 #else 57 #define REPORT kmsan_printf 58 #endif 59 60 /* -------------------------------------------------------------------------- */ 61 62 /* 63 * Part of the compiler ABI. 64 */ 65 66 typedef uint32_t msan_orig_t; 67 68 typedef struct { 69 uint8_t *shad; 70 msan_orig_t *orig; 71 } msan_meta_t; 72 73 #define MSAN_PARAM_SIZE 800 74 #define MSAN_RETVAL_SIZE 800 75 typedef struct { 76 uint8_t param[MSAN_PARAM_SIZE]; 77 uint8_t retval[MSAN_RETVAL_SIZE]; 78 uint8_t _va_arg[MSAN_PARAM_SIZE]; 79 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 80 uint64_t va_arg_overflow_size; 81 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 82 msan_orig_t retval_origin; 83 msan_orig_t origin; 84 } msan_tls_t; 85 86 /* -------------------------------------------------------------------------- */ 87 88 /* The MD code. */ 89 #include <machine/msan.h> 90 91 /* -------------------------------------------------------------------------- */ 92 93 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 94 #define MSAN_NCONTEXT 16 95 96 typedef struct { 97 size_t ctx; 98 msan_tls_t tls[MSAN_NCONTEXT]; 99 } msan_lwp_t; 100 101 static msan_tls_t dummy_tls; 102 103 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 104 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 105 static msan_lwp_t msan_lwp0; 106 static bool kmsan_enabled __read_mostly; 107 108 /* -------------------------------------------------------------------------- */ 109 110 static bool kmsan_reporting = false; 111 112 static inline void 113 kmsan_printf(const char *fmt, ...) 114 { 115 va_list ap; 116 117 va_start(ap, fmt); 118 kprintf(fmt, TOCONS, NULL, NULL, ap); 119 va_end(ap); 120 } 121 122 static inline const char * 123 kmsan_orig_name(int type) 124 { 125 switch (type) { 126 case KMSAN_TYPE_STACK: 127 return "Stack"; 128 case KMSAN_TYPE_KMEM: 129 return "Kmem"; 130 case KMSAN_TYPE_MALLOC: 131 return "Malloc"; 132 case KMSAN_TYPE_POOL: 133 return "Pool"; 134 case KMSAN_TYPE_UVM: 135 return "Uvm"; 136 default: 137 return "Unknown"; 138 } 139 } 140 141 /* 142 * The format of the string is: "----var@function". Parse it to display a nice 143 * warning. 144 */ 145 static void 146 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 147 { 148 const char *mod, *sym; 149 extern int db_active; 150 msan_orig_t *orig; 151 const char *typename; 152 char *var, *fn; 153 uintptr_t ptr; 154 char buf[128]; 155 int type; 156 157 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 158 return; 159 160 kmsan_reporting = true; 161 __insn_barrier(); 162 163 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 164 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 165 166 if (*orig == 0) { 167 REPORT("MSan: Uninitialized Memory In %s At Offset " 168 "%zu\n", hook, off); 169 goto out; 170 } 171 172 kmsan_md_orig_decode(*orig, &type, &ptr); 173 typename = kmsan_orig_name(type); 174 175 if (kmsan_md_is_pc(ptr)) { 176 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 177 REPORT("MSan: Uninitialized %s Memory In %s " 178 "At Offset %zu/%zu, IP %p\n", typename, hook, off, 179 size, (void *)ptr); 180 } else { 181 REPORT("MSan: Uninitialized %s Memory In %s " 182 "At Offset %zu/%zu, From %s()\n", typename, hook, 183 off, size, sym); 184 } 185 } else { 186 var = (char *)ptr + 4; 187 strlcpy(buf, var, sizeof(buf)); 188 var = buf; 189 fn = __builtin_strchr(buf, '@'); 190 *fn++ = '\0'; 191 REPORT("MSan: Uninitialized %s Memory In %s At Offset " 192 "%zu, Variable '%s' From %s()\n", typename, hook, off, 193 var, fn); 194 } 195 196 out: 197 kmsan_md_unwind(); 198 __insn_barrier(); 199 kmsan_reporting = false; 200 } 201 202 static void 203 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 204 { 205 const char *mod, *sym; 206 extern int db_active; 207 const char *typename; 208 char *var, *fn; 209 uintptr_t ptr; 210 char buf[128]; 211 int type; 212 213 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 214 return; 215 216 kmsan_reporting = true; 217 __insn_barrier(); 218 219 if (orig == 0) { 220 REPORT("MSan: Uninitialized Variable In %p\n", 221 (void *)pc); 222 goto out; 223 } 224 225 kmsan_md_orig_decode(orig, &type, &ptr); 226 typename = kmsan_orig_name(type); 227 228 if (kmsan_md_is_pc(ptr)) { 229 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 230 REPORT("MSan: Uninitialized %s Memory, " 231 "Origin %x\n", typename, orig); 232 } else { 233 REPORT("MSan: Uninitialized %s Memory " 234 "From %s()\n", typename, sym); 235 } 236 } else { 237 var = (char *)ptr + 4; 238 strlcpy(buf, var, sizeof(buf)); 239 var = buf; 240 fn = __builtin_strchr(buf, '@'); 241 *fn++ = '\0'; 242 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 243 var, fn); 244 } 245 246 out: 247 kmsan_md_unwind(); 248 __insn_barrier(); 249 kmsan_reporting = false; 250 } 251 252 /* -------------------------------------------------------------------------- */ 253 254 static inline msan_meta_t 255 kmsan_meta_get(void *addr, size_t size) 256 { 257 msan_meta_t ret; 258 259 if (__predict_false(!kmsan_enabled)) { 260 ret.shad = msan_dummy_shad; 261 ret.orig = (msan_orig_t *)msan_dummy_orig; 262 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 263 ret.shad = msan_dummy_shad; 264 ret.orig = (msan_orig_t *)msan_dummy_orig; 265 } else { 266 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 267 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 268 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 269 } 270 271 return ret; 272 } 273 274 static inline void 275 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 276 { 277 msan_orig_t *orig; 278 size_t i; 279 280 if (__predict_false(!kmsan_enabled)) 281 return; 282 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 283 return; 284 285 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 286 size += ((uintptr_t)orig & 0x3); 287 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 288 289 for (i = 0; i < size; i += 4) { 290 orig[i / 4] = o; 291 } 292 } 293 294 static inline void 295 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 296 { 297 uint8_t *shad; 298 299 if (__predict_false(!kmsan_enabled)) 300 return; 301 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 302 return; 303 304 shad = kmsan_md_addr_to_shad(addr); 305 __builtin_memset(shad, c, size); 306 } 307 308 static inline void 309 kmsan_meta_copy(void *dst, const void *src, size_t size) 310 { 311 uint8_t *orig_src, *orig_dst; 312 uint8_t *shad_src, *shad_dst; 313 msan_orig_t *_src, *_dst; 314 size_t i; 315 316 if (__predict_false(!kmsan_enabled)) 317 return; 318 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 319 return; 320 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 321 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 322 return; 323 } 324 325 shad_src = kmsan_md_addr_to_shad(src); 326 shad_dst = kmsan_md_addr_to_shad(dst); 327 __builtin_memmove(shad_dst, shad_src, size); 328 329 orig_src = kmsan_md_addr_to_orig(src); 330 orig_dst = kmsan_md_addr_to_orig(dst); 331 for (i = 0; i < size; i++) { 332 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 333 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 334 *_dst = *_src; 335 orig_src++; 336 orig_dst++; 337 } 338 } 339 340 static inline void 341 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 342 { 343 uint8_t *shad; 344 size_t i; 345 346 if (__predict_false(!kmsan_enabled)) 347 return; 348 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 349 return; 350 351 shad = kmsan_md_addr_to_shad(addr); 352 for (i = 0; i < size; i++) { 353 if (__predict_true(shad[i] == 0)) 354 continue; 355 kmsan_report_hook((const char *)addr + i, size, i, hook); 356 break; 357 } 358 } 359 360 void 361 kmsan_init_arg(size_t n) 362 { 363 msan_lwp_t *lwp; 364 uint8_t *arg; 365 366 if (__predict_false(!kmsan_enabled)) 367 return; 368 lwp = curlwp->l_kmsan; 369 arg = lwp->tls[lwp->ctx].param; 370 __builtin_memset(arg, 0, n); 371 } 372 373 void 374 kmsan_init_ret(size_t n) 375 { 376 msan_lwp_t *lwp; 377 uint8_t *arg; 378 379 if (__predict_false(!kmsan_enabled)) 380 return; 381 lwp = curlwp->l_kmsan; 382 arg = lwp->tls[lwp->ctx].retval; 383 __builtin_memset(arg, 0, n); 384 } 385 386 static void 387 kmsan_check_arg(size_t size, const char *hook) 388 { 389 msan_lwp_t *lwp; 390 uint8_t *arg; 391 size_t i; 392 393 if (__predict_false(!kmsan_enabled)) 394 return; 395 lwp = curlwp->l_kmsan; 396 arg = lwp->tls[lwp->ctx].param; 397 398 for (i = 0; i < size; i++) { 399 if (__predict_true(arg[i] == 0)) 400 continue; 401 kmsan_report_hook((const char *)arg + i, size, i, hook); 402 break; 403 } 404 } 405 406 void 407 kmsan_lwp_alloc(struct lwp *l) 408 { 409 msan_lwp_t *lwp; 410 411 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 412 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 413 lwp->ctx = 1; 414 415 l->l_kmsan = lwp; 416 } 417 418 void 419 kmsan_lwp_free(struct lwp *l) 420 { 421 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 422 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 423 } 424 425 void kmsan_intr_enter(void); 426 void kmsan_intr_leave(void); 427 void kmsan_softint(struct lwp *); 428 429 void 430 kmsan_intr_enter(void) 431 { 432 msan_lwp_t *lwp; 433 434 if (__predict_false(!kmsan_enabled)) 435 return; 436 lwp = curlwp->l_kmsan; 437 438 lwp->ctx++; 439 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 440 kmsan_enabled = false; 441 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 442 } 443 444 kmsan_init_arg(sizeof(void *)); 445 } 446 447 void 448 kmsan_intr_leave(void) 449 { 450 msan_lwp_t *lwp; 451 452 if (__predict_false(!kmsan_enabled)) 453 return; 454 lwp = curlwp->l_kmsan; 455 456 if (__predict_false(lwp->ctx == 0)) { 457 kmsan_enabled = false; 458 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 459 } 460 lwp->ctx--; 461 } 462 463 void 464 kmsan_softint(struct lwp *l) 465 { 466 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 467 } 468 469 /* -------------------------------------------------------------------------- */ 470 471 void 472 kmsan_shadow_map(void *addr, size_t size) 473 { 474 size_t npages, i; 475 vaddr_t va; 476 477 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 478 KASSERT(size % PAGE_SIZE == 0); 479 480 npages = size / PAGE_SIZE; 481 482 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 483 for (i = 0; i < npages; i++) { 484 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 485 } 486 487 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 488 for (i = 0; i < npages; i++) { 489 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 490 } 491 } 492 493 void 494 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 495 { 496 msan_orig_t orig; 497 498 orig = kmsan_md_orig_encode(type, pc); 499 kmsan_origin_fill(addr, orig, size); 500 } 501 502 void 503 kmsan_mark(void *addr, size_t size, uint8_t c) 504 { 505 kmsan_shadow_fill(addr, c, size); 506 } 507 508 void 509 kmsan_check_mbuf(void *buf) 510 { 511 struct mbuf *m = buf; 512 513 do { 514 kmsan_shadow_check(mtod(m, void *), m->m_len, "MbufChain"); 515 } while ((m = m->m_next) != NULL); 516 } 517 518 void 519 kmsan_check_buf(void *buf) 520 { 521 buf_t *bp = buf; 522 523 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()"); 524 } 525 526 void 527 kmsan_init(void *stack) 528 { 529 /* MD initialization. */ 530 kmsan_md_init(); 531 532 /* Map the stack. */ 533 kmsan_shadow_map(stack, USPACE); 534 535 /* Initialize the TLS for curlwp. */ 536 msan_lwp0.ctx = 1; 537 curlwp->l_kmsan = &msan_lwp0; 538 539 /* Now officially enabled. */ 540 kmsan_enabled = true; 541 } 542 543 /* -------------------------------------------------------------------------- */ 544 545 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 546 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 547 548 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 549 { 550 return kmsan_meta_get(addr, size); 551 } 552 553 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 554 { 555 return kmsan_meta_get(addr, size); 556 } 557 558 #define MSAN_META_FUNC(size) \ 559 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 560 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 561 { \ 562 return kmsan_meta_get(addr, size); \ 563 } \ 564 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 565 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 566 { \ 567 return kmsan_meta_get(addr, size); \ 568 } 569 570 MSAN_META_FUNC(1) 571 MSAN_META_FUNC(2) 572 MSAN_META_FUNC(4) 573 MSAN_META_FUNC(8) 574 575 void __msan_instrument_asm_store(void *, size_t); 576 msan_orig_t __msan_chain_origin(msan_orig_t); 577 void __msan_poison_alloca(void *, uint64_t, char *); 578 void __msan_unpoison_alloca(void *, uint64_t); 579 void __msan_warning(msan_orig_t); 580 msan_tls_t *__msan_get_context_state(void); 581 582 void __msan_instrument_asm_store(void *addr, size_t size) 583 { 584 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 585 } 586 587 msan_orig_t __msan_chain_origin(msan_orig_t origin) 588 { 589 return origin; 590 } 591 592 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 593 { 594 msan_orig_t orig; 595 596 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 597 kmsan_origin_fill(addr, orig, size); 598 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 599 } 600 601 void __msan_unpoison_alloca(void *addr, uint64_t size) 602 { 603 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 604 } 605 606 void __msan_warning(msan_orig_t origin) 607 { 608 if (__predict_false(!kmsan_enabled)) 609 return; 610 kmsan_report_inline(origin, __RET_ADDR); 611 } 612 613 msan_tls_t *__msan_get_context_state(void) 614 { 615 msan_lwp_t *lwp; 616 617 if (__predict_false(!kmsan_enabled)) 618 return &dummy_tls; 619 lwp = curlwp->l_kmsan; 620 621 return &lwp->tls[lwp->ctx]; 622 } 623 624 /* -------------------------------------------------------------------------- */ 625 626 /* 627 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 628 * initialized areas properly. 629 */ 630 631 void *kmsan_memcpy(void *dst, const void *src, size_t len) 632 { 633 /* No kmsan_check_arg, because inlined. */ 634 kmsan_init_ret(sizeof(void *)); 635 if (__predict_true(len != 0)) { 636 kmsan_meta_copy(dst, src, len); 637 } 638 return __builtin_memcpy(dst, src, len); 639 } 640 641 int 642 kmsan_memcmp(const void *b1, const void *b2, size_t len) 643 { 644 const uint8_t *_b1 = b1, *_b2 = b2; 645 size_t i; 646 647 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 648 "memcmp():args"); 649 kmsan_init_ret(sizeof(int)); 650 651 for (i = 0; i < len; i++) { 652 if (*_b1 != *_b2) { 653 kmsan_shadow_check(b1, i + 1, "memcmp():arg1"); 654 kmsan_shadow_check(b2, i + 1, "memcmp():arg2"); 655 return *_b1 - *_b2; 656 } 657 _b1++, _b2++; 658 } 659 660 return 0; 661 } 662 663 void *kmsan_memset(void *dst, int c, size_t len) 664 { 665 /* No kmsan_check_arg, because inlined. */ 666 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 667 kmsan_init_ret(sizeof(void *)); 668 return __builtin_memset(dst, c, len); 669 } 670 671 void *kmsan_memmove(void *dst, const void *src, size_t len) 672 { 673 /* No kmsan_check_arg, because inlined. */ 674 if (__predict_true(len != 0)) { 675 kmsan_meta_copy(dst, src, len); 676 } 677 kmsan_init_ret(sizeof(void *)); 678 return __builtin_memmove(dst, src, len); 679 } 680 681 __strong_alias(__msan_memcpy, kmsan_memcpy) 682 __strong_alias(__msan_memset, kmsan_memset) 683 __strong_alias(__msan_memmove, kmsan_memmove) 684 685 char * 686 kmsan_strcpy(char *dst, const char *src) 687 { 688 const char *_src = src; 689 char *_dst = dst; 690 size_t len = 0; 691 692 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 693 694 while (1) { 695 len++; 696 *dst = *src; 697 if (*src == '\0') 698 break; 699 src++, dst++; 700 } 701 702 kmsan_shadow_check(_src, len, "strcpy():arg2"); 703 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 704 kmsan_init_ret(sizeof(char *)); 705 return _dst; 706 } 707 708 int 709 kmsan_strcmp(const char *s1, const char *s2) 710 { 711 const char *_s1 = s1, *_s2 = s2; 712 size_t len = 0; 713 714 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 715 kmsan_init_ret(sizeof(int)); 716 717 while (1) { 718 len++; 719 if (*s1 != *s2) 720 break; 721 if (*s1 == '\0') { 722 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 723 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 724 return 0; 725 } 726 s1++, s2++; 727 } 728 729 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 730 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 731 732 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 733 } 734 735 size_t 736 kmsan_strlen(const char *str) 737 { 738 const char *s; 739 740 kmsan_check_arg(sizeof(str), "strlen():args"); 741 742 s = str; 743 while (1) { 744 if (*s == '\0') 745 break; 746 s++; 747 } 748 749 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1"); 750 kmsan_init_ret(sizeof(size_t)); 751 return (s - str); 752 } 753 754 char * 755 kmsan_strcat(char *dst, const char *src) 756 { 757 size_t ldst, lsrc; 758 char *ret; 759 760 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args"); 761 762 ldst = __builtin_strlen(dst); 763 lsrc = __builtin_strlen(src); 764 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1"); 765 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2"); 766 ret = __builtin_strcat(dst, src); 767 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1); 768 769 kmsan_init_ret(sizeof(char *)); 770 return ret; 771 } 772 773 char * 774 kmsan_strchr(const char *s, int c) 775 { 776 char *ret; 777 778 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args"); 779 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1"); 780 ret = __builtin_strchr(s, c); 781 782 kmsan_init_ret(sizeof(char *)); 783 return ret; 784 } 785 786 char * 787 kmsan_strrchr(const char *s, int c) 788 { 789 char *ret; 790 791 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args"); 792 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1"); 793 ret = __builtin_strrchr(s, c); 794 795 kmsan_init_ret(sizeof(char *)); 796 return ret; 797 } 798 799 #undef kcopy 800 #undef copyin 801 #undef copyout 802 #undef copyinstr 803 #undef copyoutstr 804 805 int kmsan_kcopy(const void *, void *, size_t); 806 int kmsan_copyin(const void *, void *, size_t); 807 int kmsan_copyout(const void *, void *, size_t); 808 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 809 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 810 811 int kcopy(const void *, void *, size_t); 812 int copyin(const void *, void *, size_t); 813 int copyout(const void *, void *, size_t); 814 int copyinstr(const void *, void *, size_t, size_t *); 815 int copyoutstr(const void *, void *, size_t, size_t *); 816 817 int 818 kmsan_kcopy(const void *src, void *dst, size_t len) 819 { 820 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), 821 "kcopy():args"); 822 if (__predict_true(len != 0)) { 823 kmsan_meta_copy(dst, src, len); 824 } 825 kmsan_init_ret(sizeof(int)); 826 return kcopy(src, dst, len); 827 } 828 829 int 830 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 831 { 832 int ret; 833 834 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 835 "copyin():args"); 836 ret = copyin(uaddr, kaddr, len); 837 if (ret == 0) 838 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 839 kmsan_init_ret(sizeof(int)); 840 841 return ret; 842 } 843 844 int 845 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 846 { 847 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 848 "copyout():args"); 849 kmsan_shadow_check(kaddr, len, "copyout():arg1"); 850 kmsan_init_ret(sizeof(int)); 851 return copyout(kaddr, uaddr, len); 852 } 853 854 int 855 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 856 { 857 size_t _done; 858 int ret; 859 860 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 861 sizeof(len) + sizeof(done), "copyinstr():args"); 862 ret = copyinstr(uaddr, kaddr, len, &_done); 863 if (ret == 0 || ret == ENAMETOOLONG) 864 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 865 if (done != NULL) { 866 *done = _done; 867 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 868 } 869 kmsan_init_ret(sizeof(int)); 870 871 return ret; 872 } 873 874 int 875 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 876 { 877 size_t _done; 878 int ret; 879 880 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 881 sizeof(len) + sizeof(done), "copyoutstr():args"); 882 ret = copyoutstr(kaddr, uaddr, len, &_done); 883 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1"); 884 if (done != NULL) { 885 *done = _done; 886 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 887 } 888 kmsan_init_ret(sizeof(int)); 889 890 return ret; 891 } 892 893 /* -------------------------------------------------------------------------- */ 894 895 #undef _ucas_32 896 #undef _ucas_32_mp 897 #undef _ucas_64 898 #undef _ucas_64_mp 899 #undef _ufetch_8 900 #undef _ufetch_16 901 #undef _ufetch_32 902 #undef _ufetch_64 903 #undef _ustore_8 904 #undef _ustore_16 905 #undef _ustore_32 906 #undef _ustore_64 907 908 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 909 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 910 int 911 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 912 uint32_t *ret) 913 { 914 int _ret; 915 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 916 sizeof(new) + sizeof(ret), "ucas_32():args"); 917 _ret = _ucas_32(uaddr, old, new, ret); 918 if (_ret == 0) 919 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 920 kmsan_init_ret(sizeof(int)); 921 return _ret; 922 } 923 924 #ifdef __HAVE_UCAS_MP 925 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 926 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 927 int 928 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 929 uint32_t *ret) 930 { 931 int _ret; 932 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 933 sizeof(new) + sizeof(ret), "ucas_32_mp():args"); 934 _ret = _ucas_32_mp(uaddr, old, new, ret); 935 if (_ret == 0) 936 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 937 kmsan_init_ret(sizeof(int)); 938 return _ret; 939 } 940 #endif 941 942 #ifdef _LP64 943 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 944 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 945 int 946 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 947 uint64_t *ret) 948 { 949 int _ret; 950 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 951 sizeof(new) + sizeof(ret), "ucas_64():args"); 952 _ret = _ucas_64(uaddr, old, new, ret); 953 if (_ret == 0) 954 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 955 kmsan_init_ret(sizeof(int)); 956 return _ret; 957 } 958 959 #ifdef __HAVE_UCAS_MP 960 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 961 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 962 int 963 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 964 uint64_t *ret) 965 { 966 int _ret; 967 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 968 sizeof(new) + sizeof(ret), "ucas_64_mp():args"); 969 _ret = _ucas_64_mp(uaddr, old, new, ret); 970 if (_ret == 0) 971 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 972 kmsan_init_ret(sizeof(int)); 973 return _ret; 974 } 975 #endif 976 #endif 977 978 int _ufetch_8(const uint8_t *, uint8_t *); 979 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 980 int 981 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 982 { 983 int _ret; 984 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args"); 985 _ret = _ufetch_8(uaddr, valp); 986 if (_ret == 0) 987 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 988 kmsan_init_ret(sizeof(int)); 989 return _ret; 990 } 991 992 int _ufetch_16(const uint16_t *, uint16_t *); 993 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 994 int 995 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 996 { 997 int _ret; 998 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args"); 999 _ret = _ufetch_16(uaddr, valp); 1000 if (_ret == 0) 1001 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1002 kmsan_init_ret(sizeof(int)); 1003 return _ret; 1004 } 1005 1006 int _ufetch_32(const uint32_t *, uint32_t *); 1007 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 1008 int 1009 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 1010 { 1011 int _ret; 1012 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args"); 1013 _ret = _ufetch_32(uaddr, valp); 1014 if (_ret == 0) 1015 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1016 kmsan_init_ret(sizeof(int)); 1017 return _ret; 1018 } 1019 1020 #ifdef _LP64 1021 int _ufetch_64(const uint64_t *, uint64_t *); 1022 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1023 int 1024 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1025 { 1026 int _ret; 1027 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args"); 1028 _ret = _ufetch_64(uaddr, valp); 1029 if (_ret == 0) 1030 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1031 kmsan_init_ret(sizeof(int)); 1032 return _ret; 1033 } 1034 #endif 1035 1036 int _ustore_8(uint8_t *, uint8_t); 1037 int kmsan__ustore_8(uint8_t *, uint8_t); 1038 int 1039 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1040 { 1041 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args"); 1042 kmsan_init_ret(sizeof(int)); 1043 return _ustore_8(uaddr, val); 1044 } 1045 1046 int _ustore_16(uint16_t *, uint16_t); 1047 int kmsan__ustore_16(uint16_t *, uint16_t); 1048 int 1049 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1050 { 1051 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args"); 1052 kmsan_init_ret(sizeof(int)); 1053 return _ustore_16(uaddr, val); 1054 } 1055 1056 int _ustore_32(uint32_t *, uint32_t); 1057 int kmsan__ustore_32(uint32_t *, uint32_t); 1058 int 1059 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1060 { 1061 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args"); 1062 kmsan_init_ret(sizeof(int)); 1063 return _ustore_32(uaddr, val); 1064 } 1065 1066 #ifdef _LP64 1067 int _ustore_64(uint64_t *, uint64_t); 1068 int kmsan__ustore_64(uint64_t *, uint64_t); 1069 int 1070 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1071 { 1072 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args"); 1073 kmsan_init_ret(sizeof(int)); 1074 return _ustore_64(uaddr, val); 1075 } 1076 #endif 1077 1078 /* -------------------------------------------------------------------------- */ 1079 1080 #undef atomic_add_32 1081 #undef atomic_add_int 1082 #undef atomic_add_long 1083 #undef atomic_add_ptr 1084 #undef atomic_add_64 1085 #undef atomic_add_32_nv 1086 #undef atomic_add_int_nv 1087 #undef atomic_add_long_nv 1088 #undef atomic_add_ptr_nv 1089 #undef atomic_add_64_nv 1090 #undef atomic_and_32 1091 #undef atomic_and_uint 1092 #undef atomic_and_ulong 1093 #undef atomic_and_64 1094 #undef atomic_and_32_nv 1095 #undef atomic_and_uint_nv 1096 #undef atomic_and_ulong_nv 1097 #undef atomic_and_64_nv 1098 #undef atomic_or_32 1099 #undef atomic_or_uint 1100 #undef atomic_or_ulong 1101 #undef atomic_or_64 1102 #undef atomic_or_32_nv 1103 #undef atomic_or_uint_nv 1104 #undef atomic_or_ulong_nv 1105 #undef atomic_or_64_nv 1106 #undef atomic_cas_32 1107 #undef atomic_cas_uint 1108 #undef atomic_cas_ulong 1109 #undef atomic_cas_ptr 1110 #undef atomic_cas_64 1111 #undef atomic_cas_32_ni 1112 #undef atomic_cas_uint_ni 1113 #undef atomic_cas_ulong_ni 1114 #undef atomic_cas_ptr_ni 1115 #undef atomic_cas_64_ni 1116 #undef atomic_swap_32 1117 #undef atomic_swap_uint 1118 #undef atomic_swap_ulong 1119 #undef atomic_swap_ptr 1120 #undef atomic_swap_64 1121 #undef atomic_dec_32 1122 #undef atomic_dec_uint 1123 #undef atomic_dec_ulong 1124 #undef atomic_dec_ptr 1125 #undef atomic_dec_64 1126 #undef atomic_dec_32_nv 1127 #undef atomic_dec_uint_nv 1128 #undef atomic_dec_ulong_nv 1129 #undef atomic_dec_ptr_nv 1130 #undef atomic_dec_64_nv 1131 #undef atomic_inc_32 1132 #undef atomic_inc_uint 1133 #undef atomic_inc_ulong 1134 #undef atomic_inc_ptr 1135 #undef atomic_inc_64 1136 #undef atomic_inc_32_nv 1137 #undef atomic_inc_uint_nv 1138 #undef atomic_inc_ulong_nv 1139 #undef atomic_inc_ptr_nv 1140 #undef atomic_inc_64_nv 1141 1142 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1143 void atomic_add_##name(volatile targ1 *, targ2); \ 1144 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1145 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1146 { \ 1147 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1148 "atomic_add_" #name "():args"); \ 1149 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1150 "atomic_add_" #name "():arg1"); \ 1151 atomic_add_##name(ptr, val); \ 1152 } \ 1153 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1154 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1155 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1156 { \ 1157 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1158 "atomic_add_" #name "_nv():args"); \ 1159 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1160 "atomic_add_" #name "_nv():arg1"); \ 1161 kmsan_init_ret(sizeof(tret)); \ 1162 return atomic_add_##name##_nv(ptr, val); \ 1163 } 1164 1165 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1166 void atomic_and_##name(volatile targ1 *, targ2); \ 1167 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1168 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1169 { \ 1170 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1171 "atomic_and_" #name "():args"); \ 1172 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1173 "atomic_and_" #name "():arg1"); \ 1174 atomic_and_##name(ptr, val); \ 1175 } \ 1176 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1177 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1178 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1179 { \ 1180 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1181 "atomic_and_" #name "_nv():args"); \ 1182 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1183 "atomic_and_" #name "_nv():arg1"); \ 1184 kmsan_init_ret(sizeof(tret)); \ 1185 return atomic_and_##name##_nv(ptr, val); \ 1186 } 1187 1188 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1189 void atomic_or_##name(volatile targ1 *, targ2); \ 1190 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1191 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1192 { \ 1193 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1194 "atomic_or_" #name "():args"); \ 1195 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1196 "atomic_or_" #name "():arg1"); \ 1197 atomic_or_##name(ptr, val); \ 1198 } \ 1199 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1200 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1201 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1202 { \ 1203 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1204 "atomic_or_" #name "_nv():args"); \ 1205 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1206 "atomic_or_" #name "_nv():arg1"); \ 1207 kmsan_init_ret(sizeof(tret)); \ 1208 return atomic_or_##name##_nv(ptr, val); \ 1209 } 1210 1211 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1212 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1213 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1214 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1215 { \ 1216 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1217 "atomic_cas_" #name "():args"); \ 1218 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1219 "atomic_cas_" #name "():arg1"); \ 1220 kmsan_init_ret(sizeof(tret)); \ 1221 return atomic_cas_##name(ptr, exp, new); \ 1222 } \ 1223 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1224 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1225 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1226 { \ 1227 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1228 "atomic_cas_" #name "_ni():args"); \ 1229 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1230 "atomic_cas_" #name "_ni():arg1"); \ 1231 kmsan_init_ret(sizeof(tret)); \ 1232 return atomic_cas_##name##_ni(ptr, exp, new); \ 1233 } 1234 1235 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1236 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1237 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1238 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1239 { \ 1240 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1241 "atomic_swap_" #name "():args"); \ 1242 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1243 "atomic_swap_" #name "():arg1"); \ 1244 kmsan_init_ret(sizeof(tret)); \ 1245 return atomic_swap_##name(ptr, val); \ 1246 } 1247 1248 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1249 void atomic_dec_##name(volatile targ1 *); \ 1250 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1251 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1252 { \ 1253 kmsan_check_arg(sizeof(ptr), \ 1254 "atomic_dec_" #name "():args"); \ 1255 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1256 "atomic_dec_" #name "():arg1"); \ 1257 atomic_dec_##name(ptr); \ 1258 } \ 1259 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1260 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1261 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1262 { \ 1263 kmsan_check_arg(sizeof(ptr), \ 1264 "atomic_dec_" #name "_nv():args"); \ 1265 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1266 "atomic_dec_" #name "_nv():arg1"); \ 1267 kmsan_init_ret(sizeof(tret)); \ 1268 return atomic_dec_##name##_nv(ptr); \ 1269 } 1270 1271 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1272 void atomic_inc_##name(volatile targ1 *); \ 1273 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1274 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1275 { \ 1276 kmsan_check_arg(sizeof(ptr), \ 1277 "atomic_inc_" #name "():args"); \ 1278 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1279 "atomic_inc_" #name "():arg1"); \ 1280 atomic_inc_##name(ptr); \ 1281 } \ 1282 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1283 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1284 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1285 { \ 1286 kmsan_check_arg(sizeof(ptr), \ 1287 "atomic_inc_" #name "_nv():args"); \ 1288 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1289 "atomic_inc_" #name "_nv():arg1"); \ 1290 kmsan_init_ret(sizeof(tret)); \ 1291 return atomic_inc_##name##_nv(ptr); \ 1292 } 1293 1294 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1295 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1296 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1297 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1298 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1299 1300 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1301 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1302 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1303 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1304 1305 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1306 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1307 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1308 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1309 1310 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1311 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1312 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1313 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1314 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1315 1316 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1317 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1318 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1319 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1320 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1321 1322 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1323 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1324 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1325 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1326 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1327 1328 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1329 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1330 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1331 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1332 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1333 1334 /* -------------------------------------------------------------------------- */ 1335 1336 #include <sys/bus.h> 1337 1338 #undef bus_space_read_multi_1 1339 #undef bus_space_read_multi_2 1340 #undef bus_space_read_multi_4 1341 #undef bus_space_read_multi_8 1342 #undef bus_space_read_multi_stream_1 1343 #undef bus_space_read_multi_stream_2 1344 #undef bus_space_read_multi_stream_4 1345 #undef bus_space_read_multi_stream_8 1346 #undef bus_space_read_region_1 1347 #undef bus_space_read_region_2 1348 #undef bus_space_read_region_4 1349 #undef bus_space_read_region_8 1350 #undef bus_space_read_region_stream_1 1351 #undef bus_space_read_region_stream_2 1352 #undef bus_space_read_region_stream_4 1353 #undef bus_space_read_region_stream_8 1354 1355 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1356 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1357 bus_size_t, uint##bits##_t *, bus_size_t); \ 1358 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1359 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1360 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1361 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1362 bus_size_t count) \ 1363 { \ 1364 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1365 sizeof(uint##bits##_t) * count); \ 1366 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1367 } \ 1368 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1369 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1370 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1371 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1372 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1373 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1374 bus_size_t count) \ 1375 { \ 1376 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1377 sizeof(uint##bits##_t) * count); \ 1378 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1379 } \ 1380 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1381 bus_size_t, uint##bits##_t *, bus_size_t); \ 1382 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1383 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1384 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1385 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1386 bus_size_t count) \ 1387 { \ 1388 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1389 sizeof(uint##bits##_t) * count); \ 1390 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1391 } \ 1392 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1393 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1394 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1395 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1396 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1397 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1398 bus_size_t count) \ 1399 { \ 1400 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1401 sizeof(uint##bits##_t) * count); \ 1402 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1403 } 1404 1405 MSAN_BUS_READ_FUNC(1, 8) 1406 MSAN_BUS_READ_FUNC(2, 16) 1407 MSAN_BUS_READ_FUNC(4, 32) 1408 MSAN_BUS_READ_FUNC(8, 64) 1409 1410 #undef bus_space_write_multi_1 1411 #undef bus_space_write_multi_2 1412 #undef bus_space_write_multi_4 1413 #undef bus_space_write_multi_8 1414 #undef bus_space_write_multi_stream_1 1415 #undef bus_space_write_multi_stream_2 1416 #undef bus_space_write_multi_stream_4 1417 #undef bus_space_write_multi_stream_8 1418 #undef bus_space_write_region_1 1419 #undef bus_space_write_region_2 1420 #undef bus_space_write_region_4 1421 #undef bus_space_write_region_8 1422 #undef bus_space_write_region_stream_1 1423 #undef bus_space_write_region_stream_2 1424 #undef bus_space_write_region_stream_4 1425 #undef bus_space_write_region_stream_8 1426 1427 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1428 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1429 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1430 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1431 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1432 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1433 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1434 bus_size_t count) \ 1435 { \ 1436 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1437 "bus_space_write()"); \ 1438 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1439 } \ 1440 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1441 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1442 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1443 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1444 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1445 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1446 bus_size_t count) \ 1447 { \ 1448 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1449 "bus_space_write()"); \ 1450 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1451 } \ 1452 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1453 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1454 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1455 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1456 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1457 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1458 bus_size_t count) \ 1459 { \ 1460 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1461 "bus_space_write()"); \ 1462 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1463 } \ 1464 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1465 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1466 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1467 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1468 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1469 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1470 bus_size_t count) \ 1471 { \ 1472 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1473 "bus_space_write()"); \ 1474 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1475 } 1476 1477 MSAN_BUS_WRITE_FUNC(1, 8) 1478 MSAN_BUS_WRITE_FUNC(2, 16) 1479 MSAN_BUS_WRITE_FUNC(4, 32) 1480 MSAN_BUS_WRITE_FUNC(8, 64) 1481 1482 /* -------------------------------------------------------------------------- */ 1483 1484 #include <sys/mbuf.h> 1485 1486 static void 1487 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1488 bool init, uintptr_t pc) 1489 { 1490 if (init) { 1491 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1492 } else { 1493 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp"); 1494 } 1495 } 1496 1497 static void 1498 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1499 bool init, uintptr_t pc) 1500 { 1501 bus_addr_t minlen; 1502 1503 for (; m != NULL && len != 0; m = m->m_next) { 1504 if (offset >= m->m_len) { 1505 offset -= m->m_len; 1506 continue; 1507 } 1508 1509 minlen = MIN(len, m->m_len - offset); 1510 1511 if (init) { 1512 kmsan_shadow_fill(mtod(m, char *) + offset, 1513 KMSAN_STATE_INITED, minlen); 1514 } else { 1515 kmsan_shadow_check(mtod(m, char *) + offset, 1516 minlen, "MbufDmaSyncOp"); 1517 } 1518 1519 offset = 0; 1520 len -= minlen; 1521 } 1522 } 1523 1524 static void 1525 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1526 bool init, uintptr_t pc) 1527 { 1528 bus_size_t minlen, resid; 1529 struct iovec *iov; 1530 int i; 1531 1532 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1533 return; 1534 1535 resid = uio->uio_resid; 1536 iov = uio->uio_iov; 1537 1538 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1539 minlen = MIN(resid, iov[i].iov_len); 1540 1541 if (init) { 1542 kmsan_shadow_fill(iov[i].iov_base, 1543 KMSAN_STATE_INITED, minlen); 1544 } else { 1545 kmsan_shadow_check(iov[i].iov_base, minlen, 1546 "UioDmaSyncOp"); 1547 } 1548 1549 resid -= minlen; 1550 } 1551 } 1552 1553 void 1554 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1555 { 1556 bool init; 1557 1558 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1559 return; 1560 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1561 1562 switch (map->dm_buftype) { 1563 case KMSAN_DMA_LINEAR: 1564 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1565 __RET_ADDR); 1566 break; 1567 case KMSAN_DMA_MBUF: 1568 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1569 __RET_ADDR); 1570 break; 1571 case KMSAN_DMA_UIO: 1572 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1573 __RET_ADDR); 1574 break; 1575 case KMSAN_DMA_RAW: 1576 break; 1577 default: 1578 panic("%s: impossible", __func__); 1579 } 1580 } 1581 1582 void 1583 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1584 { 1585 map->dm_buf = buf; 1586 map->dm_buflen = buflen; 1587 map->dm_buftype = type; 1588 } 1589
Indexes created Sun Sep 21 15:09:59 GMT 2025