subr_msan.c revision 1.10
1 /* $NetBSD: subr_msan.c,v 1.10 2020/04/15 16:28:28 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.10 2020/04/15 16:28:28 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/kprintf.h> 43 #include <sys/kmem.h> 44 #include <sys/mbuf.h> 45 #include <sys/buf.h> 46 #include <sys/cpu.h> 47 #include <sys/msan.h> 48 49 #include <uvm/uvm.h> 50 51 static void kmsan_printf(const char *, ...); 52 53 void kmsan_init_arg(size_t); 54 void kmsan_init_ret(size_t); 55 56 #ifdef KMSAN_PANIC 57 #define REPORT panic 58 #else 59 #define REPORT kmsan_printf 60 #endif 61 62 /* -------------------------------------------------------------------------- */ 63 64 /* 65 * Part of the compiler ABI. 66 */ 67 68 typedef uint32_t msan_orig_t; 69 70 typedef struct { 71 uint8_t *shad; 72 msan_orig_t *orig; 73 } msan_meta_t; 74 75 #define MSAN_PARAM_SIZE 800 76 #define MSAN_RETVAL_SIZE 800 77 typedef struct { 78 uint8_t param[MSAN_PARAM_SIZE]; 79 uint8_t retval[MSAN_RETVAL_SIZE]; 80 uint8_t _va_arg[MSAN_PARAM_SIZE]; 81 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 82 uint64_t va_arg_overflow_size; 83 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 84 msan_orig_t retval_origin; 85 msan_orig_t origin; 86 } msan_tls_t; 87 88 /* -------------------------------------------------------------------------- */ 89 90 /* The MD code. */ 91 #include <machine/msan.h> 92 93 /* -------------------------------------------------------------------------- */ 94 95 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 96 #define MSAN_NCONTEXT 16 97 98 typedef struct { 99 size_t ctx; 100 msan_tls_t tls[MSAN_NCONTEXT]; 101 } msan_lwp_t; 102 103 static msan_tls_t dummy_tls; 104 105 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 106 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 107 static msan_lwp_t msan_lwp0; 108 static bool kmsan_enabled __read_mostly; 109 110 /* -------------------------------------------------------------------------- */ 111 112 static bool kmsan_reporting = false; 113 114 static inline void 115 kmsan_printf(const char *fmt, ...) 116 { 117 va_list ap; 118 119 va_start(ap, fmt); 120 kprintf(fmt, TOCONS, NULL, NULL, ap); 121 va_end(ap); 122 } 123 124 static inline const char * 125 kmsan_orig_name(int type) 126 { 127 switch (type) { 128 case KMSAN_TYPE_STACK: 129 return "Stack"; 130 case KMSAN_TYPE_KMEM: 131 return "Kmem"; 132 case KMSAN_TYPE_MALLOC: 133 return "Malloc"; 134 case KMSAN_TYPE_POOL: 135 return "Pool"; 136 case KMSAN_TYPE_UVM: 137 return "Uvm"; 138 default: 139 return "Unknown"; 140 } 141 } 142 143 /* 144 * The format of the string is: "----var@function". Parse it to display a nice 145 * warning. 146 */ 147 static void 148 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 149 { 150 const char *mod, *sym; 151 extern int db_active; 152 msan_orig_t *orig; 153 const char *typename; 154 char *var, *fn; 155 uintptr_t ptr; 156 char buf[128]; 157 int type; 158 159 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 160 return; 161 162 kmsan_reporting = true; 163 __insn_barrier(); 164 165 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 166 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 167 168 if (*orig == 0) { 169 REPORT("MSan: Uninitialized Memory In %s At Offset " 170 "%zu\n", hook, off); 171 goto out; 172 } 173 174 kmsan_md_orig_decode(*orig, &type, &ptr); 175 typename = kmsan_orig_name(type); 176 177 if (kmsan_md_is_pc(ptr)) { 178 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 179 REPORT("MSan: Uninitialized %s Memory In %s " 180 "At Offset %zu/%zu, IP %p\n", typename, hook, off, 181 size, (void *)ptr); 182 } else { 183 REPORT("MSan: Uninitialized %s Memory In %s " 184 "At Offset %zu/%zu, From %s()\n", typename, hook, 185 off, size, sym); 186 } 187 } else { 188 var = (char *)ptr + 4; 189 strlcpy(buf, var, sizeof(buf)); 190 var = buf; 191 fn = __builtin_strchr(buf, '@'); 192 *fn++ = '\0'; 193 REPORT("MSan: Uninitialized %s Memory In %s At Offset " 194 "%zu, Variable '%s' From %s()\n", typename, hook, off, 195 var, fn); 196 } 197 198 out: 199 kmsan_md_unwind(); 200 __insn_barrier(); 201 kmsan_reporting = false; 202 } 203 204 static void 205 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 206 { 207 const char *mod, *sym; 208 extern int db_active; 209 const char *typename; 210 char *var, *fn; 211 uintptr_t ptr; 212 char buf[128]; 213 int type; 214 215 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 216 return; 217 218 kmsan_reporting = true; 219 __insn_barrier(); 220 221 if (orig == 0) { 222 REPORT("MSan: Uninitialized Variable In %p\n", 223 (void *)pc); 224 goto out; 225 } 226 227 kmsan_md_orig_decode(orig, &type, &ptr); 228 typename = kmsan_orig_name(type); 229 230 if (kmsan_md_is_pc(ptr)) { 231 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 232 REPORT("MSan: Uninitialized %s Memory, " 233 "Origin %x\n", typename, orig); 234 } else { 235 REPORT("MSan: Uninitialized %s Memory " 236 "From %s()\n", typename, sym); 237 } 238 } else { 239 var = (char *)ptr + 4; 240 strlcpy(buf, var, sizeof(buf)); 241 var = buf; 242 fn = __builtin_strchr(buf, '@'); 243 *fn++ = '\0'; 244 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 245 var, fn); 246 } 247 248 out: 249 kmsan_md_unwind(); 250 __insn_barrier(); 251 kmsan_reporting = false; 252 } 253 254 /* -------------------------------------------------------------------------- */ 255 256 static inline msan_meta_t 257 kmsan_meta_get(void *addr, size_t size) 258 { 259 msan_meta_t ret; 260 261 if (__predict_false(!kmsan_enabled)) { 262 ret.shad = msan_dummy_shad; 263 ret.orig = (msan_orig_t *)msan_dummy_orig; 264 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 265 ret.shad = msan_dummy_shad; 266 ret.orig = (msan_orig_t *)msan_dummy_orig; 267 } else { 268 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 269 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 270 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 271 } 272 273 return ret; 274 } 275 276 static inline void 277 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 278 { 279 msan_orig_t *orig; 280 size_t i; 281 282 if (__predict_false(!kmsan_enabled)) 283 return; 284 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 285 return; 286 287 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 288 size += ((uintptr_t)orig & 0x3); 289 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 290 291 for (i = 0; i < size; i += 4) { 292 orig[i / 4] = o; 293 } 294 } 295 296 static inline void 297 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 298 { 299 uint8_t *shad; 300 301 if (__predict_false(!kmsan_enabled)) 302 return; 303 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 304 return; 305 306 shad = kmsan_md_addr_to_shad(addr); 307 __builtin_memset(shad, c, size); 308 } 309 310 static inline void 311 kmsan_meta_copy(void *dst, const void *src, size_t size) 312 { 313 uint8_t *orig_src, *orig_dst; 314 uint8_t *shad_src, *shad_dst; 315 msan_orig_t *_src, *_dst; 316 size_t i; 317 318 if (__predict_false(!kmsan_enabled)) 319 return; 320 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 321 return; 322 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 323 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 324 return; 325 } 326 327 shad_src = kmsan_md_addr_to_shad(src); 328 shad_dst = kmsan_md_addr_to_shad(dst); 329 __builtin_memmove(shad_dst, shad_src, size); 330 331 orig_src = kmsan_md_addr_to_orig(src); 332 orig_dst = kmsan_md_addr_to_orig(dst); 333 for (i = 0; i < size; i++) { 334 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 335 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 336 *_dst = *_src; 337 orig_src++; 338 orig_dst++; 339 } 340 } 341 342 static inline void 343 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 344 { 345 uint8_t *shad; 346 size_t i; 347 348 if (__predict_false(!kmsan_enabled)) 349 return; 350 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 351 return; 352 353 shad = kmsan_md_addr_to_shad(addr); 354 for (i = 0; i < size; i++) { 355 if (__predict_true(shad[i] == 0)) 356 continue; 357 kmsan_report_hook((const char *)addr + i, size, i, hook); 358 break; 359 } 360 } 361 362 void 363 kmsan_init_arg(size_t n) 364 { 365 msan_lwp_t *lwp; 366 uint8_t *arg; 367 368 if (__predict_false(!kmsan_enabled)) 369 return; 370 lwp = curlwp->l_kmsan; 371 arg = lwp->tls[lwp->ctx].param; 372 __builtin_memset(arg, 0, n); 373 } 374 375 void 376 kmsan_init_ret(size_t n) 377 { 378 msan_lwp_t *lwp; 379 uint8_t *arg; 380 381 if (__predict_false(!kmsan_enabled)) 382 return; 383 lwp = curlwp->l_kmsan; 384 arg = lwp->tls[lwp->ctx].retval; 385 __builtin_memset(arg, 0, n); 386 } 387 388 static void 389 kmsan_check_arg(size_t size, const char *hook) 390 { 391 msan_lwp_t *lwp; 392 uint8_t *arg; 393 size_t i; 394 395 if (__predict_false(!kmsan_enabled)) 396 return; 397 lwp = curlwp->l_kmsan; 398 arg = lwp->tls[lwp->ctx].param; 399 400 for (i = 0; i < size; i++) { 401 if (__predict_true(arg[i] == 0)) 402 continue; 403 kmsan_report_hook((const char *)arg + i, size, i, hook); 404 break; 405 } 406 } 407 408 void 409 kmsan_lwp_alloc(struct lwp *l) 410 { 411 msan_lwp_t *lwp; 412 413 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 414 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 415 lwp->ctx = 1; 416 417 l->l_kmsan = lwp; 418 } 419 420 void 421 kmsan_lwp_free(struct lwp *l) 422 { 423 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 424 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 425 } 426 427 void kmsan_intr_enter(void); 428 void kmsan_intr_leave(void); 429 void kmsan_softint(struct lwp *); 430 431 void 432 kmsan_intr_enter(void) 433 { 434 msan_lwp_t *lwp; 435 436 if (__predict_false(!kmsan_enabled)) 437 return; 438 lwp = curlwp->l_kmsan; 439 440 lwp->ctx++; 441 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 442 kmsan_enabled = false; 443 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 444 } 445 446 kmsan_init_arg(sizeof(void *)); 447 } 448 449 void 450 kmsan_intr_leave(void) 451 { 452 msan_lwp_t *lwp; 453 454 if (__predict_false(!kmsan_enabled)) 455 return; 456 lwp = curlwp->l_kmsan; 457 458 if (__predict_false(lwp->ctx == 0)) { 459 kmsan_enabled = false; 460 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 461 } 462 lwp->ctx--; 463 } 464 465 void 466 kmsan_softint(struct lwp *l) 467 { 468 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 469 } 470 471 /* -------------------------------------------------------------------------- */ 472 473 void 474 kmsan_shadow_map(void *addr, size_t size) 475 { 476 size_t npages, i; 477 vaddr_t va; 478 479 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 480 KASSERT(size % PAGE_SIZE == 0); 481 482 npages = size / PAGE_SIZE; 483 484 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 485 for (i = 0; i < npages; i++) { 486 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 487 } 488 489 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 490 for (i = 0; i < npages; i++) { 491 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 492 } 493 } 494 495 void 496 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 497 { 498 msan_orig_t orig; 499 500 orig = kmsan_md_orig_encode(type, pc); 501 kmsan_origin_fill(addr, orig, size); 502 } 503 504 void 505 kmsan_mark(void *addr, size_t size, uint8_t c) 506 { 507 kmsan_shadow_fill(addr, c, size); 508 } 509 510 void 511 kmsan_check_mbuf(void *buf) 512 { 513 struct mbuf *m = buf; 514 515 do { 516 kmsan_shadow_check(mtod(m, void *), m->m_len, "if_transmit()"); 517 } while ((m = m->m_next) != NULL); 518 } 519 520 void 521 kmsan_check_buf(void *buf) 522 { 523 buf_t *bp = buf; 524 525 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()"); 526 } 527 528 void 529 kmsan_init(void *stack) 530 { 531 /* MD initialization. */ 532 kmsan_md_init(); 533 534 /* Map the stack. */ 535 kmsan_shadow_map(stack, USPACE); 536 537 /* Initialize the TLS for curlwp. */ 538 msan_lwp0.ctx = 1; 539 curlwp->l_kmsan = &msan_lwp0; 540 541 /* Now officially enabled. */ 542 kmsan_enabled = true; 543 } 544 545 /* -------------------------------------------------------------------------- */ 546 547 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 548 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 549 550 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 551 { 552 return kmsan_meta_get(addr, size); 553 } 554 555 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 556 { 557 return kmsan_meta_get(addr, size); 558 } 559 560 #define MSAN_META_FUNC(size) \ 561 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 562 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 563 { \ 564 return kmsan_meta_get(addr, size); \ 565 } \ 566 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 567 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 568 { \ 569 return kmsan_meta_get(addr, size); \ 570 } 571 572 MSAN_META_FUNC(1) 573 MSAN_META_FUNC(2) 574 MSAN_META_FUNC(4) 575 MSAN_META_FUNC(8) 576 577 void __msan_instrument_asm_store(void *, size_t); 578 msan_orig_t __msan_chain_origin(msan_orig_t); 579 void __msan_poison_alloca(void *, uint64_t, char *); 580 void __msan_unpoison_alloca(void *, uint64_t); 581 void __msan_warning(msan_orig_t); 582 msan_tls_t *__msan_get_context_state(void); 583 584 void __msan_instrument_asm_store(void *addr, size_t size) 585 { 586 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 587 } 588 589 msan_orig_t __msan_chain_origin(msan_orig_t origin) 590 { 591 return origin; 592 } 593 594 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 595 { 596 msan_orig_t orig; 597 598 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 599 kmsan_origin_fill(addr, orig, size); 600 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 601 } 602 603 void __msan_unpoison_alloca(void *addr, uint64_t size) 604 { 605 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 606 } 607 608 void __msan_warning(msan_orig_t origin) 609 { 610 if (__predict_false(!kmsan_enabled)) 611 return; 612 kmsan_report_inline(origin, __RET_ADDR); 613 } 614 615 msan_tls_t *__msan_get_context_state(void) 616 { 617 msan_lwp_t *lwp; 618 619 if (__predict_false(!kmsan_enabled)) 620 return &dummy_tls; 621 lwp = curlwp->l_kmsan; 622 623 return &lwp->tls[lwp->ctx]; 624 } 625 626 /* -------------------------------------------------------------------------- */ 627 628 /* 629 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 630 * initialized areas properly. 631 */ 632 633 void *kmsan_memcpy(void *dst, const void *src, size_t len) 634 { 635 /* No kmsan_check_arg, because inlined. */ 636 kmsan_init_ret(sizeof(void *)); 637 if (__predict_true(len != 0)) { 638 kmsan_meta_copy(dst, src, len); 639 } 640 return __builtin_memcpy(dst, src, len); 641 } 642 643 int 644 kmsan_memcmp(const void *b1, const void *b2, size_t len) 645 { 646 const uint8_t *_b1 = b1, *_b2 = b2; 647 size_t i; 648 649 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 650 "memcmp():args"); 651 kmsan_init_ret(sizeof(int)); 652 653 for (i = 0; i < len; i++) { 654 if (*_b1 != *_b2) { 655 kmsan_shadow_check(b1, i + 1, "memcmp():arg1"); 656 kmsan_shadow_check(b2, i + 1, "memcmp():arg2"); 657 return *_b1 - *_b2; 658 } 659 _b1++, _b2++; 660 } 661 662 return 0; 663 } 664 665 void *kmsan_memset(void *dst, int c, size_t len) 666 { 667 /* No kmsan_check_arg, because inlined. */ 668 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 669 kmsan_init_ret(sizeof(void *)); 670 return __builtin_memset(dst, c, len); 671 } 672 673 void *kmsan_memmove(void *dst, const void *src, size_t len) 674 { 675 /* No kmsan_check_arg, because inlined. */ 676 if (__predict_true(len != 0)) { 677 kmsan_meta_copy(dst, src, len); 678 } 679 kmsan_init_ret(sizeof(void *)); 680 return __builtin_memmove(dst, src, len); 681 } 682 683 __strong_alias(__msan_memcpy, kmsan_memcpy) 684 __strong_alias(__msan_memset, kmsan_memset) 685 __strong_alias(__msan_memmove, kmsan_memmove) 686 687 char * 688 kmsan_strcpy(char *dst, const char *src) 689 { 690 const char *_src = src; 691 char *_dst = dst; 692 size_t len = 0; 693 694 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 695 696 while (1) { 697 len++; 698 *dst = *src; 699 if (*src == '\0') 700 break; 701 src++, dst++; 702 } 703 704 kmsan_shadow_check(_src, len, "strcpy():arg2"); 705 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 706 kmsan_init_ret(sizeof(char *)); 707 return _dst; 708 } 709 710 int 711 kmsan_strcmp(const char *s1, const char *s2) 712 { 713 const char *_s1 = s1, *_s2 = s2; 714 size_t len = 0; 715 716 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 717 kmsan_init_ret(sizeof(int)); 718 719 while (1) { 720 len++; 721 if (*s1 != *s2) 722 break; 723 if (*s1 == '\0') { 724 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 725 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 726 return 0; 727 } 728 s1++, s2++; 729 } 730 731 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 732 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 733 734 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 735 } 736 737 size_t 738 kmsan_strlen(const char *str) 739 { 740 const char *s; 741 742 kmsan_check_arg(sizeof(str), "strlen():args"); 743 744 s = str; 745 while (1) { 746 if (*s == '\0') 747 break; 748 s++; 749 } 750 751 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1"); 752 kmsan_init_ret(sizeof(size_t)); 753 return (s - str); 754 } 755 756 char * 757 kmsan_strcat(char *dst, const char *src) 758 { 759 size_t ldst, lsrc; 760 char *ret; 761 762 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args"); 763 764 ldst = __builtin_strlen(dst); 765 lsrc = __builtin_strlen(src); 766 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1"); 767 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2"); 768 ret = __builtin_strcat(dst, src); 769 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1); 770 771 kmsan_init_ret(sizeof(char *)); 772 return ret; 773 } 774 775 char * 776 kmsan_strchr(const char *s, int c) 777 { 778 char *ret; 779 780 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args"); 781 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1"); 782 ret = __builtin_strchr(s, c); 783 784 kmsan_init_ret(sizeof(char *)); 785 return ret; 786 } 787 788 char * 789 kmsan_strrchr(const char *s, int c) 790 { 791 char *ret; 792 793 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args"); 794 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1"); 795 ret = __builtin_strrchr(s, c); 796 797 kmsan_init_ret(sizeof(char *)); 798 return ret; 799 } 800 801 #undef kcopy 802 #undef copystr 803 #undef copyin 804 #undef copyout 805 #undef copyinstr 806 #undef copyoutstr 807 808 int kmsan_kcopy(const void *, void *, size_t); 809 int kmsan_copystr(const void *, void *, size_t, size_t *); 810 int kmsan_copyin(const void *, void *, size_t); 811 int kmsan_copyout(const void *, void *, size_t); 812 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 813 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 814 815 int kcopy(const void *, void *, size_t); 816 int copystr(const void *, void *, size_t, size_t *); 817 int copyin(const void *, void *, size_t); 818 int copyout(const void *, void *, size_t); 819 int copyinstr(const void *, void *, size_t, size_t *); 820 int copyoutstr(const void *, void *, size_t, size_t *); 821 822 int 823 kmsan_kcopy(const void *src, void *dst, size_t len) 824 { 825 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), 826 "kcopy():args"); 827 if (__predict_true(len != 0)) { 828 kmsan_meta_copy(dst, src, len); 829 } 830 kmsan_init_ret(sizeof(int)); 831 return kcopy(src, dst, len); 832 } 833 834 int 835 kmsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 836 { 837 size_t _done; 838 int ret; 839 840 kmsan_check_arg(sizeof(kfaddr) + sizeof(kdaddr) + 841 sizeof(len) + sizeof(done), "copystr():args"); 842 ret = copystr(kfaddr, kdaddr, len, &_done); 843 if (ret == 0) 844 kmsan_meta_copy(kdaddr, kfaddr, _done); 845 if (done != NULL) { 846 *done = _done; 847 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 848 } 849 kmsan_init_ret(sizeof(int)); 850 851 return ret; 852 } 853 854 int 855 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 856 { 857 int ret; 858 859 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 860 "copyin():args"); 861 ret = copyin(uaddr, kaddr, len); 862 if (ret == 0) 863 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 864 kmsan_init_ret(sizeof(int)); 865 866 return ret; 867 } 868 869 int 870 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 871 { 872 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 873 "copyout():args"); 874 kmsan_shadow_check(kaddr, len, "copyout():arg1"); 875 kmsan_init_ret(sizeof(int)); 876 return copyout(kaddr, uaddr, len); 877 } 878 879 int 880 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 881 { 882 size_t _done; 883 int ret; 884 885 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 886 sizeof(len) + sizeof(done), "copyinstr():args"); 887 ret = copyinstr(uaddr, kaddr, len, &_done); 888 if (ret == 0 || ret == ENAMETOOLONG) 889 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 890 if (done != NULL) { 891 *done = _done; 892 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 893 } 894 kmsan_init_ret(sizeof(int)); 895 896 return ret; 897 } 898 899 int 900 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 901 { 902 size_t _done; 903 int ret; 904 905 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 906 sizeof(len) + sizeof(done), "copyoutstr():args"); 907 ret = copyoutstr(kaddr, uaddr, len, &_done); 908 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1"); 909 if (done != NULL) { 910 *done = _done; 911 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 912 } 913 kmsan_init_ret(sizeof(int)); 914 915 return ret; 916 } 917 918 /* -------------------------------------------------------------------------- */ 919 920 #undef _ucas_32 921 #undef _ucas_32_mp 922 #undef _ucas_64 923 #undef _ucas_64_mp 924 #undef _ufetch_8 925 #undef _ufetch_16 926 #undef _ufetch_32 927 #undef _ufetch_64 928 #undef _ustore_8 929 #undef _ustore_16 930 #undef _ustore_32 931 #undef _ustore_64 932 933 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 934 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 935 int 936 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 937 uint32_t *ret) 938 { 939 int _ret; 940 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 941 sizeof(new) + sizeof(ret), "ucas_32():args"); 942 _ret = _ucas_32(uaddr, old, new, ret); 943 if (_ret == 0) 944 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 945 kmsan_init_ret(sizeof(int)); 946 return _ret; 947 } 948 949 #ifdef __HAVE_UCAS_MP 950 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 951 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 952 int 953 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 954 uint32_t *ret) 955 { 956 int _ret; 957 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 958 sizeof(new) + sizeof(ret), "ucas_32_mp():args"); 959 _ret = _ucas_32_mp(uaddr, old, new, ret); 960 if (_ret == 0) 961 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 962 kmsan_init_ret(sizeof(int)); 963 return _ret; 964 } 965 #endif 966 967 #ifdef _LP64 968 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 969 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 970 int 971 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 972 uint64_t *ret) 973 { 974 int _ret; 975 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 976 sizeof(new) + sizeof(ret), "ucas_64():args"); 977 _ret = _ucas_64(uaddr, old, new, ret); 978 if (_ret == 0) 979 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 980 kmsan_init_ret(sizeof(int)); 981 return _ret; 982 } 983 984 #ifdef __HAVE_UCAS_MP 985 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 986 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 987 int 988 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 989 uint64_t *ret) 990 { 991 int _ret; 992 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 993 sizeof(new) + sizeof(ret), "ucas_64_mp():args"); 994 _ret = _ucas_64_mp(uaddr, old, new, ret); 995 if (_ret == 0) 996 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 997 kmsan_init_ret(sizeof(int)); 998 return _ret; 999 } 1000 #endif 1001 #endif 1002 1003 int _ufetch_8(const uint8_t *, uint8_t *); 1004 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 1005 int 1006 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 1007 { 1008 int _ret; 1009 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args"); 1010 _ret = _ufetch_8(uaddr, valp); 1011 if (_ret == 0) 1012 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1013 kmsan_init_ret(sizeof(int)); 1014 return _ret; 1015 } 1016 1017 int _ufetch_16(const uint16_t *, uint16_t *); 1018 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 1019 int 1020 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 1021 { 1022 int _ret; 1023 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args"); 1024 _ret = _ufetch_16(uaddr, valp); 1025 if (_ret == 0) 1026 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1027 kmsan_init_ret(sizeof(int)); 1028 return _ret; 1029 } 1030 1031 int _ufetch_32(const uint32_t *, uint32_t *); 1032 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 1033 int 1034 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 1035 { 1036 int _ret; 1037 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args"); 1038 _ret = _ufetch_32(uaddr, valp); 1039 if (_ret == 0) 1040 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1041 kmsan_init_ret(sizeof(int)); 1042 return _ret; 1043 } 1044 1045 #ifdef _LP64 1046 int _ufetch_64(const uint64_t *, uint64_t *); 1047 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1048 int 1049 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1050 { 1051 int _ret; 1052 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args"); 1053 _ret = _ufetch_64(uaddr, valp); 1054 if (_ret == 0) 1055 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1056 kmsan_init_ret(sizeof(int)); 1057 return _ret; 1058 } 1059 #endif 1060 1061 int _ustore_8(uint8_t *, uint8_t); 1062 int kmsan__ustore_8(uint8_t *, uint8_t); 1063 int 1064 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1065 { 1066 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args"); 1067 kmsan_init_ret(sizeof(int)); 1068 return _ustore_8(uaddr, val); 1069 } 1070 1071 int _ustore_16(uint16_t *, uint16_t); 1072 int kmsan__ustore_16(uint16_t *, uint16_t); 1073 int 1074 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1075 { 1076 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args"); 1077 kmsan_init_ret(sizeof(int)); 1078 return _ustore_16(uaddr, val); 1079 } 1080 1081 int _ustore_32(uint32_t *, uint32_t); 1082 int kmsan__ustore_32(uint32_t *, uint32_t); 1083 int 1084 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1085 { 1086 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args"); 1087 kmsan_init_ret(sizeof(int)); 1088 return _ustore_32(uaddr, val); 1089 } 1090 1091 #ifdef _LP64 1092 int _ustore_64(uint64_t *, uint64_t); 1093 int kmsan__ustore_64(uint64_t *, uint64_t); 1094 int 1095 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1096 { 1097 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args"); 1098 kmsan_init_ret(sizeof(int)); 1099 return _ustore_64(uaddr, val); 1100 } 1101 #endif 1102 1103 /* -------------------------------------------------------------------------- */ 1104 1105 #undef atomic_add_32 1106 #undef atomic_add_int 1107 #undef atomic_add_long 1108 #undef atomic_add_ptr 1109 #undef atomic_add_64 1110 #undef atomic_add_32_nv 1111 #undef atomic_add_int_nv 1112 #undef atomic_add_long_nv 1113 #undef atomic_add_ptr_nv 1114 #undef atomic_add_64_nv 1115 #undef atomic_and_32 1116 #undef atomic_and_uint 1117 #undef atomic_and_ulong 1118 #undef atomic_and_64 1119 #undef atomic_and_32_nv 1120 #undef atomic_and_uint_nv 1121 #undef atomic_and_ulong_nv 1122 #undef atomic_and_64_nv 1123 #undef atomic_or_32 1124 #undef atomic_or_uint 1125 #undef atomic_or_ulong 1126 #undef atomic_or_64 1127 #undef atomic_or_32_nv 1128 #undef atomic_or_uint_nv 1129 #undef atomic_or_ulong_nv 1130 #undef atomic_or_64_nv 1131 #undef atomic_cas_32 1132 #undef atomic_cas_uint 1133 #undef atomic_cas_ulong 1134 #undef atomic_cas_ptr 1135 #undef atomic_cas_64 1136 #undef atomic_cas_32_ni 1137 #undef atomic_cas_uint_ni 1138 #undef atomic_cas_ulong_ni 1139 #undef atomic_cas_ptr_ni 1140 #undef atomic_cas_64_ni 1141 #undef atomic_swap_32 1142 #undef atomic_swap_uint 1143 #undef atomic_swap_ulong 1144 #undef atomic_swap_ptr 1145 #undef atomic_swap_64 1146 #undef atomic_dec_32 1147 #undef atomic_dec_uint 1148 #undef atomic_dec_ulong 1149 #undef atomic_dec_ptr 1150 #undef atomic_dec_64 1151 #undef atomic_dec_32_nv 1152 #undef atomic_dec_uint_nv 1153 #undef atomic_dec_ulong_nv 1154 #undef atomic_dec_ptr_nv 1155 #undef atomic_dec_64_nv 1156 #undef atomic_inc_32 1157 #undef atomic_inc_uint 1158 #undef atomic_inc_ulong 1159 #undef atomic_inc_ptr 1160 #undef atomic_inc_64 1161 #undef atomic_inc_32_nv 1162 #undef atomic_inc_uint_nv 1163 #undef atomic_inc_ulong_nv 1164 #undef atomic_inc_ptr_nv 1165 #undef atomic_inc_64_nv 1166 1167 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1168 void atomic_add_##name(volatile targ1 *, targ2); \ 1169 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1170 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1171 { \ 1172 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1173 "atomic_add_" #name "():args"); \ 1174 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1175 "atomic_add_" #name "():arg1"); \ 1176 atomic_add_##name(ptr, val); \ 1177 } \ 1178 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1179 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1180 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1181 { \ 1182 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1183 "atomic_add_" #name "_nv():args"); \ 1184 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1185 "atomic_add_" #name "_nv():arg1"); \ 1186 kmsan_init_ret(sizeof(tret)); \ 1187 return atomic_add_##name##_nv(ptr, val); \ 1188 } 1189 1190 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1191 void atomic_and_##name(volatile targ1 *, targ2); \ 1192 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1193 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1194 { \ 1195 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1196 "atomic_and_" #name "():args"); \ 1197 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1198 "atomic_and_" #name "():arg1"); \ 1199 atomic_and_##name(ptr, val); \ 1200 } \ 1201 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1202 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1203 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1204 { \ 1205 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1206 "atomic_and_" #name "_nv():args"); \ 1207 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1208 "atomic_and_" #name "_nv():arg1"); \ 1209 kmsan_init_ret(sizeof(tret)); \ 1210 return atomic_and_##name##_nv(ptr, val); \ 1211 } 1212 1213 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1214 void atomic_or_##name(volatile targ1 *, targ2); \ 1215 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1216 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1217 { \ 1218 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1219 "atomic_or_" #name "():args"); \ 1220 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1221 "atomic_or_" #name "():arg1"); \ 1222 atomic_or_##name(ptr, val); \ 1223 } \ 1224 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1225 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1226 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1227 { \ 1228 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1229 "atomic_or_" #name "_nv():args"); \ 1230 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1231 "atomic_or_" #name "_nv():arg1"); \ 1232 kmsan_init_ret(sizeof(tret)); \ 1233 return atomic_or_##name##_nv(ptr, val); \ 1234 } 1235 1236 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1237 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1238 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1239 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1240 { \ 1241 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1242 "atomic_cas_" #name "():args"); \ 1243 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1244 "atomic_cas_" #name "():arg1"); \ 1245 kmsan_init_ret(sizeof(tret)); \ 1246 return atomic_cas_##name(ptr, exp, new); \ 1247 } \ 1248 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1249 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1250 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1251 { \ 1252 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1253 "atomic_cas_" #name "_ni():args"); \ 1254 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1255 "atomic_cas_" #name "_ni():arg1"); \ 1256 kmsan_init_ret(sizeof(tret)); \ 1257 return atomic_cas_##name##_ni(ptr, exp, new); \ 1258 } 1259 1260 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1261 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1262 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1263 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1264 { \ 1265 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1266 "atomic_swap_" #name "():args"); \ 1267 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1268 "atomic_swap_" #name "():arg1"); \ 1269 kmsan_init_ret(sizeof(tret)); \ 1270 return atomic_swap_##name(ptr, val); \ 1271 } 1272 1273 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1274 void atomic_dec_##name(volatile targ1 *); \ 1275 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1276 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1277 { \ 1278 kmsan_check_arg(sizeof(ptr), \ 1279 "atomic_dec_" #name "():args"); \ 1280 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1281 "atomic_dec_" #name "():arg1"); \ 1282 atomic_dec_##name(ptr); \ 1283 } \ 1284 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1285 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1286 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1287 { \ 1288 kmsan_check_arg(sizeof(ptr), \ 1289 "atomic_dec_" #name "_nv():args"); \ 1290 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1291 "atomic_dec_" #name "_nv():arg1"); \ 1292 kmsan_init_ret(sizeof(tret)); \ 1293 return atomic_dec_##name##_nv(ptr); \ 1294 } 1295 1296 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1297 void atomic_inc_##name(volatile targ1 *); \ 1298 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1299 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1300 { \ 1301 kmsan_check_arg(sizeof(ptr), \ 1302 "atomic_inc_" #name "():args"); \ 1303 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1304 "atomic_inc_" #name "():arg1"); \ 1305 atomic_inc_##name(ptr); \ 1306 } \ 1307 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1308 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1309 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1310 { \ 1311 kmsan_check_arg(sizeof(ptr), \ 1312 "atomic_inc_" #name "_nv():args"); \ 1313 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1314 "atomic_inc_" #name "_nv():arg1"); \ 1315 kmsan_init_ret(sizeof(tret)); \ 1316 return atomic_inc_##name##_nv(ptr); \ 1317 } 1318 1319 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1320 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1321 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1322 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1323 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1324 1325 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1326 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1327 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1328 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1329 1330 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1331 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1332 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1333 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1334 1335 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1336 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1337 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1338 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1339 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1340 1341 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1342 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1343 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1344 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1345 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1346 1347 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1348 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1349 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1350 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1351 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1352 1353 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1354 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1355 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1356 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1357 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1358 1359 /* -------------------------------------------------------------------------- */ 1360 1361 #include <sys/bus.h> 1362 1363 #undef bus_space_read_multi_1 1364 #undef bus_space_read_multi_2 1365 #undef bus_space_read_multi_4 1366 #undef bus_space_read_multi_8 1367 #undef bus_space_read_multi_stream_1 1368 #undef bus_space_read_multi_stream_2 1369 #undef bus_space_read_multi_stream_4 1370 #undef bus_space_read_multi_stream_8 1371 #undef bus_space_read_region_1 1372 #undef bus_space_read_region_2 1373 #undef bus_space_read_region_4 1374 #undef bus_space_read_region_8 1375 #undef bus_space_read_region_stream_1 1376 #undef bus_space_read_region_stream_2 1377 #undef bus_space_read_region_stream_4 1378 #undef bus_space_read_region_stream_8 1379 1380 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1381 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1382 bus_size_t, uint##bits##_t *, bus_size_t); \ 1383 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1384 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1385 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1386 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1387 bus_size_t count) \ 1388 { \ 1389 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1390 sizeof(uint##bits##_t) * count); \ 1391 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1392 } \ 1393 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1394 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1395 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1396 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1397 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1398 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1399 bus_size_t count) \ 1400 { \ 1401 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1402 sizeof(uint##bits##_t) * count); \ 1403 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1404 } \ 1405 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1406 bus_size_t, uint##bits##_t *, bus_size_t); \ 1407 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1408 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1409 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1410 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1411 bus_size_t count) \ 1412 { \ 1413 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1414 sizeof(uint##bits##_t) * count); \ 1415 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1416 } \ 1417 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1418 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1419 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1420 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1421 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1422 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1423 bus_size_t count) \ 1424 { \ 1425 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1426 sizeof(uint##bits##_t) * count); \ 1427 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1428 } 1429 1430 MSAN_BUS_READ_FUNC(1, 8) 1431 MSAN_BUS_READ_FUNC(2, 16) 1432 MSAN_BUS_READ_FUNC(4, 32) 1433 MSAN_BUS_READ_FUNC(8, 64) 1434 1435 #undef bus_space_write_multi_1 1436 #undef bus_space_write_multi_2 1437 #undef bus_space_write_multi_4 1438 #undef bus_space_write_multi_8 1439 #undef bus_space_write_multi_stream_1 1440 #undef bus_space_write_multi_stream_2 1441 #undef bus_space_write_multi_stream_4 1442 #undef bus_space_write_multi_stream_8 1443 #undef bus_space_write_region_1 1444 #undef bus_space_write_region_2 1445 #undef bus_space_write_region_4 1446 #undef bus_space_write_region_8 1447 #undef bus_space_write_region_stream_1 1448 #undef bus_space_write_region_stream_2 1449 #undef bus_space_write_region_stream_4 1450 #undef bus_space_write_region_stream_8 1451 1452 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1453 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1454 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1455 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1456 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1457 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1458 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1459 bus_size_t count) \ 1460 { \ 1461 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1462 "bus_space_write()"); \ 1463 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1464 } \ 1465 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1466 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1467 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1468 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1469 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1470 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1471 bus_size_t count) \ 1472 { \ 1473 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1474 "bus_space_write()"); \ 1475 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1476 } \ 1477 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1478 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1479 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1480 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1481 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1482 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1483 bus_size_t count) \ 1484 { \ 1485 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1486 "bus_space_write()"); \ 1487 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1488 } \ 1489 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1490 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1491 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1492 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1493 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1494 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1495 bus_size_t count) \ 1496 { \ 1497 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1498 "bus_space_write()"); \ 1499 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1500 } 1501 1502 MSAN_BUS_WRITE_FUNC(1, 8) 1503 MSAN_BUS_WRITE_FUNC(2, 16) 1504 MSAN_BUS_WRITE_FUNC(4, 32) 1505 MSAN_BUS_WRITE_FUNC(8, 64) 1506 1507 /* -------------------------------------------------------------------------- */ 1508 1509 #include <sys/mbuf.h> 1510 1511 static void 1512 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1513 bool init, uintptr_t pc) 1514 { 1515 if (init) { 1516 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1517 } else { 1518 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp"); 1519 } 1520 } 1521 1522 static void 1523 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1524 bool init, uintptr_t pc) 1525 { 1526 bus_addr_t minlen; 1527 1528 for (; m != NULL && len != 0; m = m->m_next) { 1529 if (offset >= m->m_len) { 1530 offset -= m->m_len; 1531 continue; 1532 } 1533 1534 minlen = MIN(len, m->m_len - offset); 1535 1536 if (init) { 1537 kmsan_shadow_fill(mtod(m, char *) + offset, 1538 KMSAN_STATE_INITED, minlen); 1539 } else { 1540 kmsan_shadow_check(mtod(m, char *) + offset, 1541 minlen, "MbufDmaSyncOp"); 1542 } 1543 1544 offset = 0; 1545 len -= minlen; 1546 } 1547 } 1548 1549 static void 1550 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1551 bool init, uintptr_t pc) 1552 { 1553 bus_size_t minlen, resid; 1554 struct iovec *iov; 1555 int i; 1556 1557 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1558 return; 1559 1560 resid = uio->uio_resid; 1561 iov = uio->uio_iov; 1562 1563 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1564 minlen = MIN(resid, iov[i].iov_len); 1565 1566 if (init) { 1567 kmsan_shadow_fill(iov[i].iov_base, 1568 KMSAN_STATE_INITED, minlen); 1569 } else { 1570 kmsan_shadow_check(iov[i].iov_base, minlen, 1571 "UioDmaSyncOp"); 1572 } 1573 1574 resid -= minlen; 1575 } 1576 } 1577 1578 void 1579 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1580 { 1581 bool init; 1582 1583 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1584 return; 1585 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1586 1587 switch (map->dm_buftype) { 1588 case KMSAN_DMA_LINEAR: 1589 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1590 __RET_ADDR); 1591 break; 1592 case KMSAN_DMA_MBUF: 1593 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1594 __RET_ADDR); 1595 break; 1596 case KMSAN_DMA_UIO: 1597 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1598 __RET_ADDR); 1599 break; 1600 case KMSAN_DMA_RAW: 1601 break; 1602 default: 1603 panic("%s: impossible", __func__); 1604 } 1605 } 1606 1607 void 1608 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1609 { 1610 map->dm_buf = buf; 1611 map->dm_buflen = buflen; 1612 map->dm_buftype = type; 1613 } 1614
Indexes created Sat Sep 20 22:09:52 GMT 2025