subr_msan.c revision 1.2
1 /* $NetBSD: subr_msan.c,v 1.2 2019/11/15 12:18:46 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define KMSAN_NO_INST 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.2 2019/11/15 12:18:46 maxv Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/device.h> 39 #include <sys/kernel.h> 40 #include <sys/param.h> 41 #include <sys/conf.h> 42 #include <sys/systm.h> 43 #include <sys/types.h> 44 #include <sys/kprintf.h> 45 #include <sys/kmem.h> 46 #include <sys/mbuf.h> 47 #include <sys/buf.h> 48 #include <sys/cpu.h> 49 #include <sys/msan.h> 50 51 #include <uvm/uvm.h> 52 53 static void kmsan_printf(const char *, ...); 54 55 #ifdef KMSAN_PANIC 56 #define REPORT panic 57 #else 58 #define REPORT kmsan_printf 59 #endif 60 61 /* -------------------------------------------------------------------------- */ 62 63 /* 64 * Part of the compiler ABI. 65 */ 66 67 typedef uint32_t msan_orig_t; 68 69 typedef struct { 70 uint8_t *shad; 71 msan_orig_t *orig; 72 } msan_meta_t; 73 74 #define MSAN_PARAM_SIZE 800 75 #define MSAN_RETVAL_SIZE 800 76 typedef struct { 77 uint8_t param[MSAN_PARAM_SIZE]; 78 uint8_t retval[MSAN_RETVAL_SIZE]; 79 uint8_t _va_arg[MSAN_PARAM_SIZE]; 80 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 81 uint64_t va_arg_overflow_size; 82 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 83 msan_orig_t retval_origin; 84 msan_orig_t origin; 85 } msan_tls_t; 86 87 /* -------------------------------------------------------------------------- */ 88 89 /* The MD code. */ 90 #include <machine/msan.h> 91 92 /* -------------------------------------------------------------------------- */ 93 94 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 95 #define MSAN_NCONTEXT 16 96 97 typedef struct { 98 size_t ctx; 99 msan_tls_t tls[MSAN_NCONTEXT]; 100 } msan_lwp_t; 101 102 static msan_tls_t dummy_tls; 103 104 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 105 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 106 static msan_lwp_t msan_lwp0; 107 static bool kmsan_enabled __read_mostly; 108 109 /* -------------------------------------------------------------------------- */ 110 111 static bool kmsan_reporting = false; 112 113 static inline void 114 kmsan_printf(const char *fmt, ...) 115 { 116 va_list ap; 117 118 va_start(ap, fmt); 119 kprintf(fmt, TOCONS, NULL, NULL, ap); 120 va_end(ap); 121 } 122 123 static inline const char * 124 kmsan_orig_name(int type) 125 { 126 switch (type) { 127 case KMSAN_TYPE_STACK: 128 return "Stack"; 129 case KMSAN_TYPE_KMEM: 130 return "Kmem"; 131 case KMSAN_TYPE_MALLOC: 132 return "Malloc"; 133 case KMSAN_TYPE_POOL: 134 return "Pool"; 135 case KMSAN_TYPE_UVM: 136 return "Uvm"; 137 default: 138 return "Unknown"; 139 } 140 } 141 142 /* 143 * The format of the string is: "----var@function". Parse it to display a nice 144 * warning. 145 */ 146 static void 147 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 148 { 149 const char *mod, *sym; 150 extern int db_active; 151 msan_orig_t *orig; 152 const char *typename; 153 char *var, *fn; 154 uintptr_t ptr; 155 char buf[128]; 156 int type; 157 158 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 159 return; 160 161 kmsan_reporting = true; 162 __insn_barrier(); 163 164 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 165 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 166 167 if (*orig == 0) { 168 REPORT("MSan: Uninitialized Memory In %s() At Offset " 169 "%zu\n", hook, off); 170 goto out; 171 } 172 173 kmsan_md_orig_decode(*orig, &type, &ptr); 174 typename = kmsan_orig_name(type); 175 176 if (kmsan_md_is_pc(ptr)) { 177 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 178 REPORT("MSan: Uninitialized %s Memory In %s() " 179 "At Offset %zu, IP %p\n", typename, hook, off, 180 (void *)ptr); 181 } else { 182 REPORT("MSan: Uninitialized %s Memory In %s() " 183 "At Offset %zu, From %s()\n", typename, hook, off, 184 sym); 185 } 186 } else { 187 var = (char *)ptr + 4; 188 strlcpy(buf, var, sizeof(buf)); 189 var = buf; 190 fn = strchr(buf, '@'); 191 *fn++ = '\0'; 192 REPORT("MSan: Uninitialized %s Memory In %s() At Offset " 193 "%zu, Variable '%s' From %s()\n", typename, hook, off, 194 var, fn); 195 } 196 197 out: 198 kmsan_md_unwind(); 199 __insn_barrier(); 200 kmsan_reporting = false; 201 } 202 203 static void 204 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 205 { 206 const char *mod, *sym; 207 extern int db_active; 208 const char *typename; 209 char *var, *fn; 210 uintptr_t ptr; 211 char buf[128]; 212 int type; 213 214 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 215 return; 216 217 kmsan_reporting = true; 218 __insn_barrier(); 219 220 if (orig == 0) { 221 REPORT("MSan: Uninitialized Variable In %p\n", 222 (void *)pc); 223 goto out; 224 } 225 226 kmsan_md_orig_decode(orig, &type, &ptr); 227 typename = kmsan_orig_name(type); 228 229 if (kmsan_md_is_pc(ptr)) { 230 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 231 REPORT("MSan: Uninitialized %s Memory, " 232 "Origin %x\n", typename, orig); 233 } else { 234 REPORT("MSan: Uninitialized %s Memory " 235 "From %s()\n", typename, sym); 236 } 237 } else { 238 var = (char *)ptr + 4; 239 strlcpy(buf, var, sizeof(buf)); 240 var = buf; 241 fn = strchr(buf, '@'); 242 *fn++ = '\0'; 243 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 244 var, fn); 245 } 246 247 out: 248 kmsan_md_unwind(); 249 __insn_barrier(); 250 kmsan_reporting = false; 251 } 252 253 /* -------------------------------------------------------------------------- */ 254 255 static inline msan_meta_t 256 kmsan_meta_get(void *addr, size_t size) 257 { 258 msan_meta_t ret; 259 260 if (__predict_false(!kmsan_enabled)) { 261 ret.shad = msan_dummy_shad; 262 ret.orig = (msan_orig_t *)msan_dummy_orig; 263 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 264 ret.shad = msan_dummy_shad; 265 ret.orig = (msan_orig_t *)msan_dummy_orig; 266 } else { 267 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 268 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 269 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 270 } 271 272 return ret; 273 } 274 275 static inline void 276 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 277 { 278 msan_orig_t *orig; 279 size_t i; 280 281 if (__predict_false(!kmsan_enabled)) 282 return; 283 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 284 return; 285 286 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 287 size += ((uintptr_t)orig & 0x3); 288 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 289 290 for (i = 0; i < size; i += 4) { 291 orig[i / 4] = o; 292 } 293 } 294 295 static inline void 296 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 297 { 298 uint8_t *shad; 299 300 if (__predict_false(!kmsan_enabled)) 301 return; 302 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 303 return; 304 305 shad = kmsan_md_addr_to_shad(addr); 306 __builtin_memset(shad, c, size); 307 } 308 309 static inline void 310 kmsan_meta_copy(void *dst, const void *src, size_t size) 311 { 312 uint8_t *orig_src, *orig_dst; 313 uint8_t *shad_src, *shad_dst; 314 msan_orig_t *_src, *_dst; 315 size_t i; 316 317 if (__predict_false(!kmsan_enabled)) 318 return; 319 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 320 return; 321 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 322 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 323 return; 324 } 325 326 shad_src = kmsan_md_addr_to_shad(src); 327 shad_dst = kmsan_md_addr_to_shad(dst); 328 __builtin_memmove(shad_dst, shad_src, size); 329 330 orig_src = kmsan_md_addr_to_orig(src); 331 orig_dst = kmsan_md_addr_to_orig(dst); 332 for (i = 0; i < size; i++) { 333 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 334 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 335 *_dst = *_src; 336 orig_src++; 337 orig_dst++; 338 } 339 } 340 341 static inline void 342 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 343 { 344 uint8_t *shad; 345 size_t i; 346 347 if (__predict_false(!kmsan_enabled)) 348 return; 349 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 350 return; 351 352 shad = kmsan_md_addr_to_shad(addr); 353 for (i = 0; i < size; i++) { 354 if (__predict_true(shad[i] == 0)) 355 continue; 356 kmsan_report_hook((const char *)addr + i, size, i, hook); 357 break; 358 } 359 } 360 361 void kmsan_init_arg(size_t); 362 void kmsan_init_ret(size_t); 363 364 void 365 kmsan_init_arg(size_t n) 366 { 367 msan_lwp_t *lwp; 368 uint8_t *arg; 369 370 if (__predict_false(!kmsan_enabled)) 371 return; 372 lwp = curlwp->l_kmsan; 373 arg = lwp->tls[lwp->ctx].param; 374 __builtin_memset(arg, 0, n); 375 } 376 377 void 378 kmsan_init_ret(size_t n) 379 { 380 msan_lwp_t *lwp; 381 uint8_t *arg; 382 383 if (__predict_false(!kmsan_enabled)) 384 return; 385 lwp = curlwp->l_kmsan; 386 arg = lwp->tls[lwp->ctx].retval; 387 __builtin_memset(arg, 0, n); 388 } 389 390 static void 391 kmsan_check_arg(size_t size, const char *hook) 392 { 393 msan_lwp_t *lwp; 394 uint8_t *arg; 395 size_t i; 396 397 if (__predict_false(!kmsan_enabled)) 398 return; 399 lwp = curlwp->l_kmsan; 400 arg = lwp->tls[lwp->ctx].param; 401 402 for (i = 0; i < size; i++) { 403 if (__predict_true(arg[i] == 0)) 404 continue; 405 kmsan_report_hook((const char *)arg + i, size, i, hook); 406 break; 407 } 408 } 409 410 void 411 kmsan_lwp_alloc(struct lwp *l) 412 { 413 msan_lwp_t *lwp; 414 415 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 416 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 417 lwp->ctx = 1; 418 419 l->l_kmsan = lwp; 420 } 421 422 void 423 kmsan_lwp_free(struct lwp *l) 424 { 425 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 426 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 427 } 428 429 void kmsan_intr_enter(void); 430 void kmsan_intr_leave(void); 431 void kmsan_softint(struct lwp *); 432 433 void 434 kmsan_intr_enter(void) 435 { 436 msan_lwp_t *lwp; 437 438 if (__predict_false(!kmsan_enabled)) 439 return; 440 lwp = curlwp->l_kmsan; 441 442 lwp->ctx++; 443 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 444 kmsan_enabled = false; 445 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 446 } 447 448 kmsan_init_arg(sizeof(void *)); 449 } 450 451 void 452 kmsan_intr_leave(void) 453 { 454 msan_lwp_t *lwp; 455 456 if (__predict_false(!kmsan_enabled)) 457 return; 458 lwp = curlwp->l_kmsan; 459 460 if (__predict_false(lwp->ctx == 0)) { 461 kmsan_enabled = false; 462 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 463 } 464 lwp->ctx--; 465 } 466 467 void 468 kmsan_softint(struct lwp *l) 469 { 470 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 471 } 472 473 /* -------------------------------------------------------------------------- */ 474 475 void 476 kmsan_shadow_map(void *addr, size_t size) 477 { 478 size_t npages, i; 479 vaddr_t va; 480 481 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 482 KASSERT(size % PAGE_SIZE == 0); 483 484 npages = size / PAGE_SIZE; 485 486 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 487 for (i = 0; i < npages; i++) { 488 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 489 } 490 491 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 492 for (i = 0; i < npages; i++) { 493 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 494 } 495 } 496 497 void 498 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 499 { 500 msan_orig_t orig; 501 502 orig = kmsan_md_orig_encode(type, pc); 503 kmsan_origin_fill(addr, orig, size); 504 } 505 506 void 507 kmsan_mark(void *addr, size_t size, uint8_t c) 508 { 509 kmsan_shadow_fill(addr, c, size); 510 } 511 512 void 513 kmsan_check_mbuf(void *buf) 514 { 515 struct mbuf *m = buf; 516 517 do { 518 kmsan_shadow_check(mtod(m, void *), m->m_len, "if_transmit"); 519 } while ((m = m->m_next) != NULL); 520 } 521 522 void 523 kmsan_check_buf(void *buf) 524 { 525 buf_t *bp = buf; 526 527 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite"); 528 } 529 530 void 531 kmsan_init(void *stack) 532 { 533 /* MD initialization. */ 534 kmsan_md_init(); 535 536 /* Map the stack. */ 537 kmsan_shadow_map(stack, USPACE); 538 539 /* Initialize the TLS for curlwp. */ 540 msan_lwp0.ctx = 1; 541 curlwp->l_kmsan = &msan_lwp0; 542 543 /* Now officially enabled. */ 544 kmsan_enabled = true; 545 } 546 547 /* -------------------------------------------------------------------------- */ 548 549 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 550 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 551 552 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 553 { 554 return kmsan_meta_get(addr, size); 555 } 556 557 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 558 { 559 return kmsan_meta_get(addr, size); 560 } 561 562 #define MSAN_META_FUNC(size) \ 563 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 564 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 565 { \ 566 return kmsan_meta_get(addr, size); \ 567 } \ 568 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 569 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 570 { \ 571 return kmsan_meta_get(addr, size); \ 572 } 573 574 MSAN_META_FUNC(1) 575 MSAN_META_FUNC(2) 576 MSAN_META_FUNC(4) 577 MSAN_META_FUNC(8) 578 579 void __msan_instrument_asm_store(void *, size_t); 580 msan_orig_t __msan_chain_origin(msan_orig_t); 581 void __msan_poison_alloca(void *, uint64_t, char *); 582 void __msan_unpoison_alloca(void *, uint64_t); 583 void __msan_warning(msan_orig_t); 584 msan_tls_t *__msan_get_context_state(void); 585 586 void __msan_instrument_asm_store(void *addr, size_t size) 587 { 588 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 589 } 590 591 msan_orig_t __msan_chain_origin(msan_orig_t origin) 592 { 593 return origin; 594 } 595 596 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 597 { 598 msan_orig_t orig; 599 600 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 601 kmsan_origin_fill(addr, orig, size); 602 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 603 } 604 605 void __msan_unpoison_alloca(void *addr, uint64_t size) 606 { 607 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 608 } 609 610 void __msan_warning(msan_orig_t origin) 611 { 612 if (__predict_false(!kmsan_enabled)) 613 return; 614 kmsan_report_inline(origin, __RET_ADDR); 615 } 616 617 msan_tls_t *__msan_get_context_state(void) 618 { 619 msan_lwp_t *lwp; 620 621 if (__predict_false(!kmsan_enabled)) 622 return &dummy_tls; 623 lwp = curlwp->l_kmsan; 624 625 return &lwp->tls[lwp->ctx]; 626 } 627 628 /* -------------------------------------------------------------------------- */ 629 630 /* 631 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 632 * initialized areas properly. 633 */ 634 635 void *kmsan_memcpy(void *dst, const void *src, size_t len) 636 { 637 /* No kmsan_check_arg, because inlined. */ 638 kmsan_init_ret(sizeof(void *)); 639 if (__predict_true(len != 0)) { 640 kmsan_meta_copy(dst, src, len); 641 } 642 return __builtin_memcpy(dst, src, len); 643 } 644 645 int 646 kmsan_memcmp(const void *b1, const void *b2, size_t len) 647 { 648 const uint8_t *_b1 = b1, *_b2 = b2; 649 size_t i; 650 651 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), "memcmp"); 652 kmsan_init_ret(sizeof(int)); 653 654 for (i = 0; i < len; i++) { 655 if (*_b1 != *_b2) { 656 kmsan_shadow_check(b1, i + 1, "memcmp"); 657 kmsan_shadow_check(b2, i + 1, "memcmp"); 658 return *_b1 - *_b2; 659 } 660 _b1++, _b2++; 661 } 662 663 return 0; 664 } 665 666 void *kmsan_memset(void *dst, int c, size_t len) 667 { 668 /* No kmsan_check_arg, because inlined. */ 669 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 670 kmsan_init_ret(sizeof(void *)); 671 return __builtin_memset(dst, c, len); 672 } 673 674 void *kmsan_memmove(void *dst, const void *src, size_t len) 675 { 676 /* No kmsan_check_arg, because inlined. */ 677 if (__predict_true(len != 0)) { 678 kmsan_meta_copy(dst, src, len); 679 } 680 kmsan_init_ret(sizeof(void *)); 681 return __builtin_memmove(dst, src, len); 682 } 683 684 __strong_alias(__msan_memcpy, kmsan_memcpy) 685 __strong_alias(__msan_memset, kmsan_memset) 686 __strong_alias(__msan_memmove, kmsan_memmove) 687 688 char * 689 kmsan_strcpy(char *dst, const char *src) 690 { 691 const char *_src = src; 692 char *_dst = dst; 693 size_t len = 0; 694 695 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy"); 696 697 while (1) { 698 len++; 699 *dst = *src; 700 if (*src == '\0') 701 break; 702 src++, dst++; 703 } 704 705 kmsan_shadow_check(_src, len, "strcpy"); 706 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 707 kmsan_init_ret(sizeof(char *)); 708 return _dst; 709 } 710 711 int 712 kmsan_strcmp(const char *s1, const char *s2) 713 { 714 const char *_s1 = s1, *_s2 = s2; 715 size_t len = 0; 716 717 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp"); 718 kmsan_init_ret(sizeof(int)); 719 720 while (1) { 721 len++; 722 if (*s1 != *s2) 723 break; 724 if (*s1 == '\0') { 725 kmsan_shadow_check(_s1, len, "strcmp"); 726 kmsan_shadow_check(_s2, len, "strcmp"); 727 return 0; 728 } 729 s1++, s2++; 730 } 731 732 kmsan_shadow_check(_s1, len, "strcmp"); 733 kmsan_shadow_check(_s2, len, "strcmp"); 734 735 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 736 } 737 738 size_t 739 kmsan_strlen(const char *str) 740 { 741 const char *s; 742 743 kmsan_check_arg(sizeof(str), "strlen"); 744 745 s = str; 746 while (1) { 747 if (*s == '\0') 748 break; 749 s++; 750 } 751 752 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen"); 753 kmsan_init_ret(sizeof(size_t)); 754 return (s - str); 755 } 756 757 #undef kcopy 758 #undef copystr 759 #undef copyin 760 #undef copyout 761 #undef copyinstr 762 #undef copyoutstr 763 764 int kmsan_kcopy(const void *, void *, size_t); 765 int kmsan_copystr(const void *, void *, size_t, size_t *); 766 int kmsan_copyin(const void *, void *, size_t); 767 int kmsan_copyout(const void *, void *, size_t); 768 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 769 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 770 771 int kcopy(const void *, void *, size_t); 772 int copystr(const void *, void *, size_t, size_t *); 773 int copyin(const void *, void *, size_t); 774 int copyout(const void *, void *, size_t); 775 int copyinstr(const void *, void *, size_t, size_t *); 776 int copyoutstr(const void *, void *, size_t, size_t *); 777 778 int 779 kmsan_kcopy(const void *src, void *dst, size_t len) 780 { 781 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), "kcopy"); 782 if (__predict_true(len != 0)) { 783 kmsan_meta_copy(dst, src, len); 784 } 785 kmsan_init_ret(sizeof(int)); 786 return kcopy(src, dst, len); 787 } 788 789 int 790 kmsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 791 { 792 size_t _done; 793 int ret; 794 795 kmsan_check_arg(sizeof(kfaddr) + sizeof(kdaddr) + 796 sizeof(len) + sizeof(done), "copystr"); 797 ret = copystr(kfaddr, kdaddr, len, &_done); 798 if (ret == 0) 799 kmsan_meta_copy(kdaddr, kfaddr, _done); 800 if (done != NULL) { 801 *done = _done; 802 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 803 } 804 kmsan_init_ret(sizeof(int)); 805 806 return ret; 807 } 808 809 int 810 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 811 { 812 int ret; 813 814 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), "copyin"); 815 ret = copyin(uaddr, kaddr, len); 816 if (ret == 0) 817 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 818 kmsan_init_ret(sizeof(int)); 819 820 return ret; 821 } 822 823 int 824 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 825 { 826 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), "copyout"); 827 kmsan_shadow_check(kaddr, len, "copyout"); 828 kmsan_init_ret(sizeof(int)); 829 return copyout(kaddr, uaddr, len); 830 } 831 832 int 833 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 834 { 835 size_t _done; 836 int ret; 837 838 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 839 sizeof(len) + sizeof(done), "copyinstr"); 840 ret = copyinstr(uaddr, kaddr, len, &_done); 841 if (ret == 0) 842 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 843 if (done != NULL) { 844 *done = _done; 845 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 846 } 847 kmsan_init_ret(sizeof(int)); 848 849 return ret; 850 } 851 852 int 853 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 854 { 855 size_t _done; 856 int ret; 857 858 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 859 sizeof(len) + sizeof(done), "copyoutstr"); 860 ret = copyoutstr(kaddr, uaddr, len, &_done); 861 kmsan_shadow_check(kaddr, _done, "copyoutstr"); 862 if (done != NULL) { 863 *done = _done; 864 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 865 } 866 kmsan_init_ret(sizeof(int)); 867 868 return ret; 869 } 870 871 /* -------------------------------------------------------------------------- */ 872 873 #undef _ucas_32 874 #undef _ucas_32_mp 875 #undef _ucas_64 876 #undef _ucas_64_mp 877 #undef _ufetch_8 878 #undef _ufetch_16 879 #undef _ufetch_32 880 #undef _ufetch_64 881 #undef _ustore_8 882 #undef _ustore_16 883 #undef _ustore_32 884 #undef _ustore_64 885 886 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 887 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 888 int 889 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 890 uint32_t *ret) 891 { 892 int _ret; 893 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 894 sizeof(new) + sizeof(ret), "ucas_32"); 895 _ret = _ucas_32(uaddr, old, new, ret); 896 if (_ret == 0) 897 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 898 kmsan_init_ret(sizeof(int)); 899 return _ret; 900 } 901 902 #ifdef __HAVE_UCAS_MP 903 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 904 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 905 int 906 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 907 uint32_t *ret) 908 { 909 int _ret; 910 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 911 sizeof(new) + sizeof(ret), "ucas_32_mp"); 912 _ret = _ucas_32_mp(uaddr, old, new, ret); 913 if (_ret == 0) 914 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 915 kmsan_init_ret(sizeof(int)); 916 return _ret; 917 } 918 #endif 919 920 #ifdef _LP64 921 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 922 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 923 int 924 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 925 uint64_t *ret) 926 { 927 int _ret; 928 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 929 sizeof(new) + sizeof(ret), "ucas_64"); 930 _ret = _ucas_64(uaddr, old, new, ret); 931 if (_ret == 0) 932 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 933 kmsan_init_ret(sizeof(int)); 934 return _ret; 935 } 936 937 #ifdef __HAVE_UCAS_MP 938 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 939 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 940 int 941 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 942 uint64_t *ret) 943 { 944 int _ret; 945 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 946 sizeof(new) + sizeof(ret), "ucas_64_mp"); 947 _ret = _ucas_64_mp(uaddr, old, new, ret); 948 if (_ret == 0) 949 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 950 kmsan_init_ret(sizeof(int)); 951 return _ret; 952 } 953 #endif 954 #endif 955 956 int _ufetch_8(const uint8_t *, uint8_t *); 957 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 958 int 959 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 960 { 961 int _ret; 962 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8"); 963 _ret = _ufetch_8(uaddr, valp); 964 if (_ret == 0) 965 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 966 kmsan_init_ret(sizeof(int)); 967 return _ret; 968 } 969 970 int _ufetch_16(const uint16_t *, uint16_t *); 971 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 972 int 973 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 974 { 975 int _ret; 976 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16"); 977 _ret = _ufetch_16(uaddr, valp); 978 if (_ret == 0) 979 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 980 kmsan_init_ret(sizeof(int)); 981 return _ret; 982 } 983 984 int _ufetch_32(const uint32_t *, uint32_t *); 985 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 986 int 987 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 988 { 989 int _ret; 990 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32"); 991 _ret = _ufetch_32(uaddr, valp); 992 if (_ret == 0) 993 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 994 kmsan_init_ret(sizeof(int)); 995 return _ret; 996 } 997 998 #ifdef _LP64 999 int _ufetch_64(const uint64_t *, uint64_t *); 1000 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1001 int 1002 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1003 { 1004 int _ret; 1005 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64"); 1006 _ret = _ufetch_64(uaddr, valp); 1007 if (_ret == 0) 1008 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1009 kmsan_init_ret(sizeof(int)); 1010 return _ret; 1011 } 1012 #endif 1013 1014 int _ustore_8(uint8_t *, uint8_t); 1015 int kmsan__ustore_8(uint8_t *, uint8_t); 1016 int 1017 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1018 { 1019 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8"); 1020 kmsan_init_ret(sizeof(int)); 1021 return _ustore_8(uaddr, val); 1022 } 1023 1024 int _ustore_16(uint16_t *, uint16_t); 1025 int kmsan__ustore_16(uint16_t *, uint16_t); 1026 int 1027 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1028 { 1029 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16"); 1030 kmsan_init_ret(sizeof(int)); 1031 return _ustore_16(uaddr, val); 1032 } 1033 1034 int _ustore_32(uint32_t *, uint32_t); 1035 int kmsan__ustore_32(uint32_t *, uint32_t); 1036 int 1037 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1038 { 1039 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32"); 1040 kmsan_init_ret(sizeof(int)); 1041 return _ustore_32(uaddr, val); 1042 } 1043 1044 #ifdef _LP64 1045 int _ustore_64(uint64_t *, uint64_t); 1046 int kmsan__ustore_64(uint64_t *, uint64_t); 1047 int 1048 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1049 { 1050 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64"); 1051 kmsan_init_ret(sizeof(int)); 1052 return _ustore_64(uaddr, val); 1053 } 1054 #endif 1055 1056 /* -------------------------------------------------------------------------- */ 1057 1058 #undef atomic_add_32 1059 #undef atomic_add_int 1060 #undef atomic_add_long 1061 #undef atomic_add_ptr 1062 #undef atomic_add_64 1063 #undef atomic_add_32_nv 1064 #undef atomic_add_int_nv 1065 #undef atomic_add_long_nv 1066 #undef atomic_add_ptr_nv 1067 #undef atomic_add_64_nv 1068 #undef atomic_and_32 1069 #undef atomic_and_uint 1070 #undef atomic_and_ulong 1071 #undef atomic_and_64 1072 #undef atomic_and_32_nv 1073 #undef atomic_and_uint_nv 1074 #undef atomic_and_ulong_nv 1075 #undef atomic_and_64_nv 1076 #undef atomic_or_32 1077 #undef atomic_or_uint 1078 #undef atomic_or_ulong 1079 #undef atomic_or_64 1080 #undef atomic_or_32_nv 1081 #undef atomic_or_uint_nv 1082 #undef atomic_or_ulong_nv 1083 #undef atomic_or_64_nv 1084 #undef atomic_cas_32 1085 #undef atomic_cas_uint 1086 #undef atomic_cas_ulong 1087 #undef atomic_cas_ptr 1088 #undef atomic_cas_64 1089 #undef atomic_cas_32_ni 1090 #undef atomic_cas_uint_ni 1091 #undef atomic_cas_ulong_ni 1092 #undef atomic_cas_ptr_ni 1093 #undef atomic_cas_64_ni 1094 #undef atomic_swap_32 1095 #undef atomic_swap_uint 1096 #undef atomic_swap_ulong 1097 #undef atomic_swap_ptr 1098 #undef atomic_swap_64 1099 #undef atomic_dec_32 1100 #undef atomic_dec_uint 1101 #undef atomic_dec_ulong 1102 #undef atomic_dec_ptr 1103 #undef atomic_dec_64 1104 #undef atomic_dec_32_nv 1105 #undef atomic_dec_uint_nv 1106 #undef atomic_dec_ulong_nv 1107 #undef atomic_dec_ptr_nv 1108 #undef atomic_dec_64_nv 1109 #undef atomic_inc_32 1110 #undef atomic_inc_uint 1111 #undef atomic_inc_ulong 1112 #undef atomic_inc_ptr 1113 #undef atomic_inc_64 1114 #undef atomic_inc_32_nv 1115 #undef atomic_inc_uint_nv 1116 #undef atomic_inc_ulong_nv 1117 #undef atomic_inc_ptr_nv 1118 #undef atomic_inc_64_nv 1119 1120 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1121 void atomic_add_##name(volatile targ1 *, targ2); \ 1122 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1123 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1124 { \ 1125 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1126 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1127 atomic_add_##name(ptr, val); \ 1128 } \ 1129 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1130 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1131 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1132 { \ 1133 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1134 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1135 kmsan_init_ret(sizeof(tret)); \ 1136 return atomic_add_##name##_nv(ptr, val); \ 1137 } 1138 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1139 void atomic_and_##name(volatile targ1 *, targ2); \ 1140 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1141 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1142 { \ 1143 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1144 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1145 atomic_and_##name(ptr, val); \ 1146 } \ 1147 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1148 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1149 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1150 { \ 1151 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1152 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1153 kmsan_init_ret(sizeof(tret)); \ 1154 return atomic_and_##name##_nv(ptr, val); \ 1155 } 1156 1157 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1158 void atomic_or_##name(volatile targ1 *, targ2); \ 1159 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1160 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1161 { \ 1162 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1163 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1164 atomic_or_##name(ptr, val); \ 1165 } \ 1166 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1167 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1168 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1169 { \ 1170 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1171 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1172 kmsan_init_ret(sizeof(tret)); \ 1173 return atomic_or_##name##_nv(ptr, val); \ 1174 } 1175 1176 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1177 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1178 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1179 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1180 { \ 1181 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1182 __func__); \ 1183 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1184 kmsan_init_ret(sizeof(tret)); \ 1185 return atomic_cas_##name(ptr, exp, new); \ 1186 } \ 1187 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1188 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1189 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1190 { \ 1191 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1192 __func__); \ 1193 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1194 kmsan_init_ret(sizeof(tret)); \ 1195 return atomic_cas_##name##_ni(ptr, exp, new); \ 1196 } 1197 1198 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1199 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1200 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1201 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1202 { \ 1203 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1204 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1205 kmsan_init_ret(sizeof(tret)); \ 1206 return atomic_swap_##name(ptr, val); \ 1207 } 1208 1209 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1210 void atomic_dec_##name(volatile targ1 *); \ 1211 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1212 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1213 { \ 1214 kmsan_check_arg(sizeof(ptr), __func__); \ 1215 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1216 atomic_dec_##name(ptr); \ 1217 } \ 1218 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1219 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1220 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1221 { \ 1222 kmsan_check_arg(sizeof(ptr), __func__); \ 1223 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1224 kmsan_init_ret(sizeof(tret)); \ 1225 return atomic_dec_##name##_nv(ptr); \ 1226 } 1227 1228 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1229 void atomic_inc_##name(volatile targ1 *); \ 1230 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1231 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1232 { \ 1233 kmsan_check_arg(sizeof(ptr), __func__); \ 1234 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1235 atomic_inc_##name(ptr); \ 1236 } \ 1237 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1238 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1239 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1240 { \ 1241 kmsan_check_arg(sizeof(ptr), __func__); \ 1242 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1243 kmsan_init_ret(sizeof(tret)); \ 1244 return atomic_inc_##name##_nv(ptr); \ 1245 } 1246 1247 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1248 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1249 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1250 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1251 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1252 1253 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1254 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1255 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1256 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1257 1258 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1259 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1260 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1261 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1262 1263 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1264 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1265 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1266 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1267 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1268 1269 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1270 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1271 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1272 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1273 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1274 1275 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1276 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1277 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1278 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1279 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1280 1281 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1282 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1283 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1284 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1285 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1286 1287 /* -------------------------------------------------------------------------- */ 1288 1289 #include <sys/bus.h> 1290 1291 #undef bus_space_read_multi_1 1292 #undef bus_space_read_multi_2 1293 #undef bus_space_read_multi_4 1294 #undef bus_space_read_multi_8 1295 #undef bus_space_read_multi_stream_1 1296 #undef bus_space_read_multi_stream_2 1297 #undef bus_space_read_multi_stream_4 1298 #undef bus_space_read_multi_stream_8 1299 #undef bus_space_read_region_1 1300 #undef bus_space_read_region_2 1301 #undef bus_space_read_region_4 1302 #undef bus_space_read_region_8 1303 #undef bus_space_read_region_stream_1 1304 #undef bus_space_read_region_stream_2 1305 #undef bus_space_read_region_stream_4 1306 #undef bus_space_read_region_stream_8 1307 1308 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1309 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1310 bus_size_t, uint##bits##_t *, bus_size_t); \ 1311 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1312 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1313 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1314 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1315 bus_size_t count) \ 1316 { \ 1317 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1318 sizeof(uint##bits##_t) * count); \ 1319 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1320 } \ 1321 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1322 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1323 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1324 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1325 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1326 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1327 bus_size_t count) \ 1328 { \ 1329 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1330 sizeof(uint##bits##_t) * count); \ 1331 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1332 } \ 1333 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1334 bus_size_t, uint##bits##_t *, bus_size_t); \ 1335 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1336 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1337 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1338 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1339 bus_size_t count) \ 1340 { \ 1341 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1342 sizeof(uint##bits##_t) * count); \ 1343 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1344 } \ 1345 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1346 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1347 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1348 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1349 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1350 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1351 bus_size_t count) \ 1352 { \ 1353 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1354 sizeof(uint##bits##_t) * count); \ 1355 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1356 } 1357 1358 MSAN_BUS_READ_FUNC(1, 8) 1359 MSAN_BUS_READ_FUNC(2, 16) 1360 MSAN_BUS_READ_FUNC(4, 32) 1361 MSAN_BUS_READ_FUNC(8, 64) 1362 1363 #undef bus_space_write_multi_1 1364 #undef bus_space_write_multi_2 1365 #undef bus_space_write_multi_4 1366 #undef bus_space_write_multi_8 1367 #undef bus_space_write_multi_stream_1 1368 #undef bus_space_write_multi_stream_2 1369 #undef bus_space_write_multi_stream_4 1370 #undef bus_space_write_multi_stream_8 1371 #undef bus_space_write_region_1 1372 #undef bus_space_write_region_2 1373 #undef bus_space_write_region_4 1374 #undef bus_space_write_region_8 1375 #undef bus_space_write_region_stream_1 1376 #undef bus_space_write_region_stream_2 1377 #undef bus_space_write_region_stream_4 1378 #undef bus_space_write_region_stream_8 1379 1380 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1381 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1382 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1383 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1384 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1385 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1386 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1387 bus_size_t count) \ 1388 { \ 1389 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1390 "bus_space_write"); \ 1391 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1392 } \ 1393 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1394 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1395 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1396 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1397 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1398 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1399 bus_size_t count) \ 1400 { \ 1401 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1402 "bus_space_write"); \ 1403 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1404 } \ 1405 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1406 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1407 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1408 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1409 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1410 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1411 bus_size_t count) \ 1412 { \ 1413 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1414 "bus_space_write"); \ 1415 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1416 } \ 1417 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1418 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1419 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1420 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1421 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1422 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1423 bus_size_t count) \ 1424 { \ 1425 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1426 "bus_space_write"); \ 1427 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1428 } 1429 1430 MSAN_BUS_WRITE_FUNC(1, 8) 1431 MSAN_BUS_WRITE_FUNC(2, 16) 1432 MSAN_BUS_WRITE_FUNC(4, 32) 1433 MSAN_BUS_WRITE_FUNC(8, 64) 1434 1435 /* -------------------------------------------------------------------------- */ 1436 1437 #include <sys/mbuf.h> 1438 1439 static void 1440 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1441 bool init, uintptr_t pc) 1442 { 1443 if (init) { 1444 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1445 } else { 1446 kmsan_shadow_check(buf + offset, len, "dma_sync_linear"); 1447 } 1448 } 1449 1450 static void 1451 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1452 bool init, uintptr_t pc) 1453 { 1454 bus_addr_t minlen; 1455 1456 for (; m != NULL && len != 0; m = m->m_next) { 1457 if (offset >= m->m_len) { 1458 offset -= m->m_len; 1459 continue; 1460 } 1461 1462 minlen = MIN(len, m->m_len - offset); 1463 1464 if (init) { 1465 kmsan_shadow_fill(mtod(m, char *) + offset, 1466 KMSAN_STATE_INITED, minlen); 1467 } else { 1468 kmsan_shadow_check(mtod(m, char *) + offset, 1469 minlen, "dma_sync_mbuf"); 1470 } 1471 1472 offset = 0; 1473 len -= minlen; 1474 } 1475 } 1476 1477 static void 1478 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1479 bool init, uintptr_t pc) 1480 { 1481 bus_size_t minlen, resid; 1482 struct iovec *iov; 1483 int i; 1484 1485 if (uio->uio_vmspace != NULL) 1486 return; 1487 1488 resid = uio->uio_resid; 1489 iov = uio->uio_iov; 1490 1491 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1492 minlen = MIN(resid, iov[i].iov_len); 1493 1494 if (init) { 1495 kmsan_shadow_fill(iov[i].iov_base, 1496 KMSAN_STATE_INITED, minlen); 1497 } else { 1498 kmsan_shadow_check(iov[i].iov_base, minlen, 1499 "dma_sync_uio"); 1500 } 1501 1502 resid -= minlen; 1503 } 1504 } 1505 1506 void 1507 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1508 { 1509 bool init; 1510 1511 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1512 return; 1513 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1514 1515 switch (map->dm_buftype) { 1516 case KMSAN_DMA_LINEAR: 1517 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1518 __RET_ADDR); 1519 break; 1520 case KMSAN_DMA_MBUF: 1521 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1522 __RET_ADDR); 1523 break; 1524 case KMSAN_DMA_UIO: 1525 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1526 __RET_ADDR); 1527 break; 1528 case KMSAN_DMA_RAW: 1529 break; 1530 default: 1531 panic("%s: impossible", __func__); 1532 } 1533 } 1534 1535 void 1536 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1537 { 1538 map->dm_buf = buf; 1539 map->dm_buflen = buflen; 1540 map->dm_buftype = type; 1541 } 1542
Indexes created Sun Sep 21 15:09:59 GMT 2025