1 /* $NetBSD: subr_msan.c,v 1.19 2023/04/11 10:19:56 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KMSAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.19 2023/04/11 10:19:56 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/kprintf.h> 42 #include <sys/kmem.h> 43 #include <sys/mbuf.h> 44 #include <sys/buf.h> 45 #include <sys/cpu.h> 46 #include <sys/msan.h> 47 48 #include <ddb/db_active.h> 49 50 static void kmsan_printf(const char *, ...); 51 52 void kmsan_init_arg(size_t); 53 void kmsan_init_ret(size_t); 54 55 #ifdef KMSAN_PANIC 56 #define REPORT panic 57 #else 58 #define REPORT kmsan_printf 59 #endif 60 61 /* -------------------------------------------------------------------------- */ 62 63 /* 64 * Part of the compiler ABI. 65 */ 66 67 typedef uint32_t msan_orig_t; 68 69 typedef struct { 70 uint8_t *shad; 71 msan_orig_t *orig; 72 } msan_meta_t; 73 74 #define MSAN_PARAM_SIZE 800 75 #define MSAN_RETVAL_SIZE 800 76 typedef struct { 77 uint8_t param[MSAN_PARAM_SIZE]; 78 uint8_t retval[MSAN_RETVAL_SIZE]; 79 uint8_t _va_arg[MSAN_PARAM_SIZE]; 80 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 81 uint64_t va_arg_overflow_size; 82 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 83 msan_orig_t retval_origin; 84 msan_orig_t origin; 85 } msan_tls_t; 86 87 /* -------------------------------------------------------------------------- */ 88 89 /* The MD code. */ 90 #include <machine/msan.h> 91 92 /* -------------------------------------------------------------------------- */ 93 94 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 95 #define MSAN_NCONTEXT 16 96 97 typedef struct { 98 size_t ctx; 99 msan_tls_t tls[MSAN_NCONTEXT]; 100 } msan_lwp_t; 101 102 static msan_tls_t dummy_tls; 103 104 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 105 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 106 static msan_lwp_t msan_lwp0; 107 static bool kmsan_enabled __read_mostly; 108 109 /* -------------------------------------------------------------------------- */ 110 111 static bool kmsan_reporting = false; 112 113 static inline void 114 kmsan_printf(const char *fmt, ...) 115 { 116 va_list ap; 117 118 va_start(ap, fmt); 119 kprintf(fmt, TOCONS, NULL, NULL, ap); 120 va_end(ap); 121 } 122 123 static inline const char * 124 kmsan_orig_name(int type) 125 { 126 switch (type) { 127 case KMSAN_TYPE_STACK: 128 return "Stack"; 129 case KMSAN_TYPE_KMEM: 130 return "Kmem"; 131 case KMSAN_TYPE_MALLOC: 132 return "Malloc"; 133 case KMSAN_TYPE_POOL: 134 return "Pool"; 135 case KMSAN_TYPE_UVM: 136 return "Uvm"; 137 default: 138 return "Unknown"; 139 } 140 } 141 142 /* 143 * The format of the string is: "----var@function". Parse it to display a nice 144 * warning. 145 */ 146 static void 147 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 148 { 149 unsigned long symstart; 150 const char *mod, *sym; 151 msan_orig_t *orig; 152 const char *typename; 153 char *var, *fn; 154 uintptr_t ptr; 155 char buf[128]; 156 int type; 157 int s; 158 159 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 160 return; 161 162 kmsan_reporting = true; 163 __insn_barrier(); 164 165 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 166 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 167 168 if (*orig == 0) { 169 REPORT("MSan: Uninitialized Memory In %s At Offset " 170 "%zu\n", hook, off); 171 goto out; 172 } 173 174 kmsan_md_orig_decode(*orig, &type, &ptr); 175 typename = kmsan_orig_name(type); 176 177 if (kmsan_md_is_pc(ptr)) { 178 s = pserialize_read_enter(); 179 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC) || 180 ksyms_getval(mod, sym, &symstart, KSYMS_PROC)) { 181 REPORT("MSan: Uninitialized %s Memory In %s " 182 "At Offset %zu/%zu, IP %p\n", typename, hook, off, 183 size, (void *)ptr); 184 } else { 185 char soff[16] = ""; 186 187 if ((vaddr_t)ptr < symstart) { 188 snprintf(soff, sizeof(soff), "-0x%"PRIxVADDR, 189 symstart - (vaddr_t)ptr); 190 } else if ((vaddr_t)ptr > symstart) { 191 snprintf(soff, sizeof(soff), "+0x%"PRIxVADDR, 192 (vaddr_t)ptr - symstart); 193 } 194 REPORT("MSan: Uninitialized %s Memory In %s " 195 "At Offset %zu/%zu, From %s%s%lx\n", 196 typename, hook, 197 off, size, sym, 198 ((unsigned long)ptr < symstart ? "-" : 199 (unsigned long)ptr > symstart ? "+" : 200 ""), 201 (unsigned long)ptr - symstart); 202 } 203 pserialize_read_exit(s); 204 } else { 205 var = (char *)ptr + 4; 206 strlcpy(buf, var, sizeof(buf)); 207 var = buf; 208 fn = __builtin_strchr(buf, '@'); 209 *fn++ = '\0'; 210 REPORT("MSan: Uninitialized %s Memory In %s At Offset " 211 "%zu, Variable '%s' From %s()\n", typename, hook, off, 212 var, fn); 213 } 214 215 out: 216 kmsan_md_unwind(); 217 __insn_barrier(); 218 kmsan_reporting = false; 219 } 220 221 static void 222 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 223 { 224 const char *mod, *sym; 225 const char *typename; 226 char *var, *fn; 227 uintptr_t ptr; 228 char buf[128]; 229 int type; 230 int s; 231 232 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 233 return; 234 235 kmsan_reporting = true; 236 __insn_barrier(); 237 238 if (orig == 0) { 239 REPORT("MSan: Uninitialized Variable In %p\n", 240 (void *)pc); 241 goto out; 242 } 243 244 kmsan_md_orig_decode(orig, &type, &ptr); 245 typename = kmsan_orig_name(type); 246 247 if (kmsan_md_is_pc(ptr)) { 248 s = pserialize_read_enter(); 249 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 250 REPORT("MSan: Uninitialized %s Memory, " 251 "Origin %x\n", typename, orig); 252 } else { 253 REPORT("MSan: Uninitialized %s Memory " 254 "From %s()\n", typename, sym); 255 } 256 pserialize_read_exit(s); 257 } else { 258 var = (char *)ptr + 4; 259 strlcpy(buf, var, sizeof(buf)); 260 var = buf; 261 fn = __builtin_strchr(buf, '@'); 262 *fn++ = '\0'; 263 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 264 var, fn); 265 } 266 267 out: 268 kmsan_md_unwind(); 269 __insn_barrier(); 270 kmsan_reporting = false; 271 } 272 273 /* -------------------------------------------------------------------------- */ 274 275 static inline msan_meta_t 276 kmsan_meta_get(void *addr, size_t size) 277 { 278 msan_meta_t ret; 279 280 if (__predict_false(!kmsan_enabled)) { 281 ret.shad = msan_dummy_shad; 282 ret.orig = (msan_orig_t *)msan_dummy_orig; 283 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 284 ret.shad = msan_dummy_shad; 285 ret.orig = (msan_orig_t *)msan_dummy_orig; 286 } else { 287 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 288 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 289 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 290 } 291 292 return ret; 293 } 294 295 static inline void 296 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 297 { 298 msan_orig_t *orig; 299 size_t i; 300 301 if (__predict_false(!kmsan_enabled)) 302 return; 303 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 304 return; 305 306 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 307 size += ((uintptr_t)orig & 0x3); 308 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 309 310 for (i = 0; i < size; i += 4) { 311 orig[i / 4] = o; 312 } 313 } 314 315 static inline void 316 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 317 { 318 uint8_t *shad; 319 320 if (__predict_false(!kmsan_enabled)) 321 return; 322 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 323 return; 324 325 shad = kmsan_md_addr_to_shad(addr); 326 __builtin_memset(shad, c, size); 327 } 328 329 static inline void 330 kmsan_meta_copy(void *dst, const void *src, size_t size) 331 { 332 uint8_t *orig_src, *orig_dst; 333 uint8_t *shad_src, *shad_dst; 334 msan_orig_t *_src, *_dst; 335 size_t i; 336 337 if (__predict_false(!kmsan_enabled)) 338 return; 339 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 340 return; 341 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 342 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 343 return; 344 } 345 346 shad_src = kmsan_md_addr_to_shad(src); 347 shad_dst = kmsan_md_addr_to_shad(dst); 348 __builtin_memmove(shad_dst, shad_src, size); 349 350 orig_src = kmsan_md_addr_to_orig(src); 351 orig_dst = kmsan_md_addr_to_orig(dst); 352 for (i = 0; i < size; i++) { 353 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 354 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 355 *_dst = *_src; 356 orig_src++; 357 orig_dst++; 358 } 359 } 360 361 static inline void 362 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 363 { 364 uint8_t *shad; 365 size_t i; 366 367 if (__predict_false(!kmsan_enabled)) 368 return; 369 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 370 return; 371 372 shad = kmsan_md_addr_to_shad(addr); 373 for (i = 0; i < size; i++) { 374 if (__predict_true(shad[i] == 0)) 375 continue; 376 kmsan_report_hook((const char *)addr + i, size, i, hook); 377 break; 378 } 379 } 380 381 void 382 kmsan_init_arg(size_t n) 383 { 384 msan_lwp_t *lwp; 385 uint8_t *arg; 386 387 if (__predict_false(!kmsan_enabled)) 388 return; 389 lwp = curlwp->l_kmsan; 390 arg = lwp->tls[lwp->ctx].param; 391 __builtin_memset(arg, 0, n); 392 } 393 394 void 395 kmsan_init_ret(size_t n) 396 { 397 msan_lwp_t *lwp; 398 uint8_t *arg; 399 400 if (__predict_false(!kmsan_enabled)) 401 return; 402 lwp = curlwp->l_kmsan; 403 arg = lwp->tls[lwp->ctx].retval; 404 __builtin_memset(arg, 0, n); 405 } 406 407 static void 408 kmsan_check_arg(size_t size, const char *hook) 409 { 410 msan_lwp_t *lwp; 411 uint8_t *arg; 412 size_t i; 413 414 if (__predict_false(!kmsan_enabled)) 415 return; 416 lwp = curlwp->l_kmsan; 417 arg = lwp->tls[lwp->ctx].param; 418 419 for (i = 0; i < size; i++) { 420 if (__predict_true(arg[i] == 0)) 421 continue; 422 kmsan_report_hook((const char *)arg + i, size, i, hook); 423 break; 424 } 425 } 426 427 void 428 kmsan_lwp_alloc(struct lwp *l) 429 { 430 msan_lwp_t *lwp; 431 432 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 433 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 434 lwp->ctx = 1; 435 436 l->l_kmsan = lwp; 437 } 438 439 void 440 kmsan_lwp_free(struct lwp *l) 441 { 442 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 443 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 444 } 445 446 void kmsan_intr_enter(void); 447 void kmsan_intr_leave(void); 448 void kmsan_softint(struct lwp *); 449 450 void 451 kmsan_intr_enter(void) 452 { 453 msan_lwp_t *lwp; 454 455 if (__predict_false(!kmsan_enabled)) 456 return; 457 lwp = curlwp->l_kmsan; 458 459 lwp->ctx++; 460 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 461 kmsan_enabled = false; 462 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 463 } 464 465 kmsan_init_arg(sizeof(void *)); 466 } 467 468 void 469 kmsan_intr_leave(void) 470 { 471 msan_lwp_t *lwp; 472 473 if (__predict_false(!kmsan_enabled)) 474 return; 475 lwp = curlwp->l_kmsan; 476 477 if (__predict_false(lwp->ctx == 0)) { 478 kmsan_enabled = false; 479 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 480 } 481 lwp->ctx--; 482 } 483 484 void 485 kmsan_softint(struct lwp *l) 486 { 487 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 488 } 489 490 /* -------------------------------------------------------------------------- */ 491 492 void 493 kmsan_shadow_map(void *addr, size_t size) 494 { 495 size_t npages, i; 496 vaddr_t va; 497 498 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 499 KASSERT(size % PAGE_SIZE == 0); 500 501 npages = size / PAGE_SIZE; 502 503 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 504 for (i = 0; i < npages; i++) { 505 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 506 } 507 508 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 509 for (i = 0; i < npages; i++) { 510 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 511 } 512 } 513 514 void 515 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 516 { 517 msan_orig_t orig; 518 519 orig = kmsan_md_orig_encode(type, pc); 520 kmsan_origin_fill(addr, orig, size); 521 } 522 523 void 524 kmsan_mark(void *addr, size_t size, uint8_t c) 525 { 526 kmsan_shadow_fill(addr, c, size); 527 } 528 529 void 530 kmsan_check_mbuf(void *buf) 531 { 532 struct mbuf *m = buf; 533 534 do { 535 kmsan_shadow_check(mtod(m, void *), m->m_len, "MbufChain"); 536 } while ((m = m->m_next) != NULL); 537 } 538 539 void 540 kmsan_check_buf(void *buf) 541 { 542 buf_t *bp = buf; 543 544 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite()"); 545 } 546 547 void 548 kmsan_init(void *stack) 549 { 550 /* MD initialization. */ 551 kmsan_md_init(); 552 553 /* Map the stack. */ 554 kmsan_shadow_map(stack, USPACE); 555 556 /* Initialize the TLS for curlwp. */ 557 msan_lwp0.ctx = 1; 558 curlwp->l_kmsan = &msan_lwp0; 559 560 /* Now officially enabled. */ 561 kmsan_enabled = true; 562 } 563 564 /* -------------------------------------------------------------------------- */ 565 566 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 567 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 568 569 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 570 { 571 return kmsan_meta_get(addr, size); 572 } 573 574 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 575 { 576 return kmsan_meta_get(addr, size); 577 } 578 579 #define MSAN_META_FUNC(size) \ 580 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 581 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 582 { \ 583 return kmsan_meta_get(addr, size); \ 584 } \ 585 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 586 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 587 { \ 588 return kmsan_meta_get(addr, size); \ 589 } 590 591 MSAN_META_FUNC(1) 592 MSAN_META_FUNC(2) 593 MSAN_META_FUNC(4) 594 MSAN_META_FUNC(8) 595 596 void __msan_instrument_asm_store(void *, size_t); 597 msan_orig_t __msan_chain_origin(msan_orig_t); 598 void __msan_poison_alloca(void *, uint64_t, char *); 599 void __msan_unpoison_alloca(void *, uint64_t); 600 void __msan_warning(msan_orig_t); 601 msan_tls_t *__msan_get_context_state(void); 602 603 void __msan_instrument_asm_store(void *addr, size_t size) 604 { 605 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 606 } 607 608 msan_orig_t __msan_chain_origin(msan_orig_t origin) 609 { 610 return origin; 611 } 612 613 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 614 { 615 msan_orig_t orig; 616 617 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 618 kmsan_origin_fill(addr, orig, size); 619 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 620 } 621 622 void __msan_unpoison_alloca(void *addr, uint64_t size) 623 { 624 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 625 } 626 627 void __msan_warning(msan_orig_t origin) 628 { 629 if (__predict_false(!kmsan_enabled)) 630 return; 631 kmsan_report_inline(origin, __RET_ADDR); 632 } 633 634 msan_tls_t *__msan_get_context_state(void) 635 { 636 msan_lwp_t *lwp; 637 638 if (__predict_false(!kmsan_enabled)) 639 return &dummy_tls; 640 lwp = curlwp->l_kmsan; 641 642 return &lwp->tls[lwp->ctx]; 643 } 644 645 /* -------------------------------------------------------------------------- */ 646 647 /* 648 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 649 * initialized areas properly. 650 */ 651 652 void *kmsan_memcpy(void *dst, const void *src, size_t len) 653 { 654 /* No kmsan_check_arg, because inlined. */ 655 kmsan_init_ret(sizeof(void *)); 656 if (__predict_true(len != 0)) { 657 kmsan_meta_copy(dst, src, len); 658 } 659 return __builtin_memcpy(dst, src, len); 660 } 661 662 int 663 kmsan_memcmp(const void *b1, const void *b2, size_t len) 664 { 665 const uint8_t *_b1 = b1, *_b2 = b2; 666 size_t i; 667 668 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 669 "memcmp():args"); 670 kmsan_init_ret(sizeof(int)); 671 672 for (i = 0; i < len; i++) { 673 if (*_b1 != *_b2) { 674 kmsan_shadow_check(b1, i + 1, "memcmp():arg1"); 675 kmsan_shadow_check(b2, i + 1, "memcmp():arg2"); 676 return *_b1 - *_b2; 677 } 678 _b1++, _b2++; 679 } 680 681 return 0; 682 } 683 684 void *kmsan_memset(void *dst, int c, size_t len) 685 { 686 /* No kmsan_check_arg, because inlined. */ 687 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 688 kmsan_init_ret(sizeof(void *)); 689 return __builtin_memset(dst, c, len); 690 } 691 692 void *kmsan_memmove(void *dst, const void *src, size_t len) 693 { 694 /* No kmsan_check_arg, because inlined. */ 695 if (__predict_true(len != 0)) { 696 kmsan_meta_copy(dst, src, len); 697 } 698 kmsan_init_ret(sizeof(void *)); 699 return __builtin_memmove(dst, src, len); 700 } 701 702 __strong_alias(__msan_memcpy, kmsan_memcpy) 703 __strong_alias(__msan_memset, kmsan_memset) 704 __strong_alias(__msan_memmove, kmsan_memmove) 705 706 char * 707 kmsan_strcpy(char *dst, const char *src) 708 { 709 const char *_src = src; 710 char *_dst = dst; 711 size_t len = 0; 712 713 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 714 715 while (1) { 716 len++; 717 *dst = *src; 718 if (*src == '\0') 719 break; 720 src++, dst++; 721 } 722 723 kmsan_shadow_check(_src, len, "strcpy():arg2"); 724 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 725 kmsan_init_ret(sizeof(char *)); 726 return _dst; 727 } 728 729 int 730 kmsan_strcmp(const char *s1, const char *s2) 731 { 732 const char *_s1 = s1, *_s2 = s2; 733 size_t len = 0; 734 735 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 736 kmsan_init_ret(sizeof(int)); 737 738 while (1) { 739 len++; 740 if (*s1 != *s2) 741 break; 742 if (*s1 == '\0') { 743 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 744 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 745 return 0; 746 } 747 s1++, s2++; 748 } 749 750 kmsan_shadow_check(_s1, len, "strcmp():arg1"); 751 kmsan_shadow_check(_s2, len, "strcmp():arg2"); 752 753 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 754 } 755 756 size_t 757 kmsan_strlen(const char *str) 758 { 759 const char *s; 760 761 kmsan_check_arg(sizeof(str), "strlen():args"); 762 763 s = str; 764 while (1) { 765 if (*s == '\0') 766 break; 767 s++; 768 } 769 770 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen():arg1"); 771 kmsan_init_ret(sizeof(size_t)); 772 return (s - str); 773 } 774 775 char * 776 kmsan_strcat(char *dst, const char *src) 777 { 778 size_t ldst, lsrc; 779 char *ret; 780 781 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcat():args"); 782 783 ldst = __builtin_strlen(dst); 784 lsrc = __builtin_strlen(src); 785 kmsan_shadow_check(dst, ldst + 1, "strcat():arg1"); 786 kmsan_shadow_check(src, lsrc + 1, "strcat():arg2"); 787 ret = __builtin_strcat(dst, src); 788 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, ldst + lsrc + 1); 789 790 kmsan_init_ret(sizeof(char *)); 791 return ret; 792 } 793 794 char * 795 kmsan_strchr(const char *s, int c) 796 { 797 char *ret; 798 799 kmsan_check_arg(sizeof(s) + sizeof(c), "strchr():args"); 800 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strchr():arg1"); 801 ret = __builtin_strchr(s, c); 802 803 kmsan_init_ret(sizeof(char *)); 804 return ret; 805 } 806 807 char * 808 kmsan_strrchr(const char *s, int c) 809 { 810 char *ret; 811 812 kmsan_check_arg(sizeof(s) + sizeof(c), "strrchr():args"); 813 kmsan_shadow_check(s, __builtin_strlen(s) + 1, "strrchr():arg1"); 814 ret = __builtin_strrchr(s, c); 815 816 kmsan_init_ret(sizeof(char *)); 817 return ret; 818 } 819 820 #undef kcopy 821 #undef copyin 822 #undef copyout 823 #undef copyinstr 824 #undef copyoutstr 825 826 int kmsan_kcopy(const void *, void *, size_t); 827 int kmsan_copyin(const void *, void *, size_t); 828 int kmsan_copyout(const void *, void *, size_t); 829 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 830 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 831 832 int kcopy(const void *, void *, size_t); 833 int copyin(const void *, void *, size_t); 834 int copyout(const void *, void *, size_t); 835 int copyinstr(const void *, void *, size_t, size_t *); 836 int copyoutstr(const void *, void *, size_t, size_t *); 837 838 int 839 kmsan_kcopy(const void *src, void *dst, size_t len) 840 { 841 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), 842 "kcopy():args"); 843 if (__predict_true(len != 0)) { 844 kmsan_meta_copy(dst, src, len); 845 } 846 kmsan_init_ret(sizeof(int)); 847 return kcopy(src, dst, len); 848 } 849 850 int 851 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 852 { 853 int ret; 854 855 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 856 "copyin():args"); 857 ret = copyin(uaddr, kaddr, len); 858 if (ret == 0) 859 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 860 kmsan_init_ret(sizeof(int)); 861 862 return ret; 863 } 864 865 int 866 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 867 { 868 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 869 "copyout():args"); 870 kmsan_shadow_check(kaddr, len, "copyout():arg1"); 871 kmsan_init_ret(sizeof(int)); 872 return copyout(kaddr, uaddr, len); 873 } 874 875 int 876 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 877 { 878 size_t _done; 879 int ret; 880 881 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 882 sizeof(len) + sizeof(done), "copyinstr():args"); 883 ret = copyinstr(uaddr, kaddr, len, &_done); 884 if (ret == 0 || ret == ENAMETOOLONG) 885 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 886 if (done != NULL) { 887 *done = _done; 888 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 889 } 890 kmsan_init_ret(sizeof(int)); 891 892 return ret; 893 } 894 895 int 896 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 897 { 898 size_t _done; 899 int ret; 900 901 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 902 sizeof(len) + sizeof(done), "copyoutstr():args"); 903 ret = copyoutstr(kaddr, uaddr, len, &_done); 904 kmsan_shadow_check(kaddr, _done, "copyoutstr():arg1"); 905 if (done != NULL) { 906 *done = _done; 907 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 908 } 909 kmsan_init_ret(sizeof(int)); 910 911 return ret; 912 } 913 914 /* -------------------------------------------------------------------------- */ 915 916 #undef _ucas_32 917 #undef _ucas_32_mp 918 #undef _ucas_64 919 #undef _ucas_64_mp 920 #undef _ufetch_8 921 #undef _ufetch_16 922 #undef _ufetch_32 923 #undef _ufetch_64 924 #undef _ustore_8 925 #undef _ustore_16 926 #undef _ustore_32 927 #undef _ustore_64 928 929 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 930 int kmsan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 931 int 932 kmsan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 933 uint32_t *ret) 934 { 935 int _ret; 936 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 937 sizeof(new) + sizeof(ret), "ucas_32():args"); 938 _ret = _ucas_32(uaddr, old, new, ret); 939 if (_ret == 0) 940 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 941 kmsan_init_ret(sizeof(int)); 942 return _ret; 943 } 944 945 #ifdef __HAVE_UCAS_MP 946 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 947 int kmsan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 948 int 949 kmsan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 950 uint32_t *ret) 951 { 952 int _ret; 953 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 954 sizeof(new) + sizeof(ret), "ucas_32_mp():args"); 955 _ret = _ucas_32_mp(uaddr, old, new, ret); 956 if (_ret == 0) 957 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 958 kmsan_init_ret(sizeof(int)); 959 return _ret; 960 } 961 #endif 962 963 #ifdef _LP64 964 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 965 int kmsan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 966 int 967 kmsan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 968 uint64_t *ret) 969 { 970 int _ret; 971 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 972 sizeof(new) + sizeof(ret), "ucas_64():args"); 973 _ret = _ucas_64(uaddr, old, new, ret); 974 if (_ret == 0) 975 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 976 kmsan_init_ret(sizeof(int)); 977 return _ret; 978 } 979 980 #ifdef __HAVE_UCAS_MP 981 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 982 int kmsan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 983 int 984 kmsan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 985 uint64_t *ret) 986 { 987 int _ret; 988 kmsan_check_arg(sizeof(uaddr) + sizeof(old) + 989 sizeof(new) + sizeof(ret), "ucas_64_mp():args"); 990 _ret = _ucas_64_mp(uaddr, old, new, ret); 991 if (_ret == 0) 992 kmsan_shadow_fill(ret, KMSAN_STATE_INITED, sizeof(*ret)); 993 kmsan_init_ret(sizeof(int)); 994 return _ret; 995 } 996 #endif 997 #endif 998 999 int _ufetch_8(const uint8_t *, uint8_t *); 1000 int kmsan__ufetch_8(const uint8_t *, uint8_t *); 1001 int 1002 kmsan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 1003 { 1004 int _ret; 1005 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_8():args"); 1006 _ret = _ufetch_8(uaddr, valp); 1007 if (_ret == 0) 1008 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1009 kmsan_init_ret(sizeof(int)); 1010 return _ret; 1011 } 1012 1013 int _ufetch_16(const uint16_t *, uint16_t *); 1014 int kmsan__ufetch_16(const uint16_t *, uint16_t *); 1015 int 1016 kmsan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 1017 { 1018 int _ret; 1019 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_16():args"); 1020 _ret = _ufetch_16(uaddr, valp); 1021 if (_ret == 0) 1022 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1023 kmsan_init_ret(sizeof(int)); 1024 return _ret; 1025 } 1026 1027 int _ufetch_32(const uint32_t *, uint32_t *); 1028 int kmsan__ufetch_32(const uint32_t *, uint32_t *); 1029 int 1030 kmsan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 1031 { 1032 int _ret; 1033 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_32():args"); 1034 _ret = _ufetch_32(uaddr, valp); 1035 if (_ret == 0) 1036 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1037 kmsan_init_ret(sizeof(int)); 1038 return _ret; 1039 } 1040 1041 #ifdef _LP64 1042 int _ufetch_64(const uint64_t *, uint64_t *); 1043 int kmsan__ufetch_64(const uint64_t *, uint64_t *); 1044 int 1045 kmsan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 1046 { 1047 int _ret; 1048 kmsan_check_arg(sizeof(uaddr) + sizeof(valp), "ufetch_64():args"); 1049 _ret = _ufetch_64(uaddr, valp); 1050 if (_ret == 0) 1051 kmsan_shadow_fill(valp, KMSAN_STATE_INITED, sizeof(*valp)); 1052 kmsan_init_ret(sizeof(int)); 1053 return _ret; 1054 } 1055 #endif 1056 1057 int _ustore_8(uint8_t *, uint8_t); 1058 int kmsan__ustore_8(uint8_t *, uint8_t); 1059 int 1060 kmsan__ustore_8(uint8_t *uaddr, uint8_t val) 1061 { 1062 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_8():args"); 1063 kmsan_init_ret(sizeof(int)); 1064 return _ustore_8(uaddr, val); 1065 } 1066 1067 int _ustore_16(uint16_t *, uint16_t); 1068 int kmsan__ustore_16(uint16_t *, uint16_t); 1069 int 1070 kmsan__ustore_16(uint16_t *uaddr, uint16_t val) 1071 { 1072 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_16():args"); 1073 kmsan_init_ret(sizeof(int)); 1074 return _ustore_16(uaddr, val); 1075 } 1076 1077 int _ustore_32(uint32_t *, uint32_t); 1078 int kmsan__ustore_32(uint32_t *, uint32_t); 1079 int 1080 kmsan__ustore_32(uint32_t *uaddr, uint32_t val) 1081 { 1082 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_32():args"); 1083 kmsan_init_ret(sizeof(int)); 1084 return _ustore_32(uaddr, val); 1085 } 1086 1087 #ifdef _LP64 1088 int _ustore_64(uint64_t *, uint64_t); 1089 int kmsan__ustore_64(uint64_t *, uint64_t); 1090 int 1091 kmsan__ustore_64(uint64_t *uaddr, uint64_t val) 1092 { 1093 kmsan_check_arg(sizeof(uaddr) + sizeof(val), "ustore_64():args"); 1094 kmsan_init_ret(sizeof(int)); 1095 return _ustore_64(uaddr, val); 1096 } 1097 #endif 1098 1099 /* -------------------------------------------------------------------------- */ 1100 1101 #undef atomic_add_32 1102 #undef atomic_add_int 1103 #undef atomic_add_long 1104 #undef atomic_add_ptr 1105 #undef atomic_add_64 1106 #undef atomic_add_32_nv 1107 #undef atomic_add_int_nv 1108 #undef atomic_add_long_nv 1109 #undef atomic_add_ptr_nv 1110 #undef atomic_add_64_nv 1111 #undef atomic_and_32 1112 #undef atomic_and_uint 1113 #undef atomic_and_ulong 1114 #undef atomic_and_64 1115 #undef atomic_and_32_nv 1116 #undef atomic_and_uint_nv 1117 #undef atomic_and_ulong_nv 1118 #undef atomic_and_64_nv 1119 #undef atomic_or_32 1120 #undef atomic_or_uint 1121 #undef atomic_or_ulong 1122 #undef atomic_or_64 1123 #undef atomic_or_32_nv 1124 #undef atomic_or_uint_nv 1125 #undef atomic_or_ulong_nv 1126 #undef atomic_or_64_nv 1127 #undef atomic_cas_32 1128 #undef atomic_cas_uint 1129 #undef atomic_cas_ulong 1130 #undef atomic_cas_ptr 1131 #undef atomic_cas_64 1132 #undef atomic_cas_32_ni 1133 #undef atomic_cas_uint_ni 1134 #undef atomic_cas_ulong_ni 1135 #undef atomic_cas_ptr_ni 1136 #undef atomic_cas_64_ni 1137 #undef atomic_swap_32 1138 #undef atomic_swap_uint 1139 #undef atomic_swap_ulong 1140 #undef atomic_swap_ptr 1141 #undef atomic_swap_64 1142 #undef atomic_dec_32 1143 #undef atomic_dec_uint 1144 #undef atomic_dec_ulong 1145 #undef atomic_dec_ptr 1146 #undef atomic_dec_64 1147 #undef atomic_dec_32_nv 1148 #undef atomic_dec_uint_nv 1149 #undef atomic_dec_ulong_nv 1150 #undef atomic_dec_ptr_nv 1151 #undef atomic_dec_64_nv 1152 #undef atomic_inc_32 1153 #undef atomic_inc_uint 1154 #undef atomic_inc_ulong 1155 #undef atomic_inc_ptr 1156 #undef atomic_inc_64 1157 #undef atomic_inc_32_nv 1158 #undef atomic_inc_uint_nv 1159 #undef atomic_inc_ulong_nv 1160 #undef atomic_inc_ptr_nv 1161 #undef atomic_inc_64_nv 1162 1163 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 1164 void atomic_add_##name(volatile targ1 *, targ2); \ 1165 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 1166 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 1167 { \ 1168 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1169 "atomic_add_" #name "():args"); \ 1170 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1171 "atomic_add_" #name "():arg1"); \ 1172 atomic_add_##name(ptr, val); \ 1173 } \ 1174 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1175 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 1176 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1177 { \ 1178 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1179 "atomic_add_" #name "_nv():args"); \ 1180 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1181 "atomic_add_" #name "_nv():arg1"); \ 1182 kmsan_init_ret(sizeof(tret)); \ 1183 return atomic_add_##name##_nv(ptr, val); \ 1184 } 1185 1186 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 1187 void atomic_and_##name(volatile targ1 *, targ2); \ 1188 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 1189 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 1190 { \ 1191 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1192 "atomic_and_" #name "():args"); \ 1193 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1194 "atomic_and_" #name "():arg1"); \ 1195 atomic_and_##name(ptr, val); \ 1196 } \ 1197 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1198 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 1199 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1200 { \ 1201 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1202 "atomic_and_" #name "_nv():args"); \ 1203 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1204 "atomic_and_" #name "_nv():arg1"); \ 1205 kmsan_init_ret(sizeof(tret)); \ 1206 return atomic_and_##name##_nv(ptr, val); \ 1207 } 1208 1209 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 1210 void atomic_or_##name(volatile targ1 *, targ2); \ 1211 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 1212 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 1213 { \ 1214 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1215 "atomic_or_" #name "():args"); \ 1216 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1217 "atomic_or_" #name "():arg1"); \ 1218 atomic_or_##name(ptr, val); \ 1219 } \ 1220 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1221 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 1222 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 1223 { \ 1224 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1225 "atomic_or_" #name "_nv():args"); \ 1226 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1227 "atomic_or_" #name "_nv():arg1"); \ 1228 kmsan_init_ret(sizeof(tret)); \ 1229 return atomic_or_##name##_nv(ptr, val); \ 1230 } 1231 1232 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 1233 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1234 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 1235 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1236 { \ 1237 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1238 "atomic_cas_" #name "():args"); \ 1239 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1240 "atomic_cas_" #name "():arg1"); \ 1241 kmsan_init_ret(sizeof(tret)); \ 1242 return atomic_cas_##name(ptr, exp, new); \ 1243 } \ 1244 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1245 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1246 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1247 { \ 1248 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1249 "atomic_cas_" #name "_ni():args"); \ 1250 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1251 "atomic_cas_" #name "_ni():arg1"); \ 1252 kmsan_init_ret(sizeof(tret)); \ 1253 return atomic_cas_##name##_ni(ptr, exp, new); \ 1254 } 1255 1256 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1257 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1258 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1259 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1260 { \ 1261 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1262 "atomic_swap_" #name "():args"); \ 1263 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1264 "atomic_swap_" #name "():arg1"); \ 1265 kmsan_init_ret(sizeof(tret)); \ 1266 return atomic_swap_##name(ptr, val); \ 1267 } 1268 1269 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1270 void atomic_dec_##name(volatile targ1 *); \ 1271 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1272 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1273 { \ 1274 kmsan_check_arg(sizeof(ptr), \ 1275 "atomic_dec_" #name "():args"); \ 1276 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1277 "atomic_dec_" #name "():arg1"); \ 1278 atomic_dec_##name(ptr); \ 1279 } \ 1280 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1281 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1282 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1283 { \ 1284 kmsan_check_arg(sizeof(ptr), \ 1285 "atomic_dec_" #name "_nv():args"); \ 1286 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1287 "atomic_dec_" #name "_nv():arg1"); \ 1288 kmsan_init_ret(sizeof(tret)); \ 1289 return atomic_dec_##name##_nv(ptr); \ 1290 } 1291 1292 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1293 void atomic_inc_##name(volatile targ1 *); \ 1294 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1295 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1296 { \ 1297 kmsan_check_arg(sizeof(ptr), \ 1298 "atomic_inc_" #name "():args"); \ 1299 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1300 "atomic_inc_" #name "():arg1"); \ 1301 atomic_inc_##name(ptr); \ 1302 } \ 1303 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1304 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1305 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1306 { \ 1307 kmsan_check_arg(sizeof(ptr), \ 1308 "atomic_inc_" #name "_nv():args"); \ 1309 kmsan_shadow_check((const void *)(uintptr_t)ptr, sizeof(tret), \ 1310 "atomic_inc_" #name "_nv():arg1"); \ 1311 kmsan_init_ret(sizeof(tret)); \ 1312 return atomic_inc_##name##_nv(ptr); \ 1313 } 1314 1315 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1316 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1317 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1318 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1319 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1320 1321 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1322 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1323 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1324 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1325 1326 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1327 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1328 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1329 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1330 1331 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1332 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1333 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1334 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1335 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1336 1337 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1338 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1339 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1340 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1341 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1342 1343 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1344 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1345 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1346 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1347 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1348 1349 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1350 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1351 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1352 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1353 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1354 1355 /* -------------------------------------------------------------------------- */ 1356 1357 #include <sys/bus.h> 1358 1359 #undef bus_space_read_multi_1 1360 #undef bus_space_read_multi_2 1361 #undef bus_space_read_multi_4 1362 #undef bus_space_read_multi_8 1363 #undef bus_space_read_multi_stream_1 1364 #undef bus_space_read_multi_stream_2 1365 #undef bus_space_read_multi_stream_4 1366 #undef bus_space_read_multi_stream_8 1367 #undef bus_space_read_region_1 1368 #undef bus_space_read_region_2 1369 #undef bus_space_read_region_4 1370 #undef bus_space_read_region_8 1371 #undef bus_space_read_region_stream_1 1372 #undef bus_space_read_region_stream_2 1373 #undef bus_space_read_region_stream_4 1374 #undef bus_space_read_region_stream_8 1375 1376 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1377 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1378 bus_size_t, uint##bits##_t *, bus_size_t); \ 1379 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1380 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1381 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1382 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1383 bus_size_t count) \ 1384 { \ 1385 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1386 sizeof(uint##bits##_t) * count); \ 1387 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1388 } \ 1389 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1390 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1391 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1392 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1393 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1394 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1395 bus_size_t count) \ 1396 { \ 1397 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1398 sizeof(uint##bits##_t) * count); \ 1399 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1400 } \ 1401 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1402 bus_size_t, uint##bits##_t *, bus_size_t); \ 1403 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1404 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1405 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1406 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1407 bus_size_t count) \ 1408 { \ 1409 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1410 sizeof(uint##bits##_t) * count); \ 1411 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1412 } \ 1413 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1414 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1415 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1416 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1417 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1418 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1419 bus_size_t count) \ 1420 { \ 1421 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1422 sizeof(uint##bits##_t) * count); \ 1423 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1424 } 1425 1426 MSAN_BUS_READ_FUNC(1, 8) 1427 MSAN_BUS_READ_FUNC(2, 16) 1428 MSAN_BUS_READ_FUNC(4, 32) 1429 MSAN_BUS_READ_FUNC(8, 64) 1430 1431 #undef bus_space_write_multi_1 1432 #undef bus_space_write_multi_2 1433 #undef bus_space_write_multi_4 1434 #undef bus_space_write_multi_8 1435 #undef bus_space_write_multi_stream_1 1436 #undef bus_space_write_multi_stream_2 1437 #undef bus_space_write_multi_stream_4 1438 #undef bus_space_write_multi_stream_8 1439 #undef bus_space_write_region_1 1440 #undef bus_space_write_region_2 1441 #undef bus_space_write_region_4 1442 #undef bus_space_write_region_8 1443 #undef bus_space_write_region_stream_1 1444 #undef bus_space_write_region_stream_2 1445 #undef bus_space_write_region_stream_4 1446 #undef bus_space_write_region_stream_8 1447 1448 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1449 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1450 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1451 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1452 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1453 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1454 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1455 bus_size_t count) \ 1456 { \ 1457 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1458 "bus_space_write()"); \ 1459 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1460 } \ 1461 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1462 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1463 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1464 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1465 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1466 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1467 bus_size_t count) \ 1468 { \ 1469 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1470 "bus_space_write()"); \ 1471 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1472 } \ 1473 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1474 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1475 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1476 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1477 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1478 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1479 bus_size_t count) \ 1480 { \ 1481 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1482 "bus_space_write()"); \ 1483 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1484 } \ 1485 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1486 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1487 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1488 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1489 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1490 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1491 bus_size_t count) \ 1492 { \ 1493 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1494 "bus_space_write()"); \ 1495 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1496 } 1497 1498 MSAN_BUS_WRITE_FUNC(1, 8) 1499 MSAN_BUS_WRITE_FUNC(2, 16) 1500 MSAN_BUS_WRITE_FUNC(4, 32) 1501 MSAN_BUS_WRITE_FUNC(8, 64) 1502 1503 /* -------------------------------------------------------------------------- */ 1504 1505 #include <sys/mbuf.h> 1506 1507 static void 1508 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1509 bool init, uintptr_t pc) 1510 { 1511 if (init) { 1512 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1513 } else { 1514 kmsan_shadow_check(buf + offset, len, "LinearDmaSyncOp"); 1515 } 1516 } 1517 1518 static void 1519 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1520 bool init, uintptr_t pc) 1521 { 1522 bus_addr_t minlen; 1523 1524 for (; m != NULL && len != 0; m = m->m_next) { 1525 if (offset >= m->m_len) { 1526 offset -= m->m_len; 1527 continue; 1528 } 1529 1530 minlen = MIN(len, m->m_len - offset); 1531 1532 if (init) { 1533 kmsan_shadow_fill(mtod(m, char *) + offset, 1534 KMSAN_STATE_INITED, minlen); 1535 } else { 1536 kmsan_shadow_check(mtod(m, char *) + offset, 1537 minlen, "MbufDmaSyncOp"); 1538 } 1539 1540 offset = 0; 1541 len -= minlen; 1542 } 1543 } 1544 1545 static void 1546 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1547 bool init, uintptr_t pc) 1548 { 1549 bus_size_t minlen, resid; 1550 struct iovec *iov; 1551 int i; 1552 1553 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1554 return; 1555 1556 resid = uio->uio_resid; 1557 iov = uio->uio_iov; 1558 1559 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1560 minlen = MIN(resid, iov[i].iov_len); 1561 1562 if (init) { 1563 kmsan_shadow_fill(iov[i].iov_base, 1564 KMSAN_STATE_INITED, minlen); 1565 } else { 1566 kmsan_shadow_check(iov[i].iov_base, minlen, 1567 "UioDmaSyncOp"); 1568 } 1569 1570 resid -= minlen; 1571 } 1572 } 1573 1574 void 1575 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1576 { 1577 bool init; 1578 1579 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1580 return; 1581 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1582 1583 switch (map->dm_buftype) { 1584 case KMSAN_DMA_LINEAR: 1585 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1586 __RET_ADDR); 1587 break; 1588 case KMSAN_DMA_MBUF: 1589 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1590 __RET_ADDR); 1591 break; 1592 case KMSAN_DMA_UIO: 1593 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1594 __RET_ADDR); 1595 break; 1596 case KMSAN_DMA_RAW: 1597 break; 1598 default: 1599 panic("%s: impossible", __func__); 1600 } 1601 } 1602 1603 void 1604 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1605 { 1606 map->dm_buf = buf; 1607 map->dm_buflen = buflen; 1608 map->dm_buftype = type; 1609 } 1610
Indexes created Sat Sep 20 22:09:52 GMT 2025