subr_msan.c revision 1.1
1 /* $NetBSD: subr_msan.c,v 1.1 2019/11/14 16:23:52 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define KMSAN_NO_INST 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.1 2019/11/14 16:23:52 maxv Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/device.h> 39 #include <sys/kernel.h> 40 #include <sys/param.h> 41 #include <sys/conf.h> 42 #include <sys/systm.h> 43 #include <sys/types.h> 44 #include <sys/kprintf.h> 45 #include <sys/kmem.h> 46 #include <sys/mbuf.h> 47 #include <sys/buf.h> 48 #include <sys/cpu.h> 49 #include <sys/msan.h> 50 51 #include <uvm/uvm.h> 52 53 static void kmsan_printf(const char *, ...); 54 55 #ifdef KMSAN_PANIC 56 #define REPORT panic 57 #else 58 #define REPORT kmsan_printf 59 #endif 60 61 /* -------------------------------------------------------------------------- */ 62 63 /* 64 * Part of the compiler ABI. 65 */ 66 67 typedef uint32_t msan_orig_t; 68 69 typedef struct { 70 uint8_t *shad; 71 msan_orig_t *orig; 72 } msan_meta_t; 73 74 #define MSAN_PARAM_SIZE 800 75 #define MSAN_RETVAL_SIZE 800 76 typedef struct { 77 uint8_t param[MSAN_PARAM_SIZE]; 78 uint8_t retval[MSAN_RETVAL_SIZE]; 79 uint8_t _va_arg[MSAN_PARAM_SIZE]; 80 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 81 uint64_t va_arg_overflow_size; 82 msan_orig_t param_origin[MSAN_PARAM_SIZE]; 83 msan_orig_t retval_origin; 84 msan_orig_t origin; 85 } msan_tls_t; 86 87 /* -------------------------------------------------------------------------- */ 88 89 /* The MD code. */ 90 #include <machine/msan.h> 91 92 /* -------------------------------------------------------------------------- */ 93 94 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 95 #define MSAN_NCONTEXT 16 96 97 typedef struct { 98 size_t ctx; 99 msan_tls_t tls[MSAN_NCONTEXT]; 100 } msan_lwp_t; 101 102 static msan_tls_t dummy_tls; 103 104 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 105 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 106 static msan_lwp_t msan_lwp0; 107 static bool kmsan_enabled __read_mostly; 108 109 /* -------------------------------------------------------------------------- */ 110 111 static bool kmsan_reporting = false; 112 113 static inline void 114 kmsan_printf(const char *fmt, ...) 115 { 116 va_list ap; 117 118 va_start(ap, fmt); 119 kprintf(fmt, TOCONS, NULL, NULL, ap); 120 va_end(ap); 121 } 122 123 static inline const char * 124 kmsan_orig_name(int type) 125 { 126 switch (type) { 127 case KMSAN_TYPE_STACK: 128 return "Stack"; 129 case KMSAN_TYPE_KMEM: 130 return "Kmem"; 131 case KMSAN_TYPE_MALLOC: 132 return "Malloc"; 133 case KMSAN_TYPE_POOL: 134 return "Pool"; 135 case KMSAN_TYPE_UVM: 136 return "Uvm"; 137 default: 138 return "Unknown"; 139 } 140 } 141 142 /* 143 * The format of the string is: "----var@function". Parse it to display a nice 144 * warning. 145 */ 146 static void 147 kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) 148 { 149 const char *mod, *sym; 150 extern int db_active; 151 msan_orig_t *orig; 152 const char *typename; 153 char *var, *fn; 154 uintptr_t ptr; 155 char buf[128]; 156 int type; 157 158 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 159 return; 160 161 kmsan_reporting = true; 162 __insn_barrier(); 163 164 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 165 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 166 167 if (*orig == 0) { 168 REPORT("MSan: Uninitialized Memory In %s() At Offset " 169 "%zu\n", hook, off); 170 goto out; 171 } 172 173 kmsan_md_orig_decode(*orig, &type, &ptr); 174 typename = kmsan_orig_name(type); 175 176 if (kmsan_md_is_pc(ptr)) { 177 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 178 REPORT("MSan: Uninitialized %s Memory In %s() " 179 "At Offset %zu, IP %p\n", typename, hook, off, 180 (void *)ptr); 181 } else { 182 REPORT("MSan: Uninitialized %s Memory In %s() " 183 "At Offset %zu, From %s()\n", typename, hook, off, 184 sym); 185 } 186 } else { 187 var = (char *)ptr + 4; 188 strlcpy(buf, var, sizeof(buf)); 189 var = buf; 190 fn = strchr(buf, '@'); 191 *fn++ = '\0'; 192 REPORT("MSan: Uninitialized %s Memory In %s() At Offset " 193 "%zu, Variable '%s' From %s()\n", typename, hook, off, 194 var, fn); 195 } 196 197 out: 198 kmsan_md_unwind(); 199 __insn_barrier(); 200 kmsan_reporting = false; 201 } 202 203 static void 204 kmsan_report_inline(msan_orig_t orig, unsigned long pc) 205 { 206 const char *mod, *sym; 207 extern int db_active; 208 const char *typename; 209 char *var, *fn; 210 uintptr_t ptr; 211 char buf[128]; 212 int type; 213 214 if (__predict_false(panicstr != NULL || db_active || kmsan_reporting)) 215 return; 216 217 kmsan_reporting = true; 218 __insn_barrier(); 219 220 if (orig == 0) { 221 REPORT("MSan: Uninitialized Variable In %p\n", 222 (void *)pc); 223 goto out; 224 } 225 226 kmsan_md_orig_decode(orig, &type, &ptr); 227 typename = kmsan_orig_name(type); 228 229 if (kmsan_md_is_pc(ptr)) { 230 if (ksyms_getname(&mod, &sym, (vaddr_t)ptr, KSYMS_PROC)) { 231 REPORT("MSan: Uninitialized %s Memory, " 232 "Origin %x\n", typename, orig); 233 } else { 234 REPORT("MSan: Uninitialized %s Memory " 235 "From %s()\n", typename, sym); 236 } 237 } else { 238 var = (char *)ptr + 4; 239 strlcpy(buf, var, sizeof(buf)); 240 var = buf; 241 fn = strchr(buf, '@'); 242 *fn++ = '\0'; 243 REPORT("MSan: Uninitialized Variable '%s' From %s()\n", 244 var, fn); 245 } 246 247 out: 248 kmsan_md_unwind(); 249 __insn_barrier(); 250 kmsan_reporting = false; 251 } 252 253 /* -------------------------------------------------------------------------- */ 254 255 static inline msan_meta_t 256 kmsan_meta_get(void *addr, size_t size) 257 { 258 msan_meta_t ret; 259 260 if (__predict_false(!kmsan_enabled)) { 261 ret.shad = msan_dummy_shad; 262 ret.orig = (msan_orig_t *)msan_dummy_orig; 263 } else if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) { 264 ret.shad = msan_dummy_shad; 265 ret.orig = (msan_orig_t *)msan_dummy_orig; 266 } else { 267 ret.shad = (void *)kmsan_md_addr_to_shad(addr); 268 ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 269 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & ~0x3); 270 } 271 272 return ret; 273 } 274 275 static inline void 276 kmsan_origin_fill(void *addr, msan_orig_t o, size_t size) 277 { 278 msan_orig_t *orig; 279 size_t i; 280 281 if (__predict_false(!kmsan_enabled)) 282 return; 283 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 284 return; 285 286 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr); 287 size += ((uintptr_t)orig & 0x3); 288 orig = (msan_orig_t *)((uintptr_t)orig & ~0x3); 289 290 for (i = 0; i < size; i += 4) { 291 orig[i / 4] = o; 292 } 293 } 294 295 static inline void 296 kmsan_shadow_fill(void *addr, uint8_t c, size_t size) 297 { 298 uint8_t *shad; 299 300 if (__predict_false(!kmsan_enabled)) 301 return; 302 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 303 return; 304 305 shad = kmsan_md_addr_to_shad(addr); 306 __builtin_memset(shad, c, size); 307 } 308 309 static inline void 310 kmsan_meta_copy(void *dst, const void *src, size_t size) 311 { 312 uint8_t *orig_src, *orig_dst; 313 uint8_t *shad_src, *shad_dst; 314 msan_orig_t *_src, *_dst; 315 size_t i; 316 317 if (__predict_false(!kmsan_enabled)) 318 return; 319 if (__predict_false(kmsan_md_unsupported((vaddr_t)dst))) 320 return; 321 if (__predict_false(kmsan_md_unsupported((vaddr_t)src))) { 322 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, size); 323 return; 324 } 325 326 shad_src = kmsan_md_addr_to_shad(src); 327 shad_dst = kmsan_md_addr_to_shad(dst); 328 __builtin_memmove(shad_dst, shad_src, size); 329 330 orig_src = kmsan_md_addr_to_orig(src); 331 orig_dst = kmsan_md_addr_to_orig(dst); 332 for (i = 0; i < size; i++) { 333 _src = (msan_orig_t *)((uintptr_t)orig_src & ~0x3); 334 _dst = (msan_orig_t *)((uintptr_t)orig_dst & ~0x3); 335 *_dst = *_src; 336 orig_src++; 337 orig_dst++; 338 } 339 } 340 341 static inline void 342 kmsan_shadow_check(const void *addr, size_t size, const char *hook) 343 { 344 uint8_t *shad; 345 size_t i; 346 347 if (__predict_false(!kmsan_enabled)) 348 return; 349 if (__predict_false(kmsan_md_unsupported((vaddr_t)addr))) 350 return; 351 352 shad = kmsan_md_addr_to_shad(addr); 353 for (i = 0; i < size; i++) { 354 if (__predict_true(shad[i] == 0)) 355 continue; 356 kmsan_report_hook((const char *)addr + i, size, i, hook); 357 break; 358 } 359 } 360 361 void kmsan_init_arg(size_t); 362 void kmsan_init_ret(size_t); 363 364 void 365 kmsan_init_arg(size_t n) 366 { 367 msan_lwp_t *lwp; 368 uint8_t *arg; 369 370 if (__predict_false(!kmsan_enabled)) 371 return; 372 lwp = curlwp->l_kmsan; 373 arg = lwp->tls[lwp->ctx].param; 374 __builtin_memset(arg, 0, n); 375 } 376 377 void 378 kmsan_init_ret(size_t n) 379 { 380 msan_lwp_t *lwp; 381 uint8_t *arg; 382 383 if (__predict_false(!kmsan_enabled)) 384 return; 385 lwp = curlwp->l_kmsan; 386 arg = lwp->tls[lwp->ctx].retval; 387 __builtin_memset(arg, 0, n); 388 } 389 390 static void 391 kmsan_check_arg(size_t size, const char *hook) 392 { 393 msan_lwp_t *lwp; 394 uint8_t *arg; 395 size_t i; 396 397 if (__predict_false(!kmsan_enabled)) 398 return; 399 lwp = curlwp->l_kmsan; 400 arg = lwp->tls[lwp->ctx].param; 401 402 for (i = 0; i < size; i++) { 403 if (__predict_true(arg[i] == 0)) 404 continue; 405 kmsan_report_hook((const char *)arg + i, size, i, hook); 406 break; 407 } 408 } 409 410 void 411 kmsan_lwp_alloc(struct lwp *l) 412 { 413 msan_lwp_t *lwp; 414 415 kmsan_init_arg(sizeof(size_t) + sizeof(km_flag_t)); 416 lwp = kmem_zalloc(sizeof(msan_lwp_t), KM_SLEEP); 417 lwp->ctx = 1; 418 419 l->l_kmsan = lwp; 420 } 421 422 void 423 kmsan_lwp_free(struct lwp *l) 424 { 425 kmsan_init_arg(sizeof(void *) + sizeof(size_t)); 426 kmem_free(l->l_kmsan, sizeof(msan_lwp_t)); 427 } 428 429 void kmsan_intr_enter(void); 430 void kmsan_intr_leave(void); 431 void kmsan_softint(struct lwp *); 432 433 void 434 kmsan_intr_enter(void) 435 { 436 msan_lwp_t *lwp; 437 438 if (__predict_false(!kmsan_enabled)) 439 return; 440 lwp = curlwp->l_kmsan; 441 442 lwp->ctx++; 443 if (__predict_false(lwp->ctx >= MSAN_NCONTEXT)) { 444 kmsan_enabled = false; 445 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 446 } 447 448 kmsan_init_arg(sizeof(void *)); 449 } 450 451 void 452 kmsan_intr_leave(void) 453 { 454 msan_lwp_t *lwp; 455 456 if (__predict_false(!kmsan_enabled)) 457 return; 458 lwp = curlwp->l_kmsan; 459 460 if (__predict_false(lwp->ctx == 0)) { 461 kmsan_enabled = false; 462 panic("%s: lwp->ctx = %zu", __func__, lwp->ctx); 463 } 464 lwp->ctx--; 465 } 466 467 void 468 kmsan_softint(struct lwp *l) 469 { 470 kmsan_init_arg(sizeof(lwp_t *) + sizeof(int)); 471 } 472 473 /* -------------------------------------------------------------------------- */ 474 475 void 476 kmsan_shadow_map(void *addr, size_t size) 477 { 478 size_t npages, i; 479 vaddr_t va; 480 481 KASSERT((vaddr_t)addr % PAGE_SIZE == 0); 482 KASSERT(size % PAGE_SIZE == 0); 483 484 npages = size / PAGE_SIZE; 485 486 va = (vaddr_t)kmsan_md_addr_to_shad(addr); 487 for (i = 0; i < npages; i++) { 488 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 489 } 490 491 va = (vaddr_t)kmsan_md_addr_to_orig(addr); 492 for (i = 0; i < npages; i++) { 493 kmsan_md_shadow_map_page(va + i * PAGE_SIZE); 494 } 495 } 496 497 void 498 kmsan_orig(void *addr, size_t size, int type, uintptr_t pc) 499 { 500 msan_orig_t orig; 501 502 orig = kmsan_md_orig_encode(type, pc); 503 kmsan_origin_fill(addr, orig, size); 504 } 505 506 void 507 kmsan_mark(void *addr, size_t size, uint8_t c) 508 { 509 kmsan_shadow_fill(addr, c, size); 510 } 511 512 void 513 kmsan_check_mbuf(void *buf) 514 { 515 struct mbuf *m = buf; 516 517 do { 518 kmsan_shadow_check(mtod(m, void *), m->m_len, "if_transmit"); 519 } while ((m = m->m_next) != NULL); 520 } 521 522 void 523 kmsan_check_buf(void *buf) 524 { 525 buf_t *bp = buf; 526 527 kmsan_shadow_check(bp->b_data, bp->b_bcount, "bwrite"); 528 } 529 530 void 531 kmsan_init(void *stack) 532 { 533 /* MD initialization. */ 534 kmsan_md_init(); 535 536 /* Map the stack. */ 537 kmsan_shadow_map(stack, USPACE); 538 539 /* Initialize the TLS for curlwp. */ 540 msan_lwp0.ctx = 1; 541 curlwp->l_kmsan = &msan_lwp0; 542 543 /* Now officially enabled. */ 544 kmsan_enabled = true; 545 } 546 547 /* -------------------------------------------------------------------------- */ 548 549 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 550 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 551 552 msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) 553 { 554 return kmsan_meta_get(addr, size); 555 } 556 557 msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) 558 { 559 return kmsan_meta_get(addr, size); 560 } 561 562 #define MSAN_META_FUNC(size) \ 563 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 564 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 565 { \ 566 return kmsan_meta_get(addr, size); \ 567 } \ 568 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 569 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 570 { \ 571 return kmsan_meta_get(addr, size); \ 572 } 573 574 MSAN_META_FUNC(1) 575 MSAN_META_FUNC(2) 576 MSAN_META_FUNC(4) 577 MSAN_META_FUNC(8) 578 579 void __msan_instrument_asm_store(void *, size_t); 580 msan_orig_t __msan_chain_origin(msan_orig_t); 581 void __msan_poison_alloca(void *, uint64_t, char *); 582 void __msan_unpoison_alloca(void *, uint64_t); 583 void __msan_warning(msan_orig_t); 584 msan_tls_t *__msan_get_context_state(void); 585 586 void __msan_instrument_asm_store(void *addr, size_t size) 587 { 588 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 589 } 590 591 msan_orig_t __msan_chain_origin(msan_orig_t origin) 592 { 593 return origin; 594 } 595 596 void __msan_poison_alloca(void *addr, uint64_t size, char *descr) 597 { 598 msan_orig_t orig; 599 600 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 601 kmsan_origin_fill(addr, orig, size); 602 kmsan_shadow_fill(addr, KMSAN_STATE_UNINIT, size); 603 } 604 605 void __msan_unpoison_alloca(void *addr, uint64_t size) 606 { 607 kmsan_shadow_fill(addr, KMSAN_STATE_INITED, size); 608 } 609 610 void __msan_warning(msan_orig_t origin) 611 { 612 if (__predict_false(!kmsan_enabled)) 613 return; 614 kmsan_report_inline(origin, __RET_ADDR); 615 } 616 617 msan_tls_t *__msan_get_context_state(void) 618 { 619 msan_lwp_t *lwp; 620 621 if (__predict_false(!kmsan_enabled)) 622 return &dummy_tls; 623 lwp = curlwp->l_kmsan; 624 625 return &lwp->tls[lwp->ctx]; 626 } 627 628 /* -------------------------------------------------------------------------- */ 629 630 /* 631 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 632 * initialized areas properly. 633 */ 634 635 void *kmsan_memcpy(void *dst, const void *src, size_t len) 636 { 637 /* No kmsan_check_arg, because inlined. */ 638 kmsan_init_ret(sizeof(void *)); 639 if (__predict_true(len != 0)) { 640 kmsan_meta_copy(dst, src, len); 641 } 642 return __builtin_memcpy(dst, src, len); 643 } 644 645 int 646 kmsan_memcmp(const void *b1, const void *b2, size_t len) 647 { 648 const uint8_t *_b1 = b1, *_b2 = b2; 649 size_t i; 650 651 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), "memcmp"); 652 kmsan_init_ret(sizeof(int)); 653 654 for (i = 0; i < len; i++) { 655 if (*_b1 != *_b2) { 656 kmsan_shadow_check(b1, i + 1, "memcmp"); 657 kmsan_shadow_check(b2, i + 1, "memcmp"); 658 return *_b1 - *_b2; 659 } 660 _b1++, _b2++; 661 } 662 663 return 0; 664 } 665 666 void *kmsan_memset(void *dst, int c, size_t len) 667 { 668 /* No kmsan_check_arg, because inlined. */ 669 kmsan_shadow_fill(dst, KMSAN_STATE_INITED, len); 670 kmsan_init_ret(sizeof(void *)); 671 return __builtin_memset(dst, c, len); 672 } 673 674 void *kmsan_memmove(void *dst, const void *src, size_t len) 675 { 676 /* No kmsan_check_arg, because inlined. */ 677 if (__predict_true(len != 0)) { 678 kmsan_meta_copy(dst, src, len); 679 } 680 kmsan_init_ret(sizeof(void *)); 681 return __builtin_memmove(dst, src, len); 682 } 683 684 __strong_alias(__msan_memcpy, kmsan_memcpy) 685 __strong_alias(__msan_memset, kmsan_memset) 686 __strong_alias(__msan_memmove, kmsan_memmove) 687 688 char * 689 kmsan_strcpy(char *dst, const char *src) 690 { 691 const char *_src = src; 692 char *_dst = dst; 693 size_t len = 0; 694 695 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy"); 696 697 while (1) { 698 len++; 699 *dst = *src; 700 if (*src == '\0') 701 break; 702 src++, dst++; 703 } 704 705 kmsan_shadow_check(_src, len, "strcpy"); 706 kmsan_shadow_fill(_dst, KMSAN_STATE_INITED, len); 707 kmsan_init_ret(sizeof(char *)); 708 return _dst; 709 } 710 711 int 712 kmsan_strcmp(const char *s1, const char *s2) 713 { 714 const char *_s1 = s1, *_s2 = s2; 715 size_t len = 0; 716 717 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp"); 718 kmsan_init_ret(sizeof(int)); 719 720 while (1) { 721 len++; 722 if (*s1 != *s2) 723 break; 724 if (*s1 == '\0') { 725 kmsan_shadow_check(_s1, len, "strcmp"); 726 kmsan_shadow_check(_s2, len, "strcmp"); 727 return 0; 728 } 729 s1++, s2++; 730 } 731 732 kmsan_shadow_check(_s1, len, "strcmp"); 733 kmsan_shadow_check(_s2, len, "strcmp"); 734 735 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 736 } 737 738 size_t 739 kmsan_strlen(const char *str) 740 { 741 const char *s; 742 743 kmsan_check_arg(sizeof(str), "strlen"); 744 745 s = str; 746 while (1) { 747 if (*s == '\0') 748 break; 749 s++; 750 } 751 752 kmsan_shadow_check(str, (size_t)(s - str) + 1, "strlen"); 753 kmsan_init_ret(sizeof(size_t)); 754 return (s - str); 755 } 756 757 #undef kcopy 758 #undef copystr 759 #undef copyin 760 #undef copyout 761 #undef copyinstr 762 #undef copyoutstr 763 764 int kmsan_kcopy(const void *, void *, size_t); 765 int kmsan_copystr(const void *, void *, size_t, size_t *); 766 int kmsan_copyin(const void *, void *, size_t); 767 int kmsan_copyout(const void *, void *, size_t); 768 int kmsan_copyinstr(const void *, void *, size_t, size_t *); 769 int kmsan_copyoutstr(const void *, void *, size_t, size_t *); 770 771 int kcopy(const void *, void *, size_t); 772 int copystr(const void *, void *, size_t, size_t *); 773 int copyin(const void *, void *, size_t); 774 int copyout(const void *, void *, size_t); 775 int copyinstr(const void *, void *, size_t, size_t *); 776 int copyoutstr(const void *, void *, size_t, size_t *); 777 778 int 779 kmsan_kcopy(const void *src, void *dst, size_t len) 780 { 781 kmsan_check_arg(sizeof(src) + sizeof(dst) + sizeof(len), "kcopy"); 782 if (__predict_true(len != 0)) { 783 kmsan_meta_copy(dst, src, len); 784 } 785 kmsan_init_ret(sizeof(int)); 786 return kcopy(src, dst, len); 787 } 788 789 int 790 kmsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 791 { 792 size_t _done; 793 int ret; 794 795 kmsan_check_arg(sizeof(kfaddr) + sizeof(kdaddr) + 796 sizeof(len) + sizeof(done), "copystr"); 797 ret = copystr(kfaddr, kdaddr, len, &_done); 798 if (ret == 0) 799 kmsan_meta_copy(kdaddr, kfaddr, _done); 800 if (done != NULL) { 801 *done = _done; 802 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 803 } 804 kmsan_init_ret(sizeof(int)); 805 806 return ret; 807 } 808 809 int 810 kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 811 { 812 int ret; 813 814 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), "copyin"); 815 ret = copyin(uaddr, kaddr, len); 816 if (ret == 0) 817 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, len); 818 kmsan_init_ret(sizeof(int)); 819 820 return ret; 821 } 822 823 int 824 kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 825 { 826 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), "copyout"); 827 kmsan_shadow_check(kaddr, len, "copyout"); 828 kmsan_init_ret(sizeof(int)); 829 return copyout(kaddr, uaddr, len); 830 } 831 832 int 833 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 834 { 835 size_t _done; 836 int ret; 837 838 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 839 sizeof(len) + sizeof(done), "copyinstr"); 840 ret = copyinstr(uaddr, kaddr, len, &_done); 841 if (ret == 0) 842 kmsan_shadow_fill(kaddr, KMSAN_STATE_INITED, _done); 843 if (done != NULL) { 844 *done = _done; 845 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 846 } 847 kmsan_init_ret(sizeof(int)); 848 849 return ret; 850 } 851 852 int 853 kmsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 854 { 855 size_t _done; 856 int ret; 857 858 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + 859 sizeof(len) + sizeof(done), "copyoutstr"); 860 ret = copyoutstr(kaddr, uaddr, len, &_done); 861 kmsan_shadow_check(kaddr, _done, "copyoutstr"); 862 if (done != NULL) { 863 *done = _done; 864 kmsan_shadow_fill(done, KMSAN_STATE_INITED, sizeof(size_t)); 865 } 866 kmsan_init_ret(sizeof(int)); 867 868 return ret; 869 } 870 871 /* -------------------------------------------------------------------------- */ 872 873 #undef atomic_add_32 874 #undef atomic_add_int 875 #undef atomic_add_long 876 #undef atomic_add_ptr 877 #undef atomic_add_64 878 #undef atomic_add_32_nv 879 #undef atomic_add_int_nv 880 #undef atomic_add_long_nv 881 #undef atomic_add_ptr_nv 882 #undef atomic_add_64_nv 883 #undef atomic_and_32 884 #undef atomic_and_uint 885 #undef atomic_and_ulong 886 #undef atomic_and_64 887 #undef atomic_and_32_nv 888 #undef atomic_and_uint_nv 889 #undef atomic_and_ulong_nv 890 #undef atomic_and_64_nv 891 #undef atomic_or_32 892 #undef atomic_or_uint 893 #undef atomic_or_ulong 894 #undef atomic_or_64 895 #undef atomic_or_32_nv 896 #undef atomic_or_uint_nv 897 #undef atomic_or_ulong_nv 898 #undef atomic_or_64_nv 899 #undef atomic_cas_32 900 #undef atomic_cas_uint 901 #undef atomic_cas_ulong 902 #undef atomic_cas_ptr 903 #undef atomic_cas_64 904 #undef atomic_cas_32_ni 905 #undef atomic_cas_uint_ni 906 #undef atomic_cas_ulong_ni 907 #undef atomic_cas_ptr_ni 908 #undef atomic_cas_64_ni 909 #undef atomic_swap_32 910 #undef atomic_swap_uint 911 #undef atomic_swap_ulong 912 #undef atomic_swap_ptr 913 #undef atomic_swap_64 914 #undef atomic_dec_32 915 #undef atomic_dec_uint 916 #undef atomic_dec_ulong 917 #undef atomic_dec_ptr 918 #undef atomic_dec_64 919 #undef atomic_dec_32_nv 920 #undef atomic_dec_uint_nv 921 #undef atomic_dec_ulong_nv 922 #undef atomic_dec_ptr_nv 923 #undef atomic_dec_64_nv 924 #undef atomic_inc_32 925 #undef atomic_inc_uint 926 #undef atomic_inc_ulong 927 #undef atomic_inc_ptr 928 #undef atomic_inc_64 929 #undef atomic_inc_32_nv 930 #undef atomic_inc_uint_nv 931 #undef atomic_inc_ulong_nv 932 #undef atomic_inc_ptr_nv 933 #undef atomic_inc_64_nv 934 935 #define MSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 936 void atomic_add_##name(volatile targ1 *, targ2); \ 937 void kmsan_atomic_add_##name(volatile targ1 *, targ2); \ 938 void kmsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 939 { \ 940 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 941 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 942 atomic_add_##name(ptr, val); \ 943 } \ 944 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 945 tret kmsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 946 tret kmsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 947 { \ 948 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 949 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 950 kmsan_init_ret(sizeof(tret)); \ 951 return atomic_add_##name##_nv(ptr, val); \ 952 } 953 #define MSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 954 void atomic_and_##name(volatile targ1 *, targ2); \ 955 void kmsan_atomic_and_##name(volatile targ1 *, targ2); \ 956 void kmsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 957 { \ 958 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 959 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 960 atomic_and_##name(ptr, val); \ 961 } \ 962 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 963 tret kmsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 964 tret kmsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 965 { \ 966 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 967 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 968 kmsan_init_ret(sizeof(tret)); \ 969 return atomic_and_##name##_nv(ptr, val); \ 970 } 971 972 #define MSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 973 void atomic_or_##name(volatile targ1 *, targ2); \ 974 void kmsan_atomic_or_##name(volatile targ1 *, targ2); \ 975 void kmsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 976 { \ 977 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 978 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 979 atomic_or_##name(ptr, val); \ 980 } \ 981 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 982 tret kmsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 983 tret kmsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 984 { \ 985 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 986 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 987 kmsan_init_ret(sizeof(tret)); \ 988 return atomic_or_##name##_nv(ptr, val); \ 989 } 990 991 #define MSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 992 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 993 tret kmsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 994 tret kmsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 995 { \ 996 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 997 __func__); \ 998 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 999 kmsan_init_ret(sizeof(tret)); \ 1000 return atomic_cas_##name(ptr, exp, new); \ 1001 } \ 1002 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1003 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 1004 tret kmsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 1005 { \ 1006 kmsan_check_arg(sizeof(ptr) + sizeof(exp) + sizeof(new), \ 1007 __func__); \ 1008 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1009 kmsan_init_ret(sizeof(tret)); \ 1010 return atomic_cas_##name##_ni(ptr, exp, new); \ 1011 } 1012 1013 #define MSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 1014 tret atomic_swap_##name(volatile targ1 *, targ2); \ 1015 tret kmsan_atomic_swap_##name(volatile targ1 *, targ2); \ 1016 tret kmsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 1017 { \ 1018 kmsan_check_arg(sizeof(ptr) + sizeof(val), __func__); \ 1019 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1020 kmsan_init_ret(sizeof(tret)); \ 1021 return atomic_swap_##name(ptr, val); \ 1022 } 1023 1024 #define MSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 1025 void atomic_dec_##name(volatile targ1 *); \ 1026 void kmsan_atomic_dec_##name(volatile targ1 *); \ 1027 void kmsan_atomic_dec_##name(volatile targ1 *ptr) \ 1028 { \ 1029 kmsan_check_arg(sizeof(ptr), __func__); \ 1030 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1031 atomic_dec_##name(ptr); \ 1032 } \ 1033 tret atomic_dec_##name##_nv(volatile targ1 *); \ 1034 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *); \ 1035 tret kmsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 1036 { \ 1037 kmsan_check_arg(sizeof(ptr), __func__); \ 1038 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1039 kmsan_init_ret(sizeof(tret)); \ 1040 return atomic_dec_##name##_nv(ptr); \ 1041 } 1042 1043 #define MSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 1044 void atomic_inc_##name(volatile targ1 *); \ 1045 void kmsan_atomic_inc_##name(volatile targ1 *); \ 1046 void kmsan_atomic_inc_##name(volatile targ1 *ptr) \ 1047 { \ 1048 kmsan_check_arg(sizeof(ptr), __func__); \ 1049 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1050 atomic_inc_##name(ptr); \ 1051 } \ 1052 tret atomic_inc_##name##_nv(volatile targ1 *); \ 1053 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *); \ 1054 tret kmsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 1055 { \ 1056 kmsan_check_arg(sizeof(ptr), __func__); \ 1057 kmsan_shadow_check((uintptr_t)ptr, sizeof(tret), __func__); \ 1058 kmsan_init_ret(sizeof(tret)); \ 1059 return atomic_inc_##name##_nv(ptr); \ 1060 } 1061 1062 MSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 1063 MSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 1064 MSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 1065 MSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 1066 MSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 1067 1068 MSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 1069 MSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 1070 MSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 1071 MSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 1072 1073 MSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 1074 MSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 1075 MSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 1076 MSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 1077 1078 MSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 1079 MSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 1080 MSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 1081 MSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 1082 MSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 1083 1084 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 1085 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 1086 MSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 1087 MSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 1088 MSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 1089 1090 MSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 1091 MSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 1092 MSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 1093 MSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 1094 MSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 1095 1096 MSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 1097 MSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 1098 MSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 1099 MSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 1100 MSAN_ATOMIC_FUNC_INC(ptr, void *, void); 1101 1102 /* -------------------------------------------------------------------------- */ 1103 1104 #include <sys/bus.h> 1105 1106 #undef bus_space_read_multi_1 1107 #undef bus_space_read_multi_2 1108 #undef bus_space_read_multi_4 1109 #undef bus_space_read_multi_8 1110 #undef bus_space_read_multi_stream_1 1111 #undef bus_space_read_multi_stream_2 1112 #undef bus_space_read_multi_stream_4 1113 #undef bus_space_read_multi_stream_8 1114 #undef bus_space_read_region_1 1115 #undef bus_space_read_region_2 1116 #undef bus_space_read_region_4 1117 #undef bus_space_read_region_8 1118 #undef bus_space_read_region_stream_1 1119 #undef bus_space_read_region_stream_2 1120 #undef bus_space_read_region_stream_4 1121 #undef bus_space_read_region_stream_8 1122 1123 #define MSAN_BUS_READ_FUNC(bytes, bits) \ 1124 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1125 bus_size_t, uint##bits##_t *, bus_size_t); \ 1126 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 1127 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1128 void kmsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 1129 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1130 bus_size_t count) \ 1131 { \ 1132 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1133 sizeof(uint##bits##_t) * count); \ 1134 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 1135 } \ 1136 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1137 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1138 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 1139 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1140 void kmsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 1141 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1142 bus_size_t count) \ 1143 { \ 1144 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1145 sizeof(uint##bits##_t) * count); \ 1146 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1147 } \ 1148 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1149 bus_size_t, uint##bits##_t *, bus_size_t); \ 1150 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 1151 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1152 void kmsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 1153 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1154 bus_size_t count) \ 1155 { \ 1156 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1157 sizeof(uint##bits##_t) * count); \ 1158 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 1159 } \ 1160 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1161 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1162 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 1163 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 1164 void kmsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1165 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1166 bus_size_t count) \ 1167 { \ 1168 kmsan_shadow_fill(buf, KMSAN_STATE_INITED, \ 1169 sizeof(uint##bits##_t) * count); \ 1170 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1171 } 1172 1173 MSAN_BUS_READ_FUNC(1, 8) 1174 MSAN_BUS_READ_FUNC(2, 16) 1175 MSAN_BUS_READ_FUNC(4, 32) 1176 MSAN_BUS_READ_FUNC(8, 64) 1177 1178 #undef bus_space_write_multi_1 1179 #undef bus_space_write_multi_2 1180 #undef bus_space_write_multi_4 1181 #undef bus_space_write_multi_8 1182 #undef bus_space_write_multi_stream_1 1183 #undef bus_space_write_multi_stream_2 1184 #undef bus_space_write_multi_stream_4 1185 #undef bus_space_write_multi_stream_8 1186 #undef bus_space_write_region_1 1187 #undef bus_space_write_region_2 1188 #undef bus_space_write_region_4 1189 #undef bus_space_write_region_8 1190 #undef bus_space_write_region_stream_1 1191 #undef bus_space_write_region_stream_2 1192 #undef bus_space_write_region_stream_4 1193 #undef bus_space_write_region_stream_8 1194 1195 #define MSAN_BUS_WRITE_FUNC(bytes, bits) \ 1196 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1197 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1198 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1199 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1200 void kmsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1201 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1202 bus_size_t count) \ 1203 { \ 1204 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1205 "bus_space_write"); \ 1206 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1207 } \ 1208 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1209 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1210 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1211 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1212 void kmsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1213 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1214 bus_size_t count) \ 1215 { \ 1216 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1217 "bus_space_write"); \ 1218 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1219 } \ 1220 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1221 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1222 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1223 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1224 void kmsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1225 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1226 bus_size_t count) \ 1227 { \ 1228 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1229 "bus_space_write"); \ 1230 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1231 } \ 1232 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1233 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1234 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1235 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1236 void kmsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1237 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1238 bus_size_t count) \ 1239 { \ 1240 kmsan_shadow_check(buf, sizeof(uint##bits##_t) * count, \ 1241 "bus_space_write"); \ 1242 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1243 } 1244 1245 MSAN_BUS_WRITE_FUNC(1, 8) 1246 MSAN_BUS_WRITE_FUNC(2, 16) 1247 MSAN_BUS_WRITE_FUNC(4, 32) 1248 MSAN_BUS_WRITE_FUNC(8, 64) 1249 1250 /* -------------------------------------------------------------------------- */ 1251 1252 #include <sys/mbuf.h> 1253 1254 static void 1255 kmsan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1256 bool init, uintptr_t pc) 1257 { 1258 if (init) { 1259 kmsan_shadow_fill(buf + offset, KMSAN_STATE_INITED, len); 1260 } else { 1261 kmsan_shadow_check(buf + offset, len, "dma_sync_linear"); 1262 } 1263 } 1264 1265 static void 1266 kmsan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1267 bool init, uintptr_t pc) 1268 { 1269 bus_addr_t minlen; 1270 1271 for (; m != NULL && len != 0; m = m->m_next) { 1272 if (offset >= m->m_len) { 1273 offset -= m->m_len; 1274 continue; 1275 } 1276 1277 minlen = MIN(len, m->m_len - offset); 1278 1279 if (init) { 1280 kmsan_shadow_fill(mtod(m, char *) + offset, 1281 KMSAN_STATE_INITED, minlen); 1282 } else { 1283 kmsan_shadow_check(mtod(m, char *) + offset, 1284 minlen, "dma_sync_mbuf"); 1285 } 1286 1287 offset = 0; 1288 len -= minlen; 1289 } 1290 } 1291 1292 static void 1293 kmsan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1294 bool init, uintptr_t pc) 1295 { 1296 bus_size_t minlen, resid; 1297 struct iovec *iov; 1298 int i; 1299 1300 if (uio->uio_vmspace != NULL) 1301 return; 1302 1303 resid = uio->uio_resid; 1304 iov = uio->uio_iov; 1305 1306 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1307 minlen = MIN(resid, iov[i].iov_len); 1308 1309 if (init) { 1310 kmsan_shadow_fill(iov[i].iov_base, 1311 KMSAN_STATE_INITED, minlen); 1312 } else { 1313 kmsan_shadow_check(iov[i].iov_base, minlen, 1314 "dma_sync_uio"); 1315 } 1316 1317 resid -= minlen; 1318 } 1319 } 1320 1321 void 1322 kmsan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1323 { 1324 bool init; 1325 1326 if ((ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTREAD)) == 0) 1327 return; 1328 init = (ops & BUS_DMASYNC_POSTREAD) != 0; 1329 1330 switch (map->dm_buftype) { 1331 case KMSAN_DMA_LINEAR: 1332 kmsan_dma_sync_linear(map->dm_buf, offset, len, init, 1333 __RET_ADDR); 1334 break; 1335 case KMSAN_DMA_MBUF: 1336 kmsan_dma_sync_mbuf(map->dm_buf, offset, len, init, 1337 __RET_ADDR); 1338 break; 1339 case KMSAN_DMA_UIO: 1340 kmsan_dma_sync_uio(map->dm_buf, offset, len, init, 1341 __RET_ADDR); 1342 break; 1343 case KMSAN_DMA_RAW: 1344 break; 1345 default: 1346 panic("%s: impossible", __func__); 1347 } 1348 } 1349 1350 void 1351 kmsan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1352 { 1353 map->dm_buf = buf; 1354 map->dm_buflen = buflen; 1355 map->dm_buftype = type; 1356 } 1357
Indexes created Mon Sep 22 13:09:51 GMT 2025