1 /* $NetBSD: subr_asan.c,v 1.28 2023/04/09 09:18:09 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KASAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.28 2023/04/09 09:18:09 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/asan.h> 42 43 #include <uvm/uvm_extern.h> 44 45 #ifdef DDB 46 #include <machine/db_machdep.h> 47 #include <ddb/db_extern.h> 48 #endif 49 50 #ifdef KASAN_PANIC 51 #define REPORT panic 52 #else 53 #define REPORT printf 54 #endif 55 56 /* ASAN constants. Part of the compiler ABI. */ 57 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 58 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1) 59 #define KASAN_ALLOCA_SCALE_SIZE 32 60 61 /* The MD code. */ 62 #include <machine/asan.h> 63 64 /* ASAN ABI version. */ 65 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 66 #define ASAN_ABI_VERSION 8 67 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 68 #define ASAN_ABI_VERSION 8 69 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 70 #define ASAN_ABI_VERSION 6 71 #else 72 #error "Unsupported compiler version" 73 #endif 74 75 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 76 77 /* Global variable descriptor. Part of the compiler ABI. */ 78 struct __asan_global_source_location { 79 const char *filename; 80 int line_no; 81 int column_no; 82 }; 83 struct __asan_global { 84 const void *beg; /* address of the global variable */ 85 size_t size; /* size of the global variable */ 86 size_t size_with_redzone; /* size with the redzone */ 87 const void *name; /* name of the variable */ 88 const void *module_name; /* name of the module where the var is declared */ 89 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 90 struct __asan_global_source_location *location; 91 #if ASAN_ABI_VERSION >= 7 92 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 93 #endif 94 }; 95 96 static bool kasan_enabled __read_mostly = false; 97 98 /* -------------------------------------------------------------------------- */ 99 100 void 101 kasan_shadow_map(void *addr, size_t size) 102 { 103 size_t sz, npages, i; 104 vaddr_t sva, eva; 105 106 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 107 108 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE; 109 110 sva = (vaddr_t)kasan_md_addr_to_shad(addr); 111 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz; 112 113 sva = rounddown(sva, PAGE_SIZE); 114 eva = roundup(eva, PAGE_SIZE); 115 116 npages = (eva - sva) / PAGE_SIZE; 117 118 KASSERT(sva >= KASAN_MD_SHADOW_START); 119 KASSERT(eva < KASAN_MD_SHADOW_END); 120 121 for (i = 0; i < npages; i++) { 122 kasan_md_shadow_map_page(sva + i * PAGE_SIZE); 123 } 124 } 125 126 static void 127 kasan_ctors(void) 128 { 129 extern Elf_Addr __CTOR_LIST__, __CTOR_END__; 130 size_t nentries, i; 131 Elf_Addr *ptr; 132 133 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) / 134 sizeof(uintptr_t); 135 136 ptr = &__CTOR_LIST__; 137 for (i = 0; i < nentries; i++) { 138 void (*func)(void); 139 140 func = (void *)(*ptr); 141 (*func)(); 142 143 ptr++; 144 } 145 } 146 147 void 148 kasan_early_init(void *stack) 149 { 150 kasan_md_early_init(stack); 151 } 152 153 void 154 kasan_init(void) 155 { 156 /* MD initialization. */ 157 kasan_md_init(); 158 159 /* Now officially enabled. */ 160 kasan_enabled = true; 161 162 /* Call the ASAN constructors. */ 163 kasan_ctors(); 164 } 165 166 static inline const char * 167 kasan_code_name(uint8_t code) 168 { 169 switch (code) { 170 case KASAN_GENERIC_REDZONE: 171 return "GenericRedZone"; 172 case KASAN_MALLOC_REDZONE: 173 return "MallocRedZone"; 174 case KASAN_KMEM_REDZONE: 175 return "KmemRedZone"; 176 case KASAN_POOL_REDZONE: 177 return "PoolRedZone"; 178 case KASAN_POOL_FREED: 179 return "PoolUseAfterFree"; 180 case 1 ... 7: 181 return "RedZonePartial"; 182 case KASAN_STACK_LEFT: 183 return "StackLeft"; 184 case KASAN_STACK_MID: 185 return "StackMiddle"; 186 case KASAN_STACK_RIGHT: 187 return "StackRight"; 188 case KASAN_USE_AFTER_RET: 189 return "UseAfterRet"; 190 case KASAN_USE_AFTER_SCOPE: 191 return "UseAfterScope"; 192 default: 193 return "Unknown"; 194 } 195 } 196 197 static void 198 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 199 uint8_t code) 200 { 201 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s," 202 " %s]\n", 203 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""), 204 (write ? "write" : "read"), kasan_code_name(code)); 205 kasan_md_unwind(); 206 } 207 208 static __always_inline void 209 kasan_shadow_1byte_markvalid(unsigned long addr) 210 { 211 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 212 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 213 214 *byte = last; 215 } 216 217 static __always_inline void 218 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 219 { 220 size_t i; 221 222 for (i = 0; i < size; i++) { 223 kasan_shadow_1byte_markvalid((unsigned long)addr+i); 224 } 225 } 226 227 static __always_inline void 228 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 229 { 230 void *shad; 231 232 if (__predict_false(size == 0)) 233 return; 234 if (__predict_false(kasan_md_unsupported((vaddr_t)addr))) 235 return; 236 237 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 238 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0); 239 240 shad = (void *)kasan_md_addr_to_shad(addr); 241 size = size >> KASAN_SHADOW_SCALE_SHIFT; 242 243 __builtin_memset(shad, code, size); 244 } 245 246 void 247 kasan_add_redzone(size_t *size) 248 { 249 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE); 250 *size += KASAN_SHADOW_SCALE_SIZE; 251 } 252 253 void 254 kasan_softint(struct lwp *l) 255 { 256 const void *stk = (const void *)uvm_lwp_getuarea(l); 257 258 kasan_shadow_Nbyte_fill(stk, USPACE, 0); 259 } 260 261 /* 262 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 263 * and the rest as invalid. There are generally two use cases: 264 * 265 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 266 * the redzone at the end of the buffer as invalid. 267 * 268 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 269 */ 270 void 271 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code) 272 { 273 size_t i, n, redz; 274 int8_t *shad; 275 276 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 277 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE); 278 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0); 279 shad = kasan_md_addr_to_shad(addr); 280 281 /* Chunks of 8 bytes, valid. */ 282 n = size / KASAN_SHADOW_SCALE_SIZE; 283 for (i = 0; i < n; i++) { 284 *shad++ = 0; 285 } 286 287 /* Possibly one chunk, mid. */ 288 if ((size & KASAN_SHADOW_MASK) != 0) { 289 *shad++ = (size & KASAN_SHADOW_MASK); 290 } 291 292 /* Chunks of 8 bytes, invalid. */ 293 n = redz / KASAN_SHADOW_SCALE_SIZE; 294 for (i = 0; i < n; i++) { 295 *shad++ = code; 296 } 297 } 298 299 /* -------------------------------------------------------------------------- */ 300 301 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 302 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 303 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 304 305 static __always_inline bool 306 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 307 { 308 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 309 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 310 311 if (__predict_true(*byte == 0 || last <= *byte)) { 312 return true; 313 } 314 *code = *byte; 315 return false; 316 } 317 318 static __always_inline bool 319 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 320 { 321 int8_t *byte, last; 322 323 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 324 return (kasan_shadow_1byte_isvalid(addr, code) && 325 kasan_shadow_1byte_isvalid(addr+1, code)); 326 } 327 328 byte = kasan_md_addr_to_shad((void *)addr); 329 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 330 331 if (__predict_true(*byte == 0 || last <= *byte)) { 332 return true; 333 } 334 *code = *byte; 335 return false; 336 } 337 338 static __always_inline bool 339 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 340 { 341 int8_t *byte, last; 342 343 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 344 return (kasan_shadow_2byte_isvalid(addr, code) && 345 kasan_shadow_2byte_isvalid(addr+2, code)); 346 } 347 348 byte = kasan_md_addr_to_shad((void *)addr); 349 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 350 351 if (__predict_true(*byte == 0 || last <= *byte)) { 352 return true; 353 } 354 *code = *byte; 355 return false; 356 } 357 358 static __always_inline bool 359 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 360 { 361 int8_t *byte, last; 362 363 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 364 return (kasan_shadow_4byte_isvalid(addr, code) && 365 kasan_shadow_4byte_isvalid(addr+4, code)); 366 } 367 368 byte = kasan_md_addr_to_shad((void *)addr); 369 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 370 371 if (__predict_true(*byte == 0 || last <= *byte)) { 372 return true; 373 } 374 *code = *byte; 375 return false; 376 } 377 378 static __always_inline bool 379 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 380 { 381 size_t i; 382 383 for (i = 0; i < size; i++) { 384 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 385 return false; 386 } 387 388 return true; 389 } 390 391 static __always_inline void 392 kasan_shadow_check(unsigned long addr, size_t size, bool write, 393 unsigned long retaddr) 394 { 395 uint8_t code; 396 bool valid; 397 398 if (__predict_false(!kasan_enabled)) 399 return; 400 #ifdef DDB 401 if (__predict_false(db_recover != NULL)) 402 return; 403 #endif 404 if (__predict_false(size == 0)) 405 return; 406 if (__predict_false(kasan_md_unsupported(addr))) 407 return; 408 409 if (__builtin_constant_p(size)) { 410 switch (size) { 411 case 1: 412 valid = kasan_shadow_1byte_isvalid(addr, &code); 413 break; 414 case 2: 415 valid = kasan_shadow_2byte_isvalid(addr, &code); 416 break; 417 case 4: 418 valid = kasan_shadow_4byte_isvalid(addr, &code); 419 break; 420 case 8: 421 valid = kasan_shadow_8byte_isvalid(addr, &code); 422 break; 423 default: 424 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 425 break; 426 } 427 } else { 428 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 429 } 430 431 if (__predict_false(!valid)) { 432 kasan_report(addr, size, write, retaddr, code); 433 } 434 } 435 436 /* -------------------------------------------------------------------------- */ 437 438 void * 439 kasan_memcpy(void *dst, const void *src, size_t len) 440 { 441 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 442 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 443 return __builtin_memcpy(dst, src, len); 444 } 445 446 int 447 kasan_memcmp(const void *b1, const void *b2, size_t len) 448 { 449 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 450 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 451 return __builtin_memcmp(b1, b2, len); 452 } 453 454 void * 455 kasan_memset(void *b, int c, size_t len) 456 { 457 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 458 return __builtin_memset(b, c, len); 459 } 460 461 void * 462 kasan_memmove(void *dst, const void *src, size_t len) 463 { 464 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 465 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 466 return __builtin_memmove(dst, src, len); 467 } 468 469 char * 470 kasan_strcpy(char *dst, const char *src) 471 { 472 char *save = dst; 473 474 while (1) { 475 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 476 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 477 *dst = *src; 478 if (*src == '\0') 479 break; 480 src++, dst++; 481 } 482 483 return save; 484 } 485 486 int 487 kasan_strcmp(const char *s1, const char *s2) 488 { 489 while (1) { 490 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 491 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 492 if (*s1 != *s2) 493 break; 494 if (*s1 == '\0') 495 return 0; 496 s1++, s2++; 497 } 498 499 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 500 } 501 502 size_t 503 kasan_strlen(const char *str) 504 { 505 const char *s; 506 507 s = str; 508 while (1) { 509 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 510 if (*s == '\0') 511 break; 512 s++; 513 } 514 515 return (s - str); 516 } 517 518 char * 519 kasan_strcat(char *dst, const char *src) 520 { 521 size_t ldst, lsrc; 522 523 ldst = __builtin_strlen(dst); 524 lsrc = __builtin_strlen(src); 525 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true, 526 __RET_ADDR); 527 kasan_shadow_check((unsigned long)src, lsrc + 1, false, 528 __RET_ADDR); 529 530 return __builtin_strcat(dst, src); 531 } 532 533 char * 534 kasan_strchr(const char *s, int c) 535 { 536 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 537 __RET_ADDR); 538 return __builtin_strchr(s, c); 539 } 540 541 char * 542 kasan_strrchr(const char *s, int c) 543 { 544 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 545 __RET_ADDR); 546 return __builtin_strrchr(s, c); 547 } 548 549 #undef kcopy 550 #undef copyinstr 551 #undef copyoutstr 552 #undef copyin 553 554 int kasan_kcopy(const void *, void *, size_t); 555 int kasan_copyinstr(const void *, void *, size_t, size_t *); 556 int kasan_copyoutstr(const void *, void *, size_t, size_t *); 557 int kasan_copyin(const void *, void *, size_t); 558 int kcopy(const void *, void *, size_t); 559 int copyinstr(const void *, void *, size_t, size_t *); 560 int copyoutstr(const void *, void *, size_t, size_t *); 561 int copyin(const void *, void *, size_t); 562 563 int 564 kasan_kcopy(const void *src, void *dst, size_t len) 565 { 566 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 567 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 568 return kcopy(src, dst, len); 569 } 570 571 int 572 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 573 { 574 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 575 return copyin(uaddr, kaddr, len); 576 } 577 578 int 579 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 580 { 581 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 582 return copyinstr(uaddr, kaddr, len, done); 583 } 584 585 int 586 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 587 { 588 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 589 return copyoutstr(kaddr, uaddr, len, done); 590 } 591 592 /* -------------------------------------------------------------------------- */ 593 594 #undef _ucas_32 595 #undef _ucas_32_mp 596 #undef _ucas_64 597 #undef _ucas_64_mp 598 #undef _ufetch_8 599 #undef _ufetch_16 600 #undef _ufetch_32 601 #undef _ufetch_64 602 603 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 604 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 605 int 606 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 607 uint32_t *ret) 608 { 609 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 610 __RET_ADDR); 611 return _ucas_32(uaddr, old, new, ret); 612 } 613 614 #ifdef __HAVE_UCAS_MP 615 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 616 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 617 int 618 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 619 uint32_t *ret) 620 { 621 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 622 __RET_ADDR); 623 return _ucas_32_mp(uaddr, old, new, ret); 624 } 625 #endif 626 627 #ifdef _LP64 628 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 629 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 630 int 631 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 632 uint64_t *ret) 633 { 634 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 635 __RET_ADDR); 636 return _ucas_64(uaddr, old, new, ret); 637 } 638 639 #ifdef __HAVE_UCAS_MP 640 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 641 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 642 int 643 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 644 uint64_t *ret) 645 { 646 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 647 __RET_ADDR); 648 return _ucas_64_mp(uaddr, old, new, ret); 649 } 650 #endif 651 #endif 652 653 int _ufetch_8(const uint8_t *, uint8_t *); 654 int kasan__ufetch_8(const uint8_t *, uint8_t *); 655 int 656 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 657 { 658 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 659 __RET_ADDR); 660 return _ufetch_8(uaddr, valp); 661 } 662 663 int _ufetch_16(const uint16_t *, uint16_t *); 664 int kasan__ufetch_16(const uint16_t *, uint16_t *); 665 int 666 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 667 { 668 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 669 __RET_ADDR); 670 return _ufetch_16(uaddr, valp); 671 } 672 673 int _ufetch_32(const uint32_t *, uint32_t *); 674 int kasan__ufetch_32(const uint32_t *, uint32_t *); 675 int 676 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 677 { 678 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 679 __RET_ADDR); 680 return _ufetch_32(uaddr, valp); 681 } 682 683 #ifdef _LP64 684 int _ufetch_64(const uint64_t *, uint64_t *); 685 int kasan__ufetch_64(const uint64_t *, uint64_t *); 686 int 687 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 688 { 689 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 690 __RET_ADDR); 691 return _ufetch_64(uaddr, valp); 692 } 693 #endif 694 695 /* -------------------------------------------------------------------------- */ 696 697 #undef atomic_add_32 698 #undef atomic_add_int 699 #undef atomic_add_long 700 #undef atomic_add_ptr 701 #undef atomic_add_64 702 #undef atomic_add_32_nv 703 #undef atomic_add_int_nv 704 #undef atomic_add_long_nv 705 #undef atomic_add_ptr_nv 706 #undef atomic_add_64_nv 707 #undef atomic_and_32 708 #undef atomic_and_uint 709 #undef atomic_and_ulong 710 #undef atomic_and_64 711 #undef atomic_and_32_nv 712 #undef atomic_and_uint_nv 713 #undef atomic_and_ulong_nv 714 #undef atomic_and_64_nv 715 #undef atomic_or_32 716 #undef atomic_or_uint 717 #undef atomic_or_ulong 718 #undef atomic_or_64 719 #undef atomic_or_32_nv 720 #undef atomic_or_uint_nv 721 #undef atomic_or_ulong_nv 722 #undef atomic_or_64_nv 723 #undef atomic_cas_32 724 #undef atomic_cas_uint 725 #undef atomic_cas_ulong 726 #undef atomic_cas_ptr 727 #undef atomic_cas_64 728 #undef atomic_cas_32_ni 729 #undef atomic_cas_uint_ni 730 #undef atomic_cas_ulong_ni 731 #undef atomic_cas_ptr_ni 732 #undef atomic_cas_64_ni 733 #undef atomic_swap_32 734 #undef atomic_swap_uint 735 #undef atomic_swap_ulong 736 #undef atomic_swap_ptr 737 #undef atomic_swap_64 738 #undef atomic_dec_32 739 #undef atomic_dec_uint 740 #undef atomic_dec_ulong 741 #undef atomic_dec_ptr 742 #undef atomic_dec_64 743 #undef atomic_dec_32_nv 744 #undef atomic_dec_uint_nv 745 #undef atomic_dec_ulong_nv 746 #undef atomic_dec_ptr_nv 747 #undef atomic_dec_64_nv 748 #undef atomic_inc_32 749 #undef atomic_inc_uint 750 #undef atomic_inc_ulong 751 #undef atomic_inc_ptr 752 #undef atomic_inc_64 753 #undef atomic_inc_32_nv 754 #undef atomic_inc_uint_nv 755 #undef atomic_inc_ulong_nv 756 #undef atomic_inc_ptr_nv 757 #undef atomic_inc_64_nv 758 759 #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 760 void atomic_add_##name(volatile targ1 *, targ2); \ 761 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 762 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 763 { \ 764 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 765 __RET_ADDR); \ 766 atomic_add_##name(ptr, val); \ 767 } \ 768 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 769 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 770 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 771 { \ 772 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 773 __RET_ADDR); \ 774 return atomic_add_##name##_nv(ptr, val); \ 775 } 776 777 #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 778 void atomic_and_##name(volatile targ1 *, targ2); \ 779 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 780 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 781 { \ 782 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 783 __RET_ADDR); \ 784 atomic_and_##name(ptr, val); \ 785 } \ 786 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 787 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 788 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 789 { \ 790 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 791 __RET_ADDR); \ 792 return atomic_and_##name##_nv(ptr, val); \ 793 } 794 795 #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 796 void atomic_or_##name(volatile targ1 *, targ2); \ 797 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 798 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 799 { \ 800 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 801 __RET_ADDR); \ 802 atomic_or_##name(ptr, val); \ 803 } \ 804 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 805 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 806 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 807 { \ 808 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 809 __RET_ADDR); \ 810 return atomic_or_##name##_nv(ptr, val); \ 811 } 812 813 #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 814 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 815 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 816 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 817 { \ 818 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 819 __RET_ADDR); \ 820 return atomic_cas_##name(ptr, exp, new); \ 821 } \ 822 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 823 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 824 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 825 { \ 826 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 827 __RET_ADDR); \ 828 return atomic_cas_##name##_ni(ptr, exp, new); \ 829 } 830 831 #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 832 tret atomic_swap_##name(volatile targ1 *, targ2); \ 833 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \ 834 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 835 { \ 836 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 837 __RET_ADDR); \ 838 return atomic_swap_##name(ptr, val); \ 839 } 840 841 #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 842 void atomic_dec_##name(volatile targ1 *); \ 843 void kasan_atomic_dec_##name(volatile targ1 *); \ 844 void kasan_atomic_dec_##name(volatile targ1 *ptr) \ 845 { \ 846 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 847 __RET_ADDR); \ 848 atomic_dec_##name(ptr); \ 849 } \ 850 tret atomic_dec_##name##_nv(volatile targ1 *); \ 851 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \ 852 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 853 { \ 854 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 855 __RET_ADDR); \ 856 return atomic_dec_##name##_nv(ptr); \ 857 } 858 859 #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 860 void atomic_inc_##name(volatile targ1 *); \ 861 void kasan_atomic_inc_##name(volatile targ1 *); \ 862 void kasan_atomic_inc_##name(volatile targ1 *ptr) \ 863 { \ 864 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 865 __RET_ADDR); \ 866 atomic_inc_##name(ptr); \ 867 } \ 868 tret atomic_inc_##name##_nv(volatile targ1 *); \ 869 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \ 870 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 871 { \ 872 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 873 __RET_ADDR); \ 874 return atomic_inc_##name##_nv(ptr); \ 875 } 876 877 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 878 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 879 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 880 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 881 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 882 883 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 884 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 885 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 886 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 887 888 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 889 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 890 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 891 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 892 893 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 894 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 895 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 896 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 897 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 898 899 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 900 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 901 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 902 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 903 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 904 905 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 906 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 907 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 908 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 909 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void); 910 911 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 912 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 913 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 914 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 915 ASAN_ATOMIC_FUNC_INC(ptr, void *, void); 916 917 /* -------------------------------------------------------------------------- */ 918 919 #ifdef __HAVE_KASAN_INSTR_BUS 920 921 #include <sys/bus.h> 922 923 #undef bus_space_read_multi_1 924 #undef bus_space_read_multi_2 925 #undef bus_space_read_multi_4 926 #undef bus_space_read_multi_8 927 #undef bus_space_read_multi_stream_1 928 #undef bus_space_read_multi_stream_2 929 #undef bus_space_read_multi_stream_4 930 #undef bus_space_read_multi_stream_8 931 #undef bus_space_read_region_1 932 #undef bus_space_read_region_2 933 #undef bus_space_read_region_4 934 #undef bus_space_read_region_8 935 #undef bus_space_read_region_stream_1 936 #undef bus_space_read_region_stream_2 937 #undef bus_space_read_region_stream_4 938 #undef bus_space_read_region_stream_8 939 #undef bus_space_write_multi_1 940 #undef bus_space_write_multi_2 941 #undef bus_space_write_multi_4 942 #undef bus_space_write_multi_8 943 #undef bus_space_write_multi_stream_1 944 #undef bus_space_write_multi_stream_2 945 #undef bus_space_write_multi_stream_4 946 #undef bus_space_write_multi_stream_8 947 #undef bus_space_write_region_1 948 #undef bus_space_write_region_2 949 #undef bus_space_write_region_4 950 #undef bus_space_write_region_8 951 #undef bus_space_write_region_stream_1 952 #undef bus_space_write_region_stream_2 953 #undef bus_space_write_region_stream_4 954 #undef bus_space_write_region_stream_8 955 956 #define ASAN_BUS_READ_FUNC(bytes, bits) \ 957 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 958 bus_size_t, uint##bits##_t *, bus_size_t); \ 959 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 960 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 961 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 962 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 963 bus_size_t count) \ 964 { \ 965 kasan_shadow_check((uintptr_t)buf, \ 966 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 967 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 968 } \ 969 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 970 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 971 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 972 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 973 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 974 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 975 bus_size_t count) \ 976 { \ 977 kasan_shadow_check((uintptr_t)buf, \ 978 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 979 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 980 } \ 981 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 982 bus_size_t, uint##bits##_t *, bus_size_t); \ 983 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \ 984 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 985 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 986 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 987 bus_size_t count) \ 988 { \ 989 kasan_shadow_check((uintptr_t)buf, \ 990 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 991 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 992 } \ 993 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 994 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 995 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 996 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 997 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 998 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 999 bus_size_t count) \ 1000 { \ 1001 kasan_shadow_check((uintptr_t)buf, \ 1002 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 1003 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1004 } 1005 1006 #define ASAN_BUS_WRITE_FUNC(bytes, bits) \ 1007 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1008 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1009 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1010 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1011 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1012 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1013 bus_size_t count) \ 1014 { \ 1015 kasan_shadow_check((uintptr_t)buf, \ 1016 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1017 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1018 } \ 1019 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1020 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1021 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1022 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1023 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1024 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1025 bus_size_t count) \ 1026 { \ 1027 kasan_shadow_check((uintptr_t)buf, \ 1028 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1029 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1030 } \ 1031 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1032 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1033 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1034 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1035 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1036 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1037 bus_size_t count) \ 1038 { \ 1039 kasan_shadow_check((uintptr_t)buf, \ 1040 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1041 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1042 } \ 1043 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1044 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1045 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1046 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1047 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1048 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1049 bus_size_t count) \ 1050 { \ 1051 kasan_shadow_check((uintptr_t)buf, \ 1052 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1053 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1054 } 1055 1056 ASAN_BUS_READ_FUNC(1, 8) 1057 ASAN_BUS_READ_FUNC(2, 16) 1058 ASAN_BUS_READ_FUNC(4, 32) 1059 ASAN_BUS_READ_FUNC(8, 64) 1060 1061 ASAN_BUS_WRITE_FUNC(1, 8) 1062 ASAN_BUS_WRITE_FUNC(2, 16) 1063 ASAN_BUS_WRITE_FUNC(4, 32) 1064 ASAN_BUS_WRITE_FUNC(8, 64) 1065 1066 #endif /* __HAVE_KASAN_INSTR_BUS */ 1067 1068 /* -------------------------------------------------------------------------- */ 1069 1070 #include <sys/mbuf.h> 1071 1072 static void 1073 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1074 bool write, uintptr_t pc) 1075 { 1076 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc); 1077 } 1078 1079 static void 1080 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1081 bool write, uintptr_t pc) 1082 { 1083 bus_addr_t minlen; 1084 1085 for (; m != NULL && len != 0; m = m->m_next) { 1086 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc); 1087 1088 if (offset >= m->m_len) { 1089 offset -= m->m_len; 1090 continue; 1091 } 1092 1093 minlen = MIN(len, m->m_len - offset); 1094 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset), 1095 minlen, write, pc); 1096 1097 offset = 0; 1098 len -= minlen; 1099 } 1100 } 1101 1102 static void 1103 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1104 bool write, uintptr_t pc) 1105 { 1106 bus_size_t minlen, resid; 1107 struct iovec *iov; 1108 int i; 1109 1110 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc); 1111 1112 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1113 return; 1114 1115 resid = uio->uio_resid; 1116 iov = uio->uio_iov; 1117 1118 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1119 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]), 1120 false, pc); 1121 minlen = MIN(resid, iov[i].iov_len); 1122 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen, 1123 write, pc); 1124 resid -= minlen; 1125 } 1126 } 1127 1128 void 1129 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1130 { 1131 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0; 1132 1133 switch (map->dm_buftype) { 1134 case KASAN_DMA_LINEAR: 1135 kasan_dma_sync_linear(map->dm_buf, offset, len, write, 1136 __RET_ADDR); 1137 break; 1138 case KASAN_DMA_MBUF: 1139 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write, 1140 __RET_ADDR); 1141 break; 1142 case KASAN_DMA_UIO: 1143 kasan_dma_sync_uio(map->dm_buf, offset, len, write, 1144 __RET_ADDR); 1145 break; 1146 case KASAN_DMA_RAW: 1147 break; 1148 default: 1149 panic("%s: impossible", __func__); 1150 } 1151 } 1152 1153 void 1154 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1155 { 1156 map->dm_buf = buf; 1157 map->dm_buflen = buflen; 1158 map->dm_buftype = type; 1159 } 1160 1161 /* -------------------------------------------------------------------------- */ 1162 1163 void __asan_register_globals(struct __asan_global *, size_t); 1164 void __asan_unregister_globals(struct __asan_global *, size_t); 1165 1166 void 1167 __asan_register_globals(struct __asan_global *globals, size_t n) 1168 { 1169 size_t i; 1170 1171 for (i = 0; i < n; i++) { 1172 kasan_mark(globals[i].beg, globals[i].size, 1173 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 1174 } 1175 } 1176 1177 void 1178 __asan_unregister_globals(struct __asan_global *globals, size_t n) 1179 { 1180 /* never called */ 1181 } 1182 1183 #define ASAN_LOAD_STORE(size) \ 1184 void __asan_load##size(unsigned long); \ 1185 void __asan_load##size(unsigned long addr) \ 1186 { \ 1187 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1188 } \ 1189 void __asan_load##size##_noabort(unsigned long); \ 1190 void __asan_load##size##_noabort(unsigned long addr) \ 1191 { \ 1192 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1193 } \ 1194 void __asan_store##size(unsigned long); \ 1195 void __asan_store##size(unsigned long addr) \ 1196 { \ 1197 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1198 } \ 1199 void __asan_store##size##_noabort(unsigned long); \ 1200 void __asan_store##size##_noabort(unsigned long addr) \ 1201 { \ 1202 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1203 } 1204 1205 ASAN_LOAD_STORE(1); 1206 ASAN_LOAD_STORE(2); 1207 ASAN_LOAD_STORE(4); 1208 ASAN_LOAD_STORE(8); 1209 ASAN_LOAD_STORE(16); 1210 1211 void __asan_loadN(unsigned long, size_t); 1212 void __asan_loadN_noabort(unsigned long, size_t); 1213 void __asan_storeN(unsigned long, size_t); 1214 void __asan_storeN_noabort(unsigned long, size_t); 1215 void __asan_handle_no_return(void); 1216 1217 void 1218 __asan_loadN(unsigned long addr, size_t size) 1219 { 1220 kasan_shadow_check(addr, size, false, __RET_ADDR); 1221 } 1222 1223 void 1224 __asan_loadN_noabort(unsigned long addr, size_t size) 1225 { 1226 kasan_shadow_check(addr, size, false, __RET_ADDR); 1227 } 1228 1229 void 1230 __asan_storeN(unsigned long addr, size_t size) 1231 { 1232 kasan_shadow_check(addr, size, true, __RET_ADDR); 1233 } 1234 1235 void 1236 __asan_storeN_noabort(unsigned long addr, size_t size) 1237 { 1238 kasan_shadow_check(addr, size, true, __RET_ADDR); 1239 } 1240 1241 void 1242 __asan_handle_no_return(void) 1243 { 1244 /* nothing */ 1245 } 1246 1247 #define ASAN_SET_SHADOW(byte) \ 1248 void __asan_set_shadow_##byte(void *, size_t); \ 1249 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1250 { \ 1251 __builtin_memset((void *)addr, 0x##byte, size); \ 1252 } 1253 1254 ASAN_SET_SHADOW(00); 1255 ASAN_SET_SHADOW(f1); 1256 ASAN_SET_SHADOW(f2); 1257 ASAN_SET_SHADOW(f3); 1258 ASAN_SET_SHADOW(f5); 1259 ASAN_SET_SHADOW(f8); 1260 1261 void __asan_poison_stack_memory(const void *, size_t); 1262 void __asan_unpoison_stack_memory(const void *, size_t); 1263 1264 void 1265 __asan_poison_stack_memory(const void *addr, size_t size) 1266 { 1267 size = roundup(size, KASAN_SHADOW_SCALE_SIZE); 1268 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1269 } 1270 1271 void 1272 __asan_unpoison_stack_memory(const void *addr, size_t size) 1273 { 1274 kasan_shadow_Nbyte_markvalid(addr, size); 1275 } 1276 1277 void __asan_alloca_poison(const void *, size_t); 1278 void __asan_allocas_unpoison(const void *, const void *); 1279 1280 void __asan_alloca_poison(const void *addr, size_t size) 1281 { 1282 const void *l, *r; 1283 1284 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0); 1285 1286 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1287 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1288 1289 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1290 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1291 KASAN_STACK_MID); 1292 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1293 } 1294 1295 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1296 { 1297 size_t size; 1298 1299 if (__predict_false(!stkbegin)) 1300 return; 1301 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1302 return; 1303 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1304 1305 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1306 } 1307
Indexes created Sat Sep 20 22:09:52 GMT 2025