subr_asan.c revision 1.26.2.1
1 /* $NetBSD: subr_asan.c,v 1.26.2.1 2021/01/03 16:35:04 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KASAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26.2.1 2021/01/03 16:35:04 thorpej Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/asan.h> 42 43 #include <uvm/uvm_extern.h> 44 45 #ifdef DDB 46 #include <machine/db_machdep.h> 47 #include <ddb/db_extern.h> 48 #endif 49 50 #ifdef KASAN_PANIC 51 #define REPORT panic 52 #else 53 #define REPORT printf 54 #endif 55 56 /* ASAN constants. Part of the compiler ABI. */ 57 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 58 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1) 59 #define KASAN_ALLOCA_SCALE_SIZE 32 60 61 /* The MD code. */ 62 #include <machine/asan.h> 63 64 /* ASAN ABI version. */ 65 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 66 #define ASAN_ABI_VERSION 8 67 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 68 #define ASAN_ABI_VERSION 8 69 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 70 #define ASAN_ABI_VERSION 6 71 #else 72 #error "Unsupported compiler version" 73 #endif 74 75 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 76 77 /* Global variable descriptor. Part of the compiler ABI. */ 78 struct __asan_global_source_location { 79 const char *filename; 80 int line_no; 81 int column_no; 82 }; 83 struct __asan_global { 84 const void *beg; /* address of the global variable */ 85 size_t size; /* size of the global variable */ 86 size_t size_with_redzone; /* size with the redzone */ 87 const void *name; /* name of the variable */ 88 const void *module_name; /* name of the module where the var is declared */ 89 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 90 struct __asan_global_source_location *location; 91 #if ASAN_ABI_VERSION >= 7 92 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 93 #endif 94 }; 95 96 static bool kasan_enabled __read_mostly = false; 97 98 /* -------------------------------------------------------------------------- */ 99 100 void 101 kasan_shadow_map(void *addr, size_t size) 102 { 103 size_t sz, npages, i; 104 vaddr_t sva, eva; 105 106 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 107 108 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE; 109 110 sva = (vaddr_t)kasan_md_addr_to_shad(addr); 111 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz; 112 113 sva = rounddown(sva, PAGE_SIZE); 114 eva = roundup(eva, PAGE_SIZE); 115 116 npages = (eva - sva) / PAGE_SIZE; 117 118 KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END); 119 120 for (i = 0; i < npages; i++) { 121 kasan_md_shadow_map_page(sva + i * PAGE_SIZE); 122 } 123 } 124 125 static void 126 kasan_ctors(void) 127 { 128 extern Elf_Addr __CTOR_LIST__, __CTOR_END__; 129 size_t nentries, i; 130 Elf_Addr *ptr; 131 132 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) / 133 sizeof(uintptr_t); 134 135 ptr = &__CTOR_LIST__; 136 for (i = 0; i < nentries; i++) { 137 void (*func)(void); 138 139 func = (void *)(*ptr); 140 (*func)(); 141 142 ptr++; 143 } 144 } 145 146 void 147 kasan_early_init(void *stack) 148 { 149 kasan_md_early_init(stack); 150 } 151 152 void 153 kasan_init(void) 154 { 155 /* MD initialization. */ 156 kasan_md_init(); 157 158 /* Now officially enabled. */ 159 kasan_enabled = true; 160 161 /* Call the ASAN constructors. */ 162 kasan_ctors(); 163 } 164 165 static inline const char * 166 kasan_code_name(uint8_t code) 167 { 168 switch (code) { 169 case KASAN_GENERIC_REDZONE: 170 return "GenericRedZone"; 171 case KASAN_MALLOC_REDZONE: 172 return "MallocRedZone"; 173 case KASAN_KMEM_REDZONE: 174 return "KmemRedZone"; 175 case KASAN_POOL_REDZONE: 176 return "PoolRedZone"; 177 case KASAN_POOL_FREED: 178 return "PoolUseAfterFree"; 179 case 1 ... 7: 180 return "RedZonePartial"; 181 case KASAN_STACK_LEFT: 182 return "StackLeft"; 183 case KASAN_STACK_MID: 184 return "StackMiddle"; 185 case KASAN_STACK_RIGHT: 186 return "StackRight"; 187 case KASAN_USE_AFTER_RET: 188 return "UseAfterRet"; 189 case KASAN_USE_AFTER_SCOPE: 190 return "UseAfterScope"; 191 default: 192 return "Unknown"; 193 } 194 } 195 196 static void 197 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 198 uint8_t code) 199 { 200 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s," 201 " %s]\n", 202 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""), 203 (write ? "write" : "read"), kasan_code_name(code)); 204 kasan_md_unwind(); 205 } 206 207 static __always_inline void 208 kasan_shadow_1byte_markvalid(unsigned long addr) 209 { 210 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 211 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 212 213 *byte = last; 214 } 215 216 static __always_inline void 217 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 218 { 219 size_t i; 220 221 for (i = 0; i < size; i++) { 222 kasan_shadow_1byte_markvalid((unsigned long)addr+i); 223 } 224 } 225 226 static __always_inline void 227 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 228 { 229 void *shad; 230 231 if (__predict_false(size == 0)) 232 return; 233 if (__predict_false(kasan_md_unsupported((vaddr_t)addr))) 234 return; 235 236 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 237 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0); 238 239 shad = (void *)kasan_md_addr_to_shad(addr); 240 size = size >> KASAN_SHADOW_SCALE_SHIFT; 241 242 __builtin_memset(shad, code, size); 243 } 244 245 void 246 kasan_add_redzone(size_t *size) 247 { 248 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE); 249 *size += KASAN_SHADOW_SCALE_SIZE; 250 } 251 252 void 253 kasan_softint(struct lwp *l) 254 { 255 const void *stk = (const void *)uvm_lwp_getuarea(l); 256 257 kasan_shadow_Nbyte_fill(stk, USPACE, 0); 258 } 259 260 /* 261 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 262 * and the rest as invalid. There are generally two use cases: 263 * 264 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 265 * the redzone at the end of the buffer as invalid. 266 * 267 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 268 */ 269 void 270 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code) 271 { 272 size_t i, n, redz; 273 int8_t *shad; 274 275 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 276 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE); 277 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0); 278 shad = kasan_md_addr_to_shad(addr); 279 280 /* Chunks of 8 bytes, valid. */ 281 n = size / KASAN_SHADOW_SCALE_SIZE; 282 for (i = 0; i < n; i++) { 283 *shad++ = 0; 284 } 285 286 /* Possibly one chunk, mid. */ 287 if ((size & KASAN_SHADOW_MASK) != 0) { 288 *shad++ = (size & KASAN_SHADOW_MASK); 289 } 290 291 /* Chunks of 8 bytes, invalid. */ 292 n = redz / KASAN_SHADOW_SCALE_SIZE; 293 for (i = 0; i < n; i++) { 294 *shad++ = code; 295 } 296 } 297 298 /* -------------------------------------------------------------------------- */ 299 300 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 301 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 302 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 303 304 static __always_inline bool 305 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 306 { 307 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 308 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 309 310 if (__predict_true(*byte == 0 || last <= *byte)) { 311 return true; 312 } 313 *code = *byte; 314 return false; 315 } 316 317 static __always_inline bool 318 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 319 { 320 int8_t *byte, last; 321 322 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 323 return (kasan_shadow_1byte_isvalid(addr, code) && 324 kasan_shadow_1byte_isvalid(addr+1, code)); 325 } 326 327 byte = kasan_md_addr_to_shad((void *)addr); 328 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 329 330 if (__predict_true(*byte == 0 || last <= *byte)) { 331 return true; 332 } 333 *code = *byte; 334 return false; 335 } 336 337 static __always_inline bool 338 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 339 { 340 int8_t *byte, last; 341 342 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 343 return (kasan_shadow_2byte_isvalid(addr, code) && 344 kasan_shadow_2byte_isvalid(addr+2, code)); 345 } 346 347 byte = kasan_md_addr_to_shad((void *)addr); 348 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 349 350 if (__predict_true(*byte == 0 || last <= *byte)) { 351 return true; 352 } 353 *code = *byte; 354 return false; 355 } 356 357 static __always_inline bool 358 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 359 { 360 int8_t *byte, last; 361 362 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 363 return (kasan_shadow_4byte_isvalid(addr, code) && 364 kasan_shadow_4byte_isvalid(addr+4, code)); 365 } 366 367 byte = kasan_md_addr_to_shad((void *)addr); 368 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 369 370 if (__predict_true(*byte == 0 || last <= *byte)) { 371 return true; 372 } 373 *code = *byte; 374 return false; 375 } 376 377 static __always_inline bool 378 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 379 { 380 size_t i; 381 382 for (i = 0; i < size; i++) { 383 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 384 return false; 385 } 386 387 return true; 388 } 389 390 static __always_inline void 391 kasan_shadow_check(unsigned long addr, size_t size, bool write, 392 unsigned long retaddr) 393 { 394 uint8_t code; 395 bool valid; 396 397 if (__predict_false(!kasan_enabled)) 398 return; 399 #ifdef DDB 400 if (__predict_false(db_recover != NULL)) 401 return; 402 #endif 403 if (__predict_false(size == 0)) 404 return; 405 if (__predict_false(kasan_md_unsupported(addr))) 406 return; 407 408 if (__builtin_constant_p(size)) { 409 switch (size) { 410 case 1: 411 valid = kasan_shadow_1byte_isvalid(addr, &code); 412 break; 413 case 2: 414 valid = kasan_shadow_2byte_isvalid(addr, &code); 415 break; 416 case 4: 417 valid = kasan_shadow_4byte_isvalid(addr, &code); 418 break; 419 case 8: 420 valid = kasan_shadow_8byte_isvalid(addr, &code); 421 break; 422 default: 423 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 424 break; 425 } 426 } else { 427 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 428 } 429 430 if (__predict_false(!valid)) { 431 kasan_report(addr, size, write, retaddr, code); 432 } 433 } 434 435 /* -------------------------------------------------------------------------- */ 436 437 void * 438 kasan_memcpy(void *dst, const void *src, size_t len) 439 { 440 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 441 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 442 return __builtin_memcpy(dst, src, len); 443 } 444 445 int 446 kasan_memcmp(const void *b1, const void *b2, size_t len) 447 { 448 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 449 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 450 return __builtin_memcmp(b1, b2, len); 451 } 452 453 void * 454 kasan_memset(void *b, int c, size_t len) 455 { 456 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 457 return __builtin_memset(b, c, len); 458 } 459 460 void * 461 kasan_memmove(void *dst, const void *src, size_t len) 462 { 463 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 464 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 465 return __builtin_memmove(dst, src, len); 466 } 467 468 char * 469 kasan_strcpy(char *dst, const char *src) 470 { 471 char *save = dst; 472 473 while (1) { 474 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 475 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 476 *dst = *src; 477 if (*src == '\0') 478 break; 479 src++, dst++; 480 } 481 482 return save; 483 } 484 485 int 486 kasan_strcmp(const char *s1, const char *s2) 487 { 488 while (1) { 489 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 490 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 491 if (*s1 != *s2) 492 break; 493 if (*s1 == '\0') 494 return 0; 495 s1++, s2++; 496 } 497 498 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 499 } 500 501 size_t 502 kasan_strlen(const char *str) 503 { 504 const char *s; 505 506 s = str; 507 while (1) { 508 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 509 if (*s == '\0') 510 break; 511 s++; 512 } 513 514 return (s - str); 515 } 516 517 char * 518 kasan_strcat(char *dst, const char *src) 519 { 520 size_t ldst, lsrc; 521 522 ldst = __builtin_strlen(dst); 523 lsrc = __builtin_strlen(src); 524 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true, 525 __RET_ADDR); 526 kasan_shadow_check((unsigned long)src, lsrc + 1, false, 527 __RET_ADDR); 528 529 return __builtin_strcat(dst, src); 530 } 531 532 char * 533 kasan_strchr(const char *s, int c) 534 { 535 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 536 __RET_ADDR); 537 return __builtin_strchr(s, c); 538 } 539 540 char * 541 kasan_strrchr(const char *s, int c) 542 { 543 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 544 __RET_ADDR); 545 return __builtin_strrchr(s, c); 546 } 547 548 #undef kcopy 549 #undef copyinstr 550 #undef copyoutstr 551 #undef copyin 552 553 int kasan_kcopy(const void *, void *, size_t); 554 int kasan_copyinstr(const void *, void *, size_t, size_t *); 555 int kasan_copyoutstr(const void *, void *, size_t, size_t *); 556 int kasan_copyin(const void *, void *, size_t); 557 int kcopy(const void *, void *, size_t); 558 int copyinstr(const void *, void *, size_t, size_t *); 559 int copyoutstr(const void *, void *, size_t, size_t *); 560 int copyin(const void *, void *, size_t); 561 562 int 563 kasan_kcopy(const void *src, void *dst, size_t len) 564 { 565 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 566 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 567 return kcopy(src, dst, len); 568 } 569 570 int 571 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 572 { 573 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 574 return copyin(uaddr, kaddr, len); 575 } 576 577 int 578 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 579 { 580 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 581 return copyinstr(uaddr, kaddr, len, done); 582 } 583 584 int 585 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 586 { 587 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 588 return copyoutstr(kaddr, uaddr, len, done); 589 } 590 591 /* -------------------------------------------------------------------------- */ 592 593 #undef _ucas_32 594 #undef _ucas_32_mp 595 #undef _ucas_64 596 #undef _ucas_64_mp 597 #undef _ufetch_8 598 #undef _ufetch_16 599 #undef _ufetch_32 600 #undef _ufetch_64 601 602 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 603 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 604 int 605 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 606 uint32_t *ret) 607 { 608 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 609 __RET_ADDR); 610 return _ucas_32(uaddr, old, new, ret); 611 } 612 613 #ifdef __HAVE_UCAS_MP 614 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 615 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 616 int 617 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 618 uint32_t *ret) 619 { 620 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 621 __RET_ADDR); 622 return _ucas_32_mp(uaddr, old, new, ret); 623 } 624 #endif 625 626 #ifdef _LP64 627 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 628 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 629 int 630 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 631 uint64_t *ret) 632 { 633 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 634 __RET_ADDR); 635 return _ucas_64(uaddr, old, new, ret); 636 } 637 638 #ifdef __HAVE_UCAS_MP 639 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 640 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 641 int 642 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 643 uint64_t *ret) 644 { 645 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 646 __RET_ADDR); 647 return _ucas_64_mp(uaddr, old, new, ret); 648 } 649 #endif 650 #endif 651 652 int _ufetch_8(const uint8_t *, uint8_t *); 653 int kasan__ufetch_8(const uint8_t *, uint8_t *); 654 int 655 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 656 { 657 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 658 __RET_ADDR); 659 return _ufetch_8(uaddr, valp); 660 } 661 662 int _ufetch_16(const uint16_t *, uint16_t *); 663 int kasan__ufetch_16(const uint16_t *, uint16_t *); 664 int 665 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 666 { 667 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 668 __RET_ADDR); 669 return _ufetch_16(uaddr, valp); 670 } 671 672 int _ufetch_32(const uint32_t *, uint32_t *); 673 int kasan__ufetch_32(const uint32_t *, uint32_t *); 674 int 675 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 676 { 677 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 678 __RET_ADDR); 679 return _ufetch_32(uaddr, valp); 680 } 681 682 #ifdef _LP64 683 int _ufetch_64(const uint64_t *, uint64_t *); 684 int kasan__ufetch_64(const uint64_t *, uint64_t *); 685 int 686 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 687 { 688 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 689 __RET_ADDR); 690 return _ufetch_64(uaddr, valp); 691 } 692 #endif 693 694 /* -------------------------------------------------------------------------- */ 695 696 #undef atomic_add_32 697 #undef atomic_add_int 698 #undef atomic_add_long 699 #undef atomic_add_ptr 700 #undef atomic_add_64 701 #undef atomic_add_32_nv 702 #undef atomic_add_int_nv 703 #undef atomic_add_long_nv 704 #undef atomic_add_ptr_nv 705 #undef atomic_add_64_nv 706 #undef atomic_and_32 707 #undef atomic_and_uint 708 #undef atomic_and_ulong 709 #undef atomic_and_64 710 #undef atomic_and_32_nv 711 #undef atomic_and_uint_nv 712 #undef atomic_and_ulong_nv 713 #undef atomic_and_64_nv 714 #undef atomic_or_32 715 #undef atomic_or_uint 716 #undef atomic_or_ulong 717 #undef atomic_or_64 718 #undef atomic_or_32_nv 719 #undef atomic_or_uint_nv 720 #undef atomic_or_ulong_nv 721 #undef atomic_or_64_nv 722 #undef atomic_cas_32 723 #undef atomic_cas_uint 724 #undef atomic_cas_ulong 725 #undef atomic_cas_ptr 726 #undef atomic_cas_64 727 #undef atomic_cas_32_ni 728 #undef atomic_cas_uint_ni 729 #undef atomic_cas_ulong_ni 730 #undef atomic_cas_ptr_ni 731 #undef atomic_cas_64_ni 732 #undef atomic_swap_32 733 #undef atomic_swap_uint 734 #undef atomic_swap_ulong 735 #undef atomic_swap_ptr 736 #undef atomic_swap_64 737 #undef atomic_dec_32 738 #undef atomic_dec_uint 739 #undef atomic_dec_ulong 740 #undef atomic_dec_ptr 741 #undef atomic_dec_64 742 #undef atomic_dec_32_nv 743 #undef atomic_dec_uint_nv 744 #undef atomic_dec_ulong_nv 745 #undef atomic_dec_ptr_nv 746 #undef atomic_dec_64_nv 747 #undef atomic_inc_32 748 #undef atomic_inc_uint 749 #undef atomic_inc_ulong 750 #undef atomic_inc_ptr 751 #undef atomic_inc_64 752 #undef atomic_inc_32_nv 753 #undef atomic_inc_uint_nv 754 #undef atomic_inc_ulong_nv 755 #undef atomic_inc_ptr_nv 756 #undef atomic_inc_64_nv 757 758 #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 759 void atomic_add_##name(volatile targ1 *, targ2); \ 760 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 761 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 762 { \ 763 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 764 __RET_ADDR); \ 765 atomic_add_##name(ptr, val); \ 766 } \ 767 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 768 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 769 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 770 { \ 771 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 772 __RET_ADDR); \ 773 return atomic_add_##name##_nv(ptr, val); \ 774 } 775 776 #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 777 void atomic_and_##name(volatile targ1 *, targ2); \ 778 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 779 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 780 { \ 781 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 782 __RET_ADDR); \ 783 atomic_and_##name(ptr, val); \ 784 } \ 785 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 786 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 787 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 788 { \ 789 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 790 __RET_ADDR); \ 791 return atomic_and_##name##_nv(ptr, val); \ 792 } 793 794 #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 795 void atomic_or_##name(volatile targ1 *, targ2); \ 796 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 797 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 798 { \ 799 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 800 __RET_ADDR); \ 801 atomic_or_##name(ptr, val); \ 802 } \ 803 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 804 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 805 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 806 { \ 807 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 808 __RET_ADDR); \ 809 return atomic_or_##name##_nv(ptr, val); \ 810 } 811 812 #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 813 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 814 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 815 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 816 { \ 817 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 818 __RET_ADDR); \ 819 return atomic_cas_##name(ptr, exp, new); \ 820 } \ 821 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 822 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 823 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 824 { \ 825 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 826 __RET_ADDR); \ 827 return atomic_cas_##name##_ni(ptr, exp, new); \ 828 } 829 830 #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 831 tret atomic_swap_##name(volatile targ1 *, targ2); \ 832 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \ 833 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 834 { \ 835 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 836 __RET_ADDR); \ 837 return atomic_swap_##name(ptr, val); \ 838 } 839 840 #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 841 void atomic_dec_##name(volatile targ1 *); \ 842 void kasan_atomic_dec_##name(volatile targ1 *); \ 843 void kasan_atomic_dec_##name(volatile targ1 *ptr) \ 844 { \ 845 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 846 __RET_ADDR); \ 847 atomic_dec_##name(ptr); \ 848 } \ 849 tret atomic_dec_##name##_nv(volatile targ1 *); \ 850 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \ 851 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 852 { \ 853 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 854 __RET_ADDR); \ 855 return atomic_dec_##name##_nv(ptr); \ 856 } 857 858 #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 859 void atomic_inc_##name(volatile targ1 *); \ 860 void kasan_atomic_inc_##name(volatile targ1 *); \ 861 void kasan_atomic_inc_##name(volatile targ1 *ptr) \ 862 { \ 863 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 864 __RET_ADDR); \ 865 atomic_inc_##name(ptr); \ 866 } \ 867 tret atomic_inc_##name##_nv(volatile targ1 *); \ 868 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \ 869 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 870 { \ 871 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 872 __RET_ADDR); \ 873 return atomic_inc_##name##_nv(ptr); \ 874 } 875 876 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 877 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 878 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 879 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 880 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 881 882 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 883 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 884 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 885 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 886 887 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 888 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 889 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 890 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 891 892 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 893 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 894 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 895 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 896 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 897 898 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 899 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 900 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 901 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 902 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 903 904 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 905 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 906 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 907 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 908 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void); 909 910 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 911 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 912 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 913 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 914 ASAN_ATOMIC_FUNC_INC(ptr, void *, void); 915 916 /* -------------------------------------------------------------------------- */ 917 918 #ifdef __HAVE_KASAN_INSTR_BUS 919 920 #include <sys/bus.h> 921 922 #undef bus_space_read_multi_1 923 #undef bus_space_read_multi_2 924 #undef bus_space_read_multi_4 925 #undef bus_space_read_multi_8 926 #undef bus_space_read_multi_stream_1 927 #undef bus_space_read_multi_stream_2 928 #undef bus_space_read_multi_stream_4 929 #undef bus_space_read_multi_stream_8 930 #undef bus_space_read_region_1 931 #undef bus_space_read_region_2 932 #undef bus_space_read_region_4 933 #undef bus_space_read_region_8 934 #undef bus_space_read_region_stream_1 935 #undef bus_space_read_region_stream_2 936 #undef bus_space_read_region_stream_4 937 #undef bus_space_read_region_stream_8 938 #undef bus_space_write_multi_1 939 #undef bus_space_write_multi_2 940 #undef bus_space_write_multi_4 941 #undef bus_space_write_multi_8 942 #undef bus_space_write_multi_stream_1 943 #undef bus_space_write_multi_stream_2 944 #undef bus_space_write_multi_stream_4 945 #undef bus_space_write_multi_stream_8 946 #undef bus_space_write_region_1 947 #undef bus_space_write_region_2 948 #undef bus_space_write_region_4 949 #undef bus_space_write_region_8 950 #undef bus_space_write_region_stream_1 951 #undef bus_space_write_region_stream_2 952 #undef bus_space_write_region_stream_4 953 #undef bus_space_write_region_stream_8 954 955 #define ASAN_BUS_READ_FUNC(bytes, bits) \ 956 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 957 bus_size_t, uint##bits##_t *, bus_size_t); \ 958 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 959 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 960 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 961 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 962 bus_size_t count) \ 963 { \ 964 kasan_shadow_check((uintptr_t)buf, \ 965 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 966 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 967 } \ 968 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 969 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 970 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 971 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 972 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 973 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 974 bus_size_t count) \ 975 { \ 976 kasan_shadow_check((uintptr_t)buf, \ 977 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 978 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 979 } \ 980 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 981 bus_size_t, uint##bits##_t *, bus_size_t); \ 982 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \ 983 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 984 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 985 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 986 bus_size_t count) \ 987 { \ 988 kasan_shadow_check((uintptr_t)buf, \ 989 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 990 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 991 } \ 992 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 993 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 994 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 995 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 996 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 997 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 998 bus_size_t count) \ 999 { \ 1000 kasan_shadow_check((uintptr_t)buf, \ 1001 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 1002 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1003 } 1004 1005 #define ASAN_BUS_WRITE_FUNC(bytes, bits) \ 1006 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1007 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1008 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1009 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1010 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1011 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1012 bus_size_t count) \ 1013 { \ 1014 kasan_shadow_check((uintptr_t)buf, \ 1015 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1016 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1017 } \ 1018 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1019 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1020 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1021 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1022 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1023 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1024 bus_size_t count) \ 1025 { \ 1026 kasan_shadow_check((uintptr_t)buf, \ 1027 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1028 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1029 } \ 1030 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1031 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1032 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1033 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1034 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1035 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1036 bus_size_t count) \ 1037 { \ 1038 kasan_shadow_check((uintptr_t)buf, \ 1039 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1040 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1041 } \ 1042 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1043 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1044 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1045 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1046 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1047 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1048 bus_size_t count) \ 1049 { \ 1050 kasan_shadow_check((uintptr_t)buf, \ 1051 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1052 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1053 } 1054 1055 ASAN_BUS_READ_FUNC(1, 8) 1056 ASAN_BUS_READ_FUNC(2, 16) 1057 ASAN_BUS_READ_FUNC(4, 32) 1058 ASAN_BUS_READ_FUNC(8, 64) 1059 1060 ASAN_BUS_WRITE_FUNC(1, 8) 1061 ASAN_BUS_WRITE_FUNC(2, 16) 1062 ASAN_BUS_WRITE_FUNC(4, 32) 1063 ASAN_BUS_WRITE_FUNC(8, 64) 1064 1065 #endif /* __HAVE_KASAN_INSTR_BUS */ 1066 1067 /* -------------------------------------------------------------------------- */ 1068 1069 #include <sys/mbuf.h> 1070 1071 static void 1072 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1073 bool write, uintptr_t pc) 1074 { 1075 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc); 1076 } 1077 1078 static void 1079 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1080 bool write, uintptr_t pc) 1081 { 1082 bus_addr_t minlen; 1083 1084 for (; m != NULL && len != 0; m = m->m_next) { 1085 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc); 1086 1087 if (offset >= m->m_len) { 1088 offset -= m->m_len; 1089 continue; 1090 } 1091 1092 minlen = MIN(len, m->m_len - offset); 1093 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset), 1094 minlen, write, pc); 1095 1096 offset = 0; 1097 len -= minlen; 1098 } 1099 } 1100 1101 static void 1102 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1103 bool write, uintptr_t pc) 1104 { 1105 bus_size_t minlen, resid; 1106 struct iovec *iov; 1107 int i; 1108 1109 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc); 1110 1111 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1112 return; 1113 1114 resid = uio->uio_resid; 1115 iov = uio->uio_iov; 1116 1117 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1118 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]), 1119 false, pc); 1120 minlen = MIN(resid, iov[i].iov_len); 1121 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen, 1122 write, pc); 1123 resid -= minlen; 1124 } 1125 } 1126 1127 void 1128 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1129 { 1130 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0; 1131 1132 switch (map->dm_buftype) { 1133 case KASAN_DMA_LINEAR: 1134 kasan_dma_sync_linear(map->dm_buf, offset, len, write, 1135 __RET_ADDR); 1136 break; 1137 case KASAN_DMA_MBUF: 1138 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write, 1139 __RET_ADDR); 1140 break; 1141 case KASAN_DMA_UIO: 1142 kasan_dma_sync_uio(map->dm_buf, offset, len, write, 1143 __RET_ADDR); 1144 break; 1145 case KASAN_DMA_RAW: 1146 break; 1147 default: 1148 panic("%s: impossible", __func__); 1149 } 1150 } 1151 1152 void 1153 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1154 { 1155 map->dm_buf = buf; 1156 map->dm_buflen = buflen; 1157 map->dm_buftype = type; 1158 } 1159 1160 /* -------------------------------------------------------------------------- */ 1161 1162 void __asan_register_globals(struct __asan_global *, size_t); 1163 void __asan_unregister_globals(struct __asan_global *, size_t); 1164 1165 void 1166 __asan_register_globals(struct __asan_global *globals, size_t n) 1167 { 1168 size_t i; 1169 1170 for (i = 0; i < n; i++) { 1171 kasan_mark(globals[i].beg, globals[i].size, 1172 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 1173 } 1174 } 1175 1176 void 1177 __asan_unregister_globals(struct __asan_global *globals, size_t n) 1178 { 1179 /* never called */ 1180 } 1181 1182 #define ASAN_LOAD_STORE(size) \ 1183 void __asan_load##size(unsigned long); \ 1184 void __asan_load##size(unsigned long addr) \ 1185 { \ 1186 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1187 } \ 1188 void __asan_load##size##_noabort(unsigned long); \ 1189 void __asan_load##size##_noabort(unsigned long addr) \ 1190 { \ 1191 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1192 } \ 1193 void __asan_store##size(unsigned long); \ 1194 void __asan_store##size(unsigned long addr) \ 1195 { \ 1196 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1197 } \ 1198 void __asan_store##size##_noabort(unsigned long); \ 1199 void __asan_store##size##_noabort(unsigned long addr) \ 1200 { \ 1201 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1202 } 1203 1204 ASAN_LOAD_STORE(1); 1205 ASAN_LOAD_STORE(2); 1206 ASAN_LOAD_STORE(4); 1207 ASAN_LOAD_STORE(8); 1208 ASAN_LOAD_STORE(16); 1209 1210 void __asan_loadN(unsigned long, size_t); 1211 void __asan_loadN_noabort(unsigned long, size_t); 1212 void __asan_storeN(unsigned long, size_t); 1213 void __asan_storeN_noabort(unsigned long, size_t); 1214 void __asan_handle_no_return(void); 1215 1216 void 1217 __asan_loadN(unsigned long addr, size_t size) 1218 { 1219 kasan_shadow_check(addr, size, false, __RET_ADDR); 1220 } 1221 1222 void 1223 __asan_loadN_noabort(unsigned long addr, size_t size) 1224 { 1225 kasan_shadow_check(addr, size, false, __RET_ADDR); 1226 } 1227 1228 void 1229 __asan_storeN(unsigned long addr, size_t size) 1230 { 1231 kasan_shadow_check(addr, size, true, __RET_ADDR); 1232 } 1233 1234 void 1235 __asan_storeN_noabort(unsigned long addr, size_t size) 1236 { 1237 kasan_shadow_check(addr, size, true, __RET_ADDR); 1238 } 1239 1240 void 1241 __asan_handle_no_return(void) 1242 { 1243 /* nothing */ 1244 } 1245 1246 #define ASAN_SET_SHADOW(byte) \ 1247 void __asan_set_shadow_##byte(void *, size_t); \ 1248 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1249 { \ 1250 __builtin_memset((void *)addr, 0x##byte, size); \ 1251 } 1252 1253 ASAN_SET_SHADOW(00); 1254 ASAN_SET_SHADOW(f1); 1255 ASAN_SET_SHADOW(f2); 1256 ASAN_SET_SHADOW(f3); 1257 ASAN_SET_SHADOW(f5); 1258 ASAN_SET_SHADOW(f8); 1259 1260 void __asan_poison_stack_memory(const void *, size_t); 1261 void __asan_unpoison_stack_memory(const void *, size_t); 1262 1263 void 1264 __asan_poison_stack_memory(const void *addr, size_t size) 1265 { 1266 size = roundup(size, KASAN_SHADOW_SCALE_SIZE); 1267 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1268 } 1269 1270 void 1271 __asan_unpoison_stack_memory(const void *addr, size_t size) 1272 { 1273 kasan_shadow_Nbyte_markvalid(addr, size); 1274 } 1275 1276 void __asan_alloca_poison(const void *, size_t); 1277 void __asan_allocas_unpoison(const void *, const void *); 1278 1279 void __asan_alloca_poison(const void *addr, size_t size) 1280 { 1281 const void *l, *r; 1282 1283 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0); 1284 1285 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1286 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1287 1288 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1289 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1290 KASAN_STACK_MID); 1291 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1292 } 1293 1294 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1295 { 1296 size_t size; 1297 1298 if (__predict_false(!stkbegin)) 1299 return; 1300 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1301 return; 1302 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1303 1304 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1305 } 1306
Indexes created Sun Sep 21 20:09:37 GMT 2025