subr_asan.c revision 1.25
1 /* $NetBSD: subr_asan.c,v 1.25 2020/09/05 16:30:12 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.25 2020/09/05 16:30:12 riastradh Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/asan.h> 43 44 #include <uvm/uvm_extern.h> 45 46 #ifdef KASAN_PANIC 47 #define REPORT panic 48 #else 49 #define REPORT printf 50 #endif 51 52 /* ASAN constants. Part of the compiler ABI. */ 53 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 54 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1) 55 #define KASAN_ALLOCA_SCALE_SIZE 32 56 57 /* The MD code. */ 58 #include <machine/asan.h> 59 60 /* ASAN ABI version. */ 61 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 62 #define ASAN_ABI_VERSION 8 63 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 64 #define ASAN_ABI_VERSION 8 65 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 66 #define ASAN_ABI_VERSION 6 67 #else 68 #error "Unsupported compiler version" 69 #endif 70 71 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 72 73 /* Global variable descriptor. Part of the compiler ABI. */ 74 struct __asan_global_source_location { 75 const char *filename; 76 int line_no; 77 int column_no; 78 }; 79 struct __asan_global { 80 const void *beg; /* address of the global variable */ 81 size_t size; /* size of the global variable */ 82 size_t size_with_redzone; /* size with the redzone */ 83 const void *name; /* name of the variable */ 84 const void *module_name; /* name of the module where the var is declared */ 85 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 86 struct __asan_global_source_location *location; 87 #if ASAN_ABI_VERSION >= 7 88 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 89 #endif 90 }; 91 92 static bool kasan_enabled __read_mostly = false; 93 94 /* -------------------------------------------------------------------------- */ 95 96 void 97 kasan_shadow_map(void *addr, size_t size) 98 { 99 size_t sz, npages, i; 100 vaddr_t sva, eva; 101 102 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 103 104 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE; 105 106 sva = (vaddr_t)kasan_md_addr_to_shad(addr); 107 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz; 108 109 sva = rounddown(sva, PAGE_SIZE); 110 eva = roundup(eva, PAGE_SIZE); 111 112 npages = (eva - sva) / PAGE_SIZE; 113 114 KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END); 115 116 for (i = 0; i < npages; i++) { 117 kasan_md_shadow_map_page(sva + i * PAGE_SIZE); 118 } 119 } 120 121 static void 122 kasan_ctors(void) 123 { 124 extern Elf_Addr __CTOR_LIST__, __CTOR_END__; 125 size_t nentries, i; 126 Elf_Addr *ptr; 127 128 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) / 129 sizeof(uintptr_t); 130 131 ptr = &__CTOR_LIST__; 132 for (i = 0; i < nentries; i++) { 133 void (*func)(void); 134 135 func = (void *)(*ptr); 136 (*func)(); 137 138 ptr++; 139 } 140 } 141 142 void 143 kasan_early_init(void *stack) 144 { 145 kasan_md_early_init(stack); 146 } 147 148 void 149 kasan_init(void) 150 { 151 /* MD initialization. */ 152 kasan_md_init(); 153 154 /* Now officially enabled. */ 155 kasan_enabled = true; 156 157 /* Call the ASAN constructors. */ 158 kasan_ctors(); 159 } 160 161 static inline const char * 162 kasan_code_name(uint8_t code) 163 { 164 switch (code) { 165 case KASAN_GENERIC_REDZONE: 166 return "GenericRedZone"; 167 case KASAN_MALLOC_REDZONE: 168 return "MallocRedZone"; 169 case KASAN_KMEM_REDZONE: 170 return "KmemRedZone"; 171 case KASAN_POOL_REDZONE: 172 return "PoolRedZone"; 173 case KASAN_POOL_FREED: 174 return "PoolUseAfterFree"; 175 case 1 ... 7: 176 return "RedZonePartial"; 177 case KASAN_STACK_LEFT: 178 return "StackLeft"; 179 case KASAN_STACK_MID: 180 return "StackMiddle"; 181 case KASAN_STACK_RIGHT: 182 return "StackRight"; 183 case KASAN_USE_AFTER_RET: 184 return "UseAfterRet"; 185 case KASAN_USE_AFTER_SCOPE: 186 return "UseAfterScope"; 187 default: 188 return "Unknown"; 189 } 190 } 191 192 static void 193 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 194 uint8_t code) 195 { 196 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s," 197 " %s]\n", 198 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""), 199 (write ? "write" : "read"), kasan_code_name(code)); 200 kasan_md_unwind(); 201 } 202 203 static __always_inline void 204 kasan_shadow_1byte_markvalid(unsigned long addr) 205 { 206 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 207 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 208 209 *byte = last; 210 } 211 212 static __always_inline void 213 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 214 { 215 size_t i; 216 217 for (i = 0; i < size; i++) { 218 kasan_shadow_1byte_markvalid((unsigned long)addr+i); 219 } 220 } 221 222 static __always_inline void 223 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 224 { 225 void *shad; 226 227 if (__predict_false(size == 0)) 228 return; 229 if (__predict_false(kasan_md_unsupported((vaddr_t)addr))) 230 return; 231 232 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 233 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0); 234 235 shad = (void *)kasan_md_addr_to_shad(addr); 236 size = size >> KASAN_SHADOW_SCALE_SHIFT; 237 238 __builtin_memset(shad, code, size); 239 } 240 241 void 242 kasan_add_redzone(size_t *size) 243 { 244 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE); 245 *size += KASAN_SHADOW_SCALE_SIZE; 246 } 247 248 void 249 kasan_softint(struct lwp *l) 250 { 251 const void *stk = (const void *)uvm_lwp_getuarea(l); 252 253 kasan_shadow_Nbyte_fill(stk, USPACE, 0); 254 } 255 256 /* 257 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 258 * and the rest as invalid. There are generally two use cases: 259 * 260 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 261 * the redzone at the end of the buffer as invalid. 262 * 263 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 264 */ 265 void 266 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code) 267 { 268 size_t i, n, redz; 269 int8_t *shad; 270 271 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 272 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE); 273 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0); 274 shad = kasan_md_addr_to_shad(addr); 275 276 /* Chunks of 8 bytes, valid. */ 277 n = size / KASAN_SHADOW_SCALE_SIZE; 278 for (i = 0; i < n; i++) { 279 *shad++ = 0; 280 } 281 282 /* Possibly one chunk, mid. */ 283 if ((size & KASAN_SHADOW_MASK) != 0) { 284 *shad++ = (size & KASAN_SHADOW_MASK); 285 } 286 287 /* Chunks of 8 bytes, invalid. */ 288 n = redz / KASAN_SHADOW_SCALE_SIZE; 289 for (i = 0; i < n; i++) { 290 *shad++ = code; 291 } 292 } 293 294 /* -------------------------------------------------------------------------- */ 295 296 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 297 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 298 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 299 300 static __always_inline bool 301 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 302 { 303 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 304 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 305 306 if (__predict_true(*byte == 0 || last <= *byte)) { 307 return true; 308 } 309 *code = *byte; 310 return false; 311 } 312 313 static __always_inline bool 314 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 315 { 316 int8_t *byte, last; 317 318 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 319 return (kasan_shadow_1byte_isvalid(addr, code) && 320 kasan_shadow_1byte_isvalid(addr+1, code)); 321 } 322 323 byte = kasan_md_addr_to_shad((void *)addr); 324 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 325 326 if (__predict_true(*byte == 0 || last <= *byte)) { 327 return true; 328 } 329 *code = *byte; 330 return false; 331 } 332 333 static __always_inline bool 334 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 335 { 336 int8_t *byte, last; 337 338 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 339 return (kasan_shadow_2byte_isvalid(addr, code) && 340 kasan_shadow_2byte_isvalid(addr+2, code)); 341 } 342 343 byte = kasan_md_addr_to_shad((void *)addr); 344 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 345 346 if (__predict_true(*byte == 0 || last <= *byte)) { 347 return true; 348 } 349 *code = *byte; 350 return false; 351 } 352 353 static __always_inline bool 354 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 355 { 356 int8_t *byte, last; 357 358 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 359 return (kasan_shadow_4byte_isvalid(addr, code) && 360 kasan_shadow_4byte_isvalid(addr+4, code)); 361 } 362 363 byte = kasan_md_addr_to_shad((void *)addr); 364 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 365 366 if (__predict_true(*byte == 0 || last <= *byte)) { 367 return true; 368 } 369 *code = *byte; 370 return false; 371 } 372 373 static __always_inline bool 374 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 375 { 376 size_t i; 377 378 for (i = 0; i < size; i++) { 379 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 380 return false; 381 } 382 383 return true; 384 } 385 386 static __always_inline void 387 kasan_shadow_check(unsigned long addr, size_t size, bool write, 388 unsigned long retaddr) 389 { 390 uint8_t code; 391 bool valid; 392 393 if (__predict_false(!kasan_enabled)) 394 return; 395 if (__predict_false(size == 0)) 396 return; 397 if (__predict_false(kasan_md_unsupported(addr))) 398 return; 399 400 if (__builtin_constant_p(size)) { 401 switch (size) { 402 case 1: 403 valid = kasan_shadow_1byte_isvalid(addr, &code); 404 break; 405 case 2: 406 valid = kasan_shadow_2byte_isvalid(addr, &code); 407 break; 408 case 4: 409 valid = kasan_shadow_4byte_isvalid(addr, &code); 410 break; 411 case 8: 412 valid = kasan_shadow_8byte_isvalid(addr, &code); 413 break; 414 default: 415 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 416 break; 417 } 418 } else { 419 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 420 } 421 422 if (__predict_false(!valid)) { 423 kasan_report(addr, size, write, retaddr, code); 424 } 425 } 426 427 /* -------------------------------------------------------------------------- */ 428 429 void * 430 kasan_memcpy(void *dst, const void *src, size_t len) 431 { 432 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 433 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 434 return __builtin_memcpy(dst, src, len); 435 } 436 437 int 438 kasan_memcmp(const void *b1, const void *b2, size_t len) 439 { 440 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 441 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 442 return __builtin_memcmp(b1, b2, len); 443 } 444 445 void * 446 kasan_memset(void *b, int c, size_t len) 447 { 448 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 449 return __builtin_memset(b, c, len); 450 } 451 452 void * 453 kasan_memmove(void *dst, const void *src, size_t len) 454 { 455 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 456 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 457 return __builtin_memmove(dst, src, len); 458 } 459 460 char * 461 kasan_strcpy(char *dst, const char *src) 462 { 463 char *save = dst; 464 465 while (1) { 466 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 467 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 468 *dst = *src; 469 if (*src == '\0') 470 break; 471 src++, dst++; 472 } 473 474 return save; 475 } 476 477 int 478 kasan_strcmp(const char *s1, const char *s2) 479 { 480 while (1) { 481 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 482 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 483 if (*s1 != *s2) 484 break; 485 if (*s1 == '\0') 486 return 0; 487 s1++, s2++; 488 } 489 490 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 491 } 492 493 size_t 494 kasan_strlen(const char *str) 495 { 496 const char *s; 497 498 s = str; 499 while (1) { 500 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 501 if (*s == '\0') 502 break; 503 s++; 504 } 505 506 return (s - str); 507 } 508 509 char * 510 kasan_strcat(char *dst, const char *src) 511 { 512 size_t ldst, lsrc; 513 514 ldst = __builtin_strlen(dst); 515 lsrc = __builtin_strlen(src); 516 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true, 517 __RET_ADDR); 518 kasan_shadow_check((unsigned long)src, lsrc + 1, false, 519 __RET_ADDR); 520 521 return __builtin_strcat(dst, src); 522 } 523 524 char * 525 kasan_strchr(const char *s, int c) 526 { 527 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 528 __RET_ADDR); 529 return __builtin_strchr(s, c); 530 } 531 532 char * 533 kasan_strrchr(const char *s, int c) 534 { 535 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 536 __RET_ADDR); 537 return __builtin_strrchr(s, c); 538 } 539 540 #undef kcopy 541 #undef copyinstr 542 #undef copyoutstr 543 #undef copyin 544 545 int kasan_kcopy(const void *, void *, size_t); 546 int kasan_copyinstr(const void *, void *, size_t, size_t *); 547 int kasan_copyoutstr(const void *, void *, size_t, size_t *); 548 int kasan_copyin(const void *, void *, size_t); 549 int kcopy(const void *, void *, size_t); 550 int copyinstr(const void *, void *, size_t, size_t *); 551 int copyoutstr(const void *, void *, size_t, size_t *); 552 int copyin(const void *, void *, size_t); 553 554 int 555 kasan_kcopy(const void *src, void *dst, size_t len) 556 { 557 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 558 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 559 return kcopy(src, dst, len); 560 } 561 562 int 563 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 564 { 565 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 566 return copyin(uaddr, kaddr, len); 567 } 568 569 int 570 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 571 { 572 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 573 return copyinstr(uaddr, kaddr, len, done); 574 } 575 576 int 577 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 578 { 579 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 580 return copyoutstr(kaddr, uaddr, len, done); 581 } 582 583 /* -------------------------------------------------------------------------- */ 584 585 #undef _ucas_32 586 #undef _ucas_32_mp 587 #undef _ucas_64 588 #undef _ucas_64_mp 589 #undef _ufetch_8 590 #undef _ufetch_16 591 #undef _ufetch_32 592 #undef _ufetch_64 593 594 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 595 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 596 int 597 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 598 uint32_t *ret) 599 { 600 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 601 __RET_ADDR); 602 return _ucas_32(uaddr, old, new, ret); 603 } 604 605 #ifdef __HAVE_UCAS_MP 606 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 607 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 608 int 609 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 610 uint32_t *ret) 611 { 612 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 613 __RET_ADDR); 614 return _ucas_32_mp(uaddr, old, new, ret); 615 } 616 #endif 617 618 #ifdef _LP64 619 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 620 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 621 int 622 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 623 uint64_t *ret) 624 { 625 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 626 __RET_ADDR); 627 return _ucas_64(uaddr, old, new, ret); 628 } 629 630 #ifdef __HAVE_UCAS_MP 631 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 632 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 633 int 634 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 635 uint64_t *ret) 636 { 637 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 638 __RET_ADDR); 639 return _ucas_64_mp(uaddr, old, new, ret); 640 } 641 #endif 642 #endif 643 644 int _ufetch_8(const uint8_t *, uint8_t *); 645 int kasan__ufetch_8(const uint8_t *, uint8_t *); 646 int 647 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 648 { 649 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 650 __RET_ADDR); 651 return _ufetch_8(uaddr, valp); 652 } 653 654 int _ufetch_16(const uint16_t *, uint16_t *); 655 int kasan__ufetch_16(const uint16_t *, uint16_t *); 656 int 657 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 658 { 659 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 660 __RET_ADDR); 661 return _ufetch_16(uaddr, valp); 662 } 663 664 int _ufetch_32(const uint32_t *, uint32_t *); 665 int kasan__ufetch_32(const uint32_t *, uint32_t *); 666 int 667 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 668 { 669 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 670 __RET_ADDR); 671 return _ufetch_32(uaddr, valp); 672 } 673 674 #ifdef _LP64 675 int _ufetch_64(const uint64_t *, uint64_t *); 676 int kasan__ufetch_64(const uint64_t *, uint64_t *); 677 int 678 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 679 { 680 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 681 __RET_ADDR); 682 return _ufetch_64(uaddr, valp); 683 } 684 #endif 685 686 /* -------------------------------------------------------------------------- */ 687 688 #undef atomic_add_32 689 #undef atomic_add_int 690 #undef atomic_add_long 691 #undef atomic_add_ptr 692 #undef atomic_add_64 693 #undef atomic_add_32_nv 694 #undef atomic_add_int_nv 695 #undef atomic_add_long_nv 696 #undef atomic_add_ptr_nv 697 #undef atomic_add_64_nv 698 #undef atomic_and_32 699 #undef atomic_and_uint 700 #undef atomic_and_ulong 701 #undef atomic_and_64 702 #undef atomic_and_32_nv 703 #undef atomic_and_uint_nv 704 #undef atomic_and_ulong_nv 705 #undef atomic_and_64_nv 706 #undef atomic_or_32 707 #undef atomic_or_uint 708 #undef atomic_or_ulong 709 #undef atomic_or_64 710 #undef atomic_or_32_nv 711 #undef atomic_or_uint_nv 712 #undef atomic_or_ulong_nv 713 #undef atomic_or_64_nv 714 #undef atomic_cas_32 715 #undef atomic_cas_uint 716 #undef atomic_cas_ulong 717 #undef atomic_cas_ptr 718 #undef atomic_cas_64 719 #undef atomic_cas_32_ni 720 #undef atomic_cas_uint_ni 721 #undef atomic_cas_ulong_ni 722 #undef atomic_cas_ptr_ni 723 #undef atomic_cas_64_ni 724 #undef atomic_swap_32 725 #undef atomic_swap_uint 726 #undef atomic_swap_ulong 727 #undef atomic_swap_ptr 728 #undef atomic_swap_64 729 #undef atomic_dec_32 730 #undef atomic_dec_uint 731 #undef atomic_dec_ulong 732 #undef atomic_dec_ptr 733 #undef atomic_dec_64 734 #undef atomic_dec_32_nv 735 #undef atomic_dec_uint_nv 736 #undef atomic_dec_ulong_nv 737 #undef atomic_dec_ptr_nv 738 #undef atomic_dec_64_nv 739 #undef atomic_inc_32 740 #undef atomic_inc_uint 741 #undef atomic_inc_ulong 742 #undef atomic_inc_ptr 743 #undef atomic_inc_64 744 #undef atomic_inc_32_nv 745 #undef atomic_inc_uint_nv 746 #undef atomic_inc_ulong_nv 747 #undef atomic_inc_ptr_nv 748 #undef atomic_inc_64_nv 749 750 #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 751 void atomic_add_##name(volatile targ1 *, targ2); \ 752 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 753 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 754 { \ 755 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 756 __RET_ADDR); \ 757 atomic_add_##name(ptr, val); \ 758 } \ 759 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 760 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 761 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 762 { \ 763 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 764 __RET_ADDR); \ 765 return atomic_add_##name##_nv(ptr, val); \ 766 } 767 768 #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 769 void atomic_and_##name(volatile targ1 *, targ2); \ 770 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 771 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 772 { \ 773 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 774 __RET_ADDR); \ 775 atomic_and_##name(ptr, val); \ 776 } \ 777 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 778 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 779 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 780 { \ 781 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 782 __RET_ADDR); \ 783 return atomic_and_##name##_nv(ptr, val); \ 784 } 785 786 #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 787 void atomic_or_##name(volatile targ1 *, targ2); \ 788 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 789 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 790 { \ 791 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 792 __RET_ADDR); \ 793 atomic_or_##name(ptr, val); \ 794 } \ 795 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 796 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 797 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 798 { \ 799 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 800 __RET_ADDR); \ 801 return atomic_or_##name##_nv(ptr, val); \ 802 } 803 804 #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 805 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 806 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 807 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 808 { \ 809 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 810 __RET_ADDR); \ 811 return atomic_cas_##name(ptr, exp, new); \ 812 } \ 813 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 814 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 815 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 816 { \ 817 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 818 __RET_ADDR); \ 819 return atomic_cas_##name##_ni(ptr, exp, new); \ 820 } 821 822 #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 823 tret atomic_swap_##name(volatile targ1 *, targ2); \ 824 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \ 825 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 826 { \ 827 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 828 __RET_ADDR); \ 829 return atomic_swap_##name(ptr, val); \ 830 } 831 832 #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 833 void atomic_dec_##name(volatile targ1 *); \ 834 void kasan_atomic_dec_##name(volatile targ1 *); \ 835 void kasan_atomic_dec_##name(volatile targ1 *ptr) \ 836 { \ 837 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 838 __RET_ADDR); \ 839 atomic_dec_##name(ptr); \ 840 } \ 841 tret atomic_dec_##name##_nv(volatile targ1 *); \ 842 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \ 843 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 844 { \ 845 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 846 __RET_ADDR); \ 847 return atomic_dec_##name##_nv(ptr); \ 848 } 849 850 #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 851 void atomic_inc_##name(volatile targ1 *); \ 852 void kasan_atomic_inc_##name(volatile targ1 *); \ 853 void kasan_atomic_inc_##name(volatile targ1 *ptr) \ 854 { \ 855 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 856 __RET_ADDR); \ 857 atomic_inc_##name(ptr); \ 858 } \ 859 tret atomic_inc_##name##_nv(volatile targ1 *); \ 860 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \ 861 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 862 { \ 863 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 864 __RET_ADDR); \ 865 return atomic_inc_##name##_nv(ptr); \ 866 } 867 868 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 869 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 870 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 871 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 872 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 873 874 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 875 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 876 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 877 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 878 879 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 880 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 881 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 882 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 883 884 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 885 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 886 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 887 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 888 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 889 890 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 891 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 892 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 893 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 894 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 895 896 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 897 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 898 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 899 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 900 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void); 901 902 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 903 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 904 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 905 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 906 ASAN_ATOMIC_FUNC_INC(ptr, void *, void); 907 908 /* -------------------------------------------------------------------------- */ 909 910 #ifdef __HAVE_KASAN_INSTR_BUS 911 912 #include <sys/bus.h> 913 914 #undef bus_space_read_multi_1 915 #undef bus_space_read_multi_2 916 #undef bus_space_read_multi_4 917 #undef bus_space_read_multi_8 918 #undef bus_space_read_multi_stream_1 919 #undef bus_space_read_multi_stream_2 920 #undef bus_space_read_multi_stream_4 921 #undef bus_space_read_multi_stream_8 922 #undef bus_space_read_region_1 923 #undef bus_space_read_region_2 924 #undef bus_space_read_region_4 925 #undef bus_space_read_region_8 926 #undef bus_space_read_region_stream_1 927 #undef bus_space_read_region_stream_2 928 #undef bus_space_read_region_stream_4 929 #undef bus_space_read_region_stream_8 930 #undef bus_space_write_multi_1 931 #undef bus_space_write_multi_2 932 #undef bus_space_write_multi_4 933 #undef bus_space_write_multi_8 934 #undef bus_space_write_multi_stream_1 935 #undef bus_space_write_multi_stream_2 936 #undef bus_space_write_multi_stream_4 937 #undef bus_space_write_multi_stream_8 938 #undef bus_space_write_region_1 939 #undef bus_space_write_region_2 940 #undef bus_space_write_region_4 941 #undef bus_space_write_region_8 942 #undef bus_space_write_region_stream_1 943 #undef bus_space_write_region_stream_2 944 #undef bus_space_write_region_stream_4 945 #undef bus_space_write_region_stream_8 946 947 #define ASAN_BUS_READ_FUNC(bytes, bits) \ 948 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 949 bus_size_t, uint##bits##_t *, bus_size_t); \ 950 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 951 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 952 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 953 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 954 bus_size_t count) \ 955 { \ 956 kasan_shadow_check((uintptr_t)buf, \ 957 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 958 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 959 } \ 960 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 961 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 962 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 963 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 964 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 965 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 966 bus_size_t count) \ 967 { \ 968 kasan_shadow_check((uintptr_t)buf, \ 969 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 970 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 971 } \ 972 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 973 bus_size_t, uint##bits##_t *, bus_size_t); \ 974 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \ 975 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 976 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 977 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 978 bus_size_t count) \ 979 { \ 980 kasan_shadow_check((uintptr_t)buf, \ 981 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 982 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 983 } \ 984 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 985 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 986 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 987 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 988 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 989 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 990 bus_size_t count) \ 991 { \ 992 kasan_shadow_check((uintptr_t)buf, \ 993 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 994 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 995 } 996 997 #define ASAN_BUS_WRITE_FUNC(bytes, bits) \ 998 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 999 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1000 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1001 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1002 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1003 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1004 bus_size_t count) \ 1005 { \ 1006 kasan_shadow_check((uintptr_t)buf, \ 1007 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1008 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1009 } \ 1010 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1011 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1012 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1013 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1014 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1015 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1016 bus_size_t count) \ 1017 { \ 1018 kasan_shadow_check((uintptr_t)buf, \ 1019 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1020 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1021 } \ 1022 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1023 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1024 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1025 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1026 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1027 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1028 bus_size_t count) \ 1029 { \ 1030 kasan_shadow_check((uintptr_t)buf, \ 1031 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1032 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1033 } \ 1034 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1035 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1036 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1037 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1038 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1039 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1040 bus_size_t count) \ 1041 { \ 1042 kasan_shadow_check((uintptr_t)buf, \ 1043 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1044 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1045 } 1046 1047 ASAN_BUS_READ_FUNC(1, 8) 1048 ASAN_BUS_READ_FUNC(2, 16) 1049 ASAN_BUS_READ_FUNC(4, 32) 1050 ASAN_BUS_READ_FUNC(8, 64) 1051 1052 ASAN_BUS_WRITE_FUNC(1, 8) 1053 ASAN_BUS_WRITE_FUNC(2, 16) 1054 ASAN_BUS_WRITE_FUNC(4, 32) 1055 ASAN_BUS_WRITE_FUNC(8, 64) 1056 1057 #endif /* __HAVE_KASAN_INSTR_BUS */ 1058 1059 /* -------------------------------------------------------------------------- */ 1060 1061 #include <sys/mbuf.h> 1062 1063 static void 1064 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1065 bool write, uintptr_t pc) 1066 { 1067 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc); 1068 } 1069 1070 static void 1071 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1072 bool write, uintptr_t pc) 1073 { 1074 bus_addr_t minlen; 1075 1076 for (; m != NULL && len != 0; m = m->m_next) { 1077 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc); 1078 1079 if (offset >= m->m_len) { 1080 offset -= m->m_len; 1081 continue; 1082 } 1083 1084 minlen = MIN(len, m->m_len - offset); 1085 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset), 1086 minlen, write, pc); 1087 1088 offset = 0; 1089 len -= minlen; 1090 } 1091 } 1092 1093 static void 1094 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1095 bool write, uintptr_t pc) 1096 { 1097 bus_size_t minlen, resid; 1098 struct iovec *iov; 1099 int i; 1100 1101 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc); 1102 1103 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1104 return; 1105 1106 resid = uio->uio_resid; 1107 iov = uio->uio_iov; 1108 1109 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1110 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]), 1111 false, pc); 1112 minlen = MIN(resid, iov[i].iov_len); 1113 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen, 1114 write, pc); 1115 resid -= minlen; 1116 } 1117 } 1118 1119 void 1120 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1121 { 1122 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0; 1123 1124 switch (map->dm_buftype) { 1125 case KASAN_DMA_LINEAR: 1126 kasan_dma_sync_linear(map->dm_buf, offset, len, write, 1127 __RET_ADDR); 1128 break; 1129 case KASAN_DMA_MBUF: 1130 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write, 1131 __RET_ADDR); 1132 break; 1133 case KASAN_DMA_UIO: 1134 kasan_dma_sync_uio(map->dm_buf, offset, len, write, 1135 __RET_ADDR); 1136 break; 1137 case KASAN_DMA_RAW: 1138 break; 1139 default: 1140 panic("%s: impossible", __func__); 1141 } 1142 } 1143 1144 void 1145 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1146 { 1147 map->dm_buf = buf; 1148 map->dm_buflen = buflen; 1149 map->dm_buftype = type; 1150 } 1151 1152 /* -------------------------------------------------------------------------- */ 1153 1154 void __asan_register_globals(struct __asan_global *, size_t); 1155 void __asan_unregister_globals(struct __asan_global *, size_t); 1156 1157 void 1158 __asan_register_globals(struct __asan_global *globals, size_t n) 1159 { 1160 size_t i; 1161 1162 for (i = 0; i < n; i++) { 1163 kasan_mark(globals[i].beg, globals[i].size, 1164 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 1165 } 1166 } 1167 1168 void 1169 __asan_unregister_globals(struct __asan_global *globals, size_t n) 1170 { 1171 /* never called */ 1172 } 1173 1174 #define ASAN_LOAD_STORE(size) \ 1175 void __asan_load##size(unsigned long); \ 1176 void __asan_load##size(unsigned long addr) \ 1177 { \ 1178 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1179 } \ 1180 void __asan_load##size##_noabort(unsigned long); \ 1181 void __asan_load##size##_noabort(unsigned long addr) \ 1182 { \ 1183 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1184 } \ 1185 void __asan_store##size(unsigned long); \ 1186 void __asan_store##size(unsigned long addr) \ 1187 { \ 1188 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1189 } \ 1190 void __asan_store##size##_noabort(unsigned long); \ 1191 void __asan_store##size##_noabort(unsigned long addr) \ 1192 { \ 1193 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1194 } 1195 1196 ASAN_LOAD_STORE(1); 1197 ASAN_LOAD_STORE(2); 1198 ASAN_LOAD_STORE(4); 1199 ASAN_LOAD_STORE(8); 1200 ASAN_LOAD_STORE(16); 1201 1202 void __asan_loadN(unsigned long, size_t); 1203 void __asan_loadN_noabort(unsigned long, size_t); 1204 void __asan_storeN(unsigned long, size_t); 1205 void __asan_storeN_noabort(unsigned long, size_t); 1206 void __asan_handle_no_return(void); 1207 1208 void 1209 __asan_loadN(unsigned long addr, size_t size) 1210 { 1211 kasan_shadow_check(addr, size, false, __RET_ADDR); 1212 } 1213 1214 void 1215 __asan_loadN_noabort(unsigned long addr, size_t size) 1216 { 1217 kasan_shadow_check(addr, size, false, __RET_ADDR); 1218 } 1219 1220 void 1221 __asan_storeN(unsigned long addr, size_t size) 1222 { 1223 kasan_shadow_check(addr, size, true, __RET_ADDR); 1224 } 1225 1226 void 1227 __asan_storeN_noabort(unsigned long addr, size_t size) 1228 { 1229 kasan_shadow_check(addr, size, true, __RET_ADDR); 1230 } 1231 1232 void 1233 __asan_handle_no_return(void) 1234 { 1235 /* nothing */ 1236 } 1237 1238 #define ASAN_SET_SHADOW(byte) \ 1239 void __asan_set_shadow_##byte(void *, size_t); \ 1240 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1241 { \ 1242 __builtin_memset((void *)addr, 0x##byte, size); \ 1243 } 1244 1245 ASAN_SET_SHADOW(00); 1246 ASAN_SET_SHADOW(f1); 1247 ASAN_SET_SHADOW(f2); 1248 ASAN_SET_SHADOW(f3); 1249 ASAN_SET_SHADOW(f5); 1250 ASAN_SET_SHADOW(f8); 1251 1252 void __asan_poison_stack_memory(const void *, size_t); 1253 void __asan_unpoison_stack_memory(const void *, size_t); 1254 1255 void 1256 __asan_poison_stack_memory(const void *addr, size_t size) 1257 { 1258 size = roundup(size, KASAN_SHADOW_SCALE_SIZE); 1259 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1260 } 1261 1262 void 1263 __asan_unpoison_stack_memory(const void *addr, size_t size) 1264 { 1265 kasan_shadow_Nbyte_markvalid(addr, size); 1266 } 1267 1268 void __asan_alloca_poison(const void *, size_t); 1269 void __asan_allocas_unpoison(const void *, const void *); 1270 1271 void __asan_alloca_poison(const void *addr, size_t size) 1272 { 1273 const void *l, *r; 1274 1275 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0); 1276 1277 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1278 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1279 1280 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1281 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1282 KASAN_STACK_MID); 1283 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1284 } 1285 1286 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1287 { 1288 size_t size; 1289 1290 if (__predict_false(!stkbegin)) 1291 return; 1292 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1293 return; 1294 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1295 1296 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1297 } 1298
Indexes created Mon Sep 22 21:09:42 GMT 2025