subr_asan.c revision 1.9.2.3
1 /* $NetBSD: subr_asan.c,v 1.9.2.3 2020/04/13 08:05:04 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.9.2.3 2020/04/13 08:05:04 martin Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/asan.h> 43 44 #include <uvm/uvm.h> 45 46 #ifdef KASAN_PANIC 47 #define REPORT panic 48 #else 49 #define REPORT printf 50 #endif 51 52 /* ASAN constants. Part of the compiler ABI. */ 53 #define KASAN_SHADOW_SCALE_SHIFT 3 54 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 55 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1) 56 #define KASAN_ALLOCA_SCALE_SIZE 32 57 58 /* The MD code. */ 59 #include <machine/asan.h> 60 61 /* ASAN ABI version. */ 62 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 63 #define ASAN_ABI_VERSION 8 64 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 65 #define ASAN_ABI_VERSION 8 66 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 67 #define ASAN_ABI_VERSION 6 68 #else 69 #error "Unsupported compiler version" 70 #endif 71 72 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 73 74 /* Global variable descriptor. Part of the compiler ABI. */ 75 struct __asan_global_source_location { 76 const char *filename; 77 int line_no; 78 int column_no; 79 }; 80 struct __asan_global { 81 const void *beg; /* address of the global variable */ 82 size_t size; /* size of the global variable */ 83 size_t size_with_redzone; /* size with the redzone */ 84 const void *name; /* name of the variable */ 85 const void *module_name; /* name of the module where the var is declared */ 86 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 87 struct __asan_global_source_location *location; 88 #if ASAN_ABI_VERSION >= 7 89 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 90 #endif 91 }; 92 93 static bool kasan_enabled __read_mostly = false; 94 95 /* -------------------------------------------------------------------------- */ 96 97 void 98 kasan_shadow_map(void *addr, size_t size) 99 { 100 size_t sz, npages, i; 101 vaddr_t sva, eva; 102 103 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 104 105 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE; 106 107 sva = (vaddr_t)kasan_md_addr_to_shad(addr); 108 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz; 109 110 sva = rounddown(sva, PAGE_SIZE); 111 eva = roundup(eva, PAGE_SIZE); 112 113 npages = (eva - sva) / PAGE_SIZE; 114 115 KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END); 116 117 for (i = 0; i < npages; i++) { 118 kasan_md_shadow_map_page(sva + i * PAGE_SIZE); 119 } 120 } 121 122 static void 123 kasan_ctors(void) 124 { 125 extern uint64_t __CTOR_LIST__, __CTOR_END__; 126 size_t nentries, i; 127 uint64_t *ptr; 128 129 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) / 130 sizeof(uintptr_t); 131 132 ptr = &__CTOR_LIST__; 133 for (i = 0; i < nentries; i++) { 134 void (*func)(void); 135 136 func = (void *)(*ptr); 137 (*func)(); 138 139 ptr++; 140 } 141 } 142 143 void 144 kasan_early_init(void *stack) 145 { 146 kasan_md_early_init(stack); 147 } 148 149 void 150 kasan_init(void) 151 { 152 /* MD initialization. */ 153 kasan_md_init(); 154 155 /* Now officially enabled. */ 156 kasan_enabled = true; 157 158 /* Call the ASAN constructors. */ 159 kasan_ctors(); 160 } 161 162 static inline const char * 163 kasan_code_name(uint8_t code) 164 { 165 switch (code) { 166 case KASAN_GENERIC_REDZONE: 167 return "GenericRedZone"; 168 case KASAN_MALLOC_REDZONE: 169 return "MallocRedZone"; 170 case KASAN_KMEM_REDZONE: 171 return "KmemRedZone"; 172 case KASAN_POOL_REDZONE: 173 return "PoolRedZone"; 174 case KASAN_POOL_FREED: 175 return "PoolUseAfterFree"; 176 case 1 ... 7: 177 return "RedZonePartial"; 178 case KASAN_STACK_LEFT: 179 return "StackLeft"; 180 case KASAN_STACK_MID: 181 return "StackMiddle"; 182 case KASAN_STACK_RIGHT: 183 return "StackRight"; 184 case KASAN_USE_AFTER_RET: 185 return "UseAfterRet"; 186 case KASAN_USE_AFTER_SCOPE: 187 return "UseAfterScope"; 188 default: 189 return "Unknown"; 190 } 191 } 192 193 static void 194 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 195 uint8_t code) 196 { 197 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s," 198 " %s]\n", 199 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""), 200 (write ? "write" : "read"), kasan_code_name(code)); 201 kasan_md_unwind(); 202 } 203 204 static __always_inline void 205 kasan_shadow_1byte_markvalid(unsigned long addr) 206 { 207 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 208 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 209 210 *byte = last; 211 } 212 213 static __always_inline void 214 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 215 { 216 size_t i; 217 218 for (i = 0; i < size; i++) { 219 kasan_shadow_1byte_markvalid((unsigned long)addr+i); 220 } 221 } 222 223 static __always_inline void 224 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 225 { 226 void *shad; 227 228 if (__predict_false(size == 0)) 229 return; 230 if (__predict_false(kasan_md_unsupported((vaddr_t)addr))) 231 return; 232 233 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 234 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0); 235 236 shad = (void *)kasan_md_addr_to_shad(addr); 237 size = size >> KASAN_SHADOW_SCALE_SHIFT; 238 239 __builtin_memset(shad, code, size); 240 } 241 242 void 243 kasan_add_redzone(size_t *size) 244 { 245 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE); 246 *size += KASAN_SHADOW_SCALE_SIZE; 247 } 248 249 void 250 kasan_softint(struct lwp *l) 251 { 252 const void *stk = (const void *)uvm_lwp_getuarea(l); 253 254 kasan_shadow_Nbyte_fill(stk, USPACE, 0); 255 } 256 257 /* 258 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 259 * and the rest as invalid. There are generally two use cases: 260 * 261 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 262 * the redzone at the end of the buffer as invalid. 263 * 264 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 265 */ 266 void 267 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code) 268 { 269 size_t i, n, redz; 270 int8_t *shad; 271 272 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 273 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE); 274 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0); 275 shad = kasan_md_addr_to_shad(addr); 276 277 /* Chunks of 8 bytes, valid. */ 278 n = size / KASAN_SHADOW_SCALE_SIZE; 279 for (i = 0; i < n; i++) { 280 *shad++ = 0; 281 } 282 283 /* Possibly one chunk, mid. */ 284 if ((size & KASAN_SHADOW_MASK) != 0) { 285 *shad++ = (size & KASAN_SHADOW_MASK); 286 } 287 288 /* Chunks of 8 bytes, invalid. */ 289 n = redz / KASAN_SHADOW_SCALE_SIZE; 290 for (i = 0; i < n; i++) { 291 *shad++ = code; 292 } 293 } 294 295 /* -------------------------------------------------------------------------- */ 296 297 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 298 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 299 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 300 301 static __always_inline bool 302 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 303 { 304 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 305 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 306 307 if (__predict_true(*byte == 0 || last <= *byte)) { 308 return true; 309 } 310 *code = *byte; 311 return false; 312 } 313 314 static __always_inline bool 315 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 316 { 317 int8_t *byte, last; 318 319 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 320 return (kasan_shadow_1byte_isvalid(addr, code) && 321 kasan_shadow_1byte_isvalid(addr+1, code)); 322 } 323 324 byte = kasan_md_addr_to_shad((void *)addr); 325 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 326 327 if (__predict_true(*byte == 0 || last <= *byte)) { 328 return true; 329 } 330 *code = *byte; 331 return false; 332 } 333 334 static __always_inline bool 335 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 336 { 337 int8_t *byte, last; 338 339 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 340 return (kasan_shadow_2byte_isvalid(addr, code) && 341 kasan_shadow_2byte_isvalid(addr+2, code)); 342 } 343 344 byte = kasan_md_addr_to_shad((void *)addr); 345 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 346 347 if (__predict_true(*byte == 0 || last <= *byte)) { 348 return true; 349 } 350 *code = *byte; 351 return false; 352 } 353 354 static __always_inline bool 355 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 356 { 357 int8_t *byte, last; 358 359 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 360 return (kasan_shadow_4byte_isvalid(addr, code) && 361 kasan_shadow_4byte_isvalid(addr+4, code)); 362 } 363 364 byte = kasan_md_addr_to_shad((void *)addr); 365 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 366 367 if (__predict_true(*byte == 0 || last <= *byte)) { 368 return true; 369 } 370 *code = *byte; 371 return false; 372 } 373 374 static __always_inline bool 375 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 376 { 377 size_t i; 378 379 for (i = 0; i < size; i++) { 380 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 381 return false; 382 } 383 384 return true; 385 } 386 387 static __always_inline void 388 kasan_shadow_check(unsigned long addr, size_t size, bool write, 389 unsigned long retaddr) 390 { 391 uint8_t code; 392 bool valid; 393 394 if (__predict_false(!kasan_enabled)) 395 return; 396 if (__predict_false(size == 0)) 397 return; 398 if (__predict_false(kasan_md_unsupported(addr))) 399 return; 400 401 if (__builtin_constant_p(size)) { 402 switch (size) { 403 case 1: 404 valid = kasan_shadow_1byte_isvalid(addr, &code); 405 break; 406 case 2: 407 valid = kasan_shadow_2byte_isvalid(addr, &code); 408 break; 409 case 4: 410 valid = kasan_shadow_4byte_isvalid(addr, &code); 411 break; 412 case 8: 413 valid = kasan_shadow_8byte_isvalid(addr, &code); 414 break; 415 default: 416 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 417 break; 418 } 419 } else { 420 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 421 } 422 423 if (__predict_false(!valid)) { 424 kasan_report(addr, size, write, retaddr, code); 425 } 426 } 427 428 /* -------------------------------------------------------------------------- */ 429 430 void * 431 kasan_memcpy(void *dst, const void *src, size_t len) 432 { 433 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 434 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 435 return __builtin_memcpy(dst, src, len); 436 } 437 438 int 439 kasan_memcmp(const void *b1, const void *b2, size_t len) 440 { 441 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 442 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 443 return __builtin_memcmp(b1, b2, len); 444 } 445 446 void * 447 kasan_memset(void *b, int c, size_t len) 448 { 449 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 450 return __builtin_memset(b, c, len); 451 } 452 453 void * 454 kasan_memmove(void *dst, const void *src, size_t len) 455 { 456 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 457 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 458 return __builtin_memmove(dst, src, len); 459 } 460 461 char * 462 kasan_strcpy(char *dst, const char *src) 463 { 464 char *save = dst; 465 466 while (1) { 467 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 468 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 469 *dst = *src; 470 if (*src == '\0') 471 break; 472 src++, dst++; 473 } 474 475 return save; 476 } 477 478 int 479 kasan_strcmp(const char *s1, const char *s2) 480 { 481 while (1) { 482 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 483 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 484 if (*s1 != *s2) 485 break; 486 if (*s1 == '\0') 487 return 0; 488 s1++, s2++; 489 } 490 491 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 492 } 493 494 size_t 495 kasan_strlen(const char *str) 496 { 497 const char *s; 498 499 s = str; 500 while (1) { 501 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 502 if (*s == '\0') 503 break; 504 s++; 505 } 506 507 return (s - str); 508 } 509 510 char * 511 kasan_strcat(char *dst, const char *src) 512 { 513 size_t ldst, lsrc; 514 515 ldst = __builtin_strlen(dst); 516 lsrc = __builtin_strlen(src); 517 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true, 518 __RET_ADDR); 519 kasan_shadow_check((unsigned long)src, lsrc + 1, false, 520 __RET_ADDR); 521 522 return __builtin_strcat(dst, src); 523 } 524 525 char * 526 kasan_strchr(const char *s, int c) 527 { 528 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 529 __RET_ADDR); 530 return __builtin_strchr(s, c); 531 } 532 533 char * 534 kasan_strrchr(const char *s, int c) 535 { 536 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 537 __RET_ADDR); 538 return __builtin_strrchr(s, c); 539 } 540 541 #undef kcopy 542 #undef copystr 543 #undef copyinstr 544 #undef copyoutstr 545 #undef copyin 546 547 int kasan_kcopy(const void *, void *, size_t); 548 int kasan_copystr(const void *, void *, size_t, size_t *); 549 int kasan_copyinstr(const void *, void *, size_t, size_t *); 550 int kasan_copyoutstr(const void *, void *, size_t, size_t *); 551 int kasan_copyin(const void *, void *, size_t); 552 int kcopy(const void *, void *, size_t); 553 int copystr(const void *, void *, size_t, size_t *); 554 int copyinstr(const void *, void *, size_t, size_t *); 555 int copyoutstr(const void *, void *, size_t, size_t *); 556 int copyin(const void *, void *, size_t); 557 558 int 559 kasan_kcopy(const void *src, void *dst, size_t len) 560 { 561 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 562 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 563 return kcopy(src, dst, len); 564 } 565 566 int 567 kasan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 568 { 569 kasan_shadow_check((unsigned long)kdaddr, len, true, __RET_ADDR); 570 return copystr(kfaddr, kdaddr, len, done); 571 } 572 573 int 574 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 575 { 576 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 577 return copyin(uaddr, kaddr, len); 578 } 579 580 int 581 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 582 { 583 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 584 return copyinstr(uaddr, kaddr, len, done); 585 } 586 587 int 588 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 589 { 590 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 591 return copyoutstr(kaddr, uaddr, len, done); 592 } 593 594 /* -------------------------------------------------------------------------- */ 595 596 #undef _ucas_32 597 #undef _ucas_32_mp 598 #undef _ucas_64 599 #undef _ucas_64_mp 600 #undef _ufetch_8 601 #undef _ufetch_16 602 #undef _ufetch_32 603 #undef _ufetch_64 604 605 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 606 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 607 int 608 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 609 uint32_t *ret) 610 { 611 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 612 __RET_ADDR); 613 return _ucas_32(uaddr, old, new, ret); 614 } 615 616 #ifdef __HAVE_UCAS_MP 617 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 618 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 619 int 620 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 621 uint32_t *ret) 622 { 623 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 624 __RET_ADDR); 625 return _ucas_32_mp(uaddr, old, new, ret); 626 } 627 #endif 628 629 #ifdef _LP64 630 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 631 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 632 int 633 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 634 uint64_t *ret) 635 { 636 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 637 __RET_ADDR); 638 return _ucas_64(uaddr, old, new, ret); 639 } 640 641 #ifdef __HAVE_UCAS_MP 642 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 643 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 644 int 645 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 646 uint64_t *ret) 647 { 648 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 649 __RET_ADDR); 650 return _ucas_64_mp(uaddr, old, new, ret); 651 } 652 #endif 653 #endif 654 655 int _ufetch_8(const uint8_t *, uint8_t *); 656 int kasan__ufetch_8(const uint8_t *, uint8_t *); 657 int 658 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 659 { 660 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 661 __RET_ADDR); 662 return _ufetch_8(uaddr, valp); 663 } 664 665 int _ufetch_16(const uint16_t *, uint16_t *); 666 int kasan__ufetch_16(const uint16_t *, uint16_t *); 667 int 668 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 669 { 670 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 671 __RET_ADDR); 672 return _ufetch_16(uaddr, valp); 673 } 674 675 int _ufetch_32(const uint32_t *, uint32_t *); 676 int kasan__ufetch_32(const uint32_t *, uint32_t *); 677 int 678 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 679 { 680 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 681 __RET_ADDR); 682 return _ufetch_32(uaddr, valp); 683 } 684 685 #ifdef _LP64 686 int _ufetch_64(const uint64_t *, uint64_t *); 687 int kasan__ufetch_64(const uint64_t *, uint64_t *); 688 int 689 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 690 { 691 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 692 __RET_ADDR); 693 return _ufetch_64(uaddr, valp); 694 } 695 #endif 696 697 /* -------------------------------------------------------------------------- */ 698 699 #undef atomic_add_32 700 #undef atomic_add_int 701 #undef atomic_add_long 702 #undef atomic_add_ptr 703 #undef atomic_add_64 704 #undef atomic_add_32_nv 705 #undef atomic_add_int_nv 706 #undef atomic_add_long_nv 707 #undef atomic_add_ptr_nv 708 #undef atomic_add_64_nv 709 #undef atomic_and_32 710 #undef atomic_and_uint 711 #undef atomic_and_ulong 712 #undef atomic_and_64 713 #undef atomic_and_32_nv 714 #undef atomic_and_uint_nv 715 #undef atomic_and_ulong_nv 716 #undef atomic_and_64_nv 717 #undef atomic_or_32 718 #undef atomic_or_uint 719 #undef atomic_or_ulong 720 #undef atomic_or_64 721 #undef atomic_or_32_nv 722 #undef atomic_or_uint_nv 723 #undef atomic_or_ulong_nv 724 #undef atomic_or_64_nv 725 #undef atomic_cas_32 726 #undef atomic_cas_uint 727 #undef atomic_cas_ulong 728 #undef atomic_cas_ptr 729 #undef atomic_cas_64 730 #undef atomic_cas_32_ni 731 #undef atomic_cas_uint_ni 732 #undef atomic_cas_ulong_ni 733 #undef atomic_cas_ptr_ni 734 #undef atomic_cas_64_ni 735 #undef atomic_swap_32 736 #undef atomic_swap_uint 737 #undef atomic_swap_ulong 738 #undef atomic_swap_ptr 739 #undef atomic_swap_64 740 #undef atomic_dec_32 741 #undef atomic_dec_uint 742 #undef atomic_dec_ulong 743 #undef atomic_dec_ptr 744 #undef atomic_dec_64 745 #undef atomic_dec_32_nv 746 #undef atomic_dec_uint_nv 747 #undef atomic_dec_ulong_nv 748 #undef atomic_dec_ptr_nv 749 #undef atomic_dec_64_nv 750 #undef atomic_inc_32 751 #undef atomic_inc_uint 752 #undef atomic_inc_ulong 753 #undef atomic_inc_ptr 754 #undef atomic_inc_64 755 #undef atomic_inc_32_nv 756 #undef atomic_inc_uint_nv 757 #undef atomic_inc_ulong_nv 758 #undef atomic_inc_ptr_nv 759 #undef atomic_inc_64_nv 760 761 #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 762 void atomic_add_##name(volatile targ1 *, targ2); \ 763 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 764 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 765 { \ 766 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 767 __RET_ADDR); \ 768 atomic_add_##name(ptr, val); \ 769 } \ 770 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 771 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 772 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 773 { \ 774 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 775 __RET_ADDR); \ 776 return atomic_add_##name##_nv(ptr, val); \ 777 } 778 779 #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 780 void atomic_and_##name(volatile targ1 *, targ2); \ 781 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 782 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 783 { \ 784 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 785 __RET_ADDR); \ 786 atomic_and_##name(ptr, val); \ 787 } \ 788 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 789 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 790 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 791 { \ 792 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 793 __RET_ADDR); \ 794 return atomic_and_##name##_nv(ptr, val); \ 795 } 796 797 #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 798 void atomic_or_##name(volatile targ1 *, targ2); \ 799 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 800 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 801 { \ 802 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 803 __RET_ADDR); \ 804 atomic_or_##name(ptr, val); \ 805 } \ 806 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 807 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 808 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 809 { \ 810 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 811 __RET_ADDR); \ 812 return atomic_or_##name##_nv(ptr, val); \ 813 } 814 815 #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 816 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 817 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 818 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 819 { \ 820 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 821 __RET_ADDR); \ 822 return atomic_cas_##name(ptr, exp, new); \ 823 } \ 824 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 825 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 826 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 827 { \ 828 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 829 __RET_ADDR); \ 830 return atomic_cas_##name##_ni(ptr, exp, new); \ 831 } 832 833 #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 834 tret atomic_swap_##name(volatile targ1 *, targ2); \ 835 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \ 836 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 837 { \ 838 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 839 __RET_ADDR); \ 840 return atomic_swap_##name(ptr, val); \ 841 } 842 843 #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 844 void atomic_dec_##name(volatile targ1 *); \ 845 void kasan_atomic_dec_##name(volatile targ1 *); \ 846 void kasan_atomic_dec_##name(volatile targ1 *ptr) \ 847 { \ 848 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 849 __RET_ADDR); \ 850 atomic_dec_##name(ptr); \ 851 } \ 852 tret atomic_dec_##name##_nv(volatile targ1 *); \ 853 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \ 854 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 855 { \ 856 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 857 __RET_ADDR); \ 858 return atomic_dec_##name##_nv(ptr); \ 859 } 860 861 #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 862 void atomic_inc_##name(volatile targ1 *); \ 863 void kasan_atomic_inc_##name(volatile targ1 *); \ 864 void kasan_atomic_inc_##name(volatile targ1 *ptr) \ 865 { \ 866 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 867 __RET_ADDR); \ 868 atomic_inc_##name(ptr); \ 869 } \ 870 tret atomic_inc_##name##_nv(volatile targ1 *); \ 871 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \ 872 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 873 { \ 874 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 875 __RET_ADDR); \ 876 return atomic_inc_##name##_nv(ptr); \ 877 } 878 879 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 880 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 881 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 882 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 883 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 884 885 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 886 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 887 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 888 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 889 890 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 891 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 892 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 893 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 894 895 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 896 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 897 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 898 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 899 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 900 901 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 902 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 903 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 904 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 905 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 906 907 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 908 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 909 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 910 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 911 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void); 912 913 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 914 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 915 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 916 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 917 ASAN_ATOMIC_FUNC_INC(ptr, void *, void); 918 919 /* -------------------------------------------------------------------------- */ 920 921 #ifdef __HAVE_KASAN_INSTR_BUS 922 923 #include <sys/bus.h> 924 925 #undef bus_space_read_multi_1 926 #undef bus_space_read_multi_2 927 #undef bus_space_read_multi_4 928 #undef bus_space_read_multi_8 929 #undef bus_space_read_multi_stream_1 930 #undef bus_space_read_multi_stream_2 931 #undef bus_space_read_multi_stream_4 932 #undef bus_space_read_multi_stream_8 933 #undef bus_space_read_region_1 934 #undef bus_space_read_region_2 935 #undef bus_space_read_region_4 936 #undef bus_space_read_region_8 937 #undef bus_space_read_region_stream_1 938 #undef bus_space_read_region_stream_2 939 #undef bus_space_read_region_stream_4 940 #undef bus_space_read_region_stream_8 941 #undef bus_space_write_multi_1 942 #undef bus_space_write_multi_2 943 #undef bus_space_write_multi_4 944 #undef bus_space_write_multi_8 945 #undef bus_space_write_multi_stream_1 946 #undef bus_space_write_multi_stream_2 947 #undef bus_space_write_multi_stream_4 948 #undef bus_space_write_multi_stream_8 949 #undef bus_space_write_region_1 950 #undef bus_space_write_region_2 951 #undef bus_space_write_region_4 952 #undef bus_space_write_region_8 953 #undef bus_space_write_region_stream_1 954 #undef bus_space_write_region_stream_2 955 #undef bus_space_write_region_stream_4 956 #undef bus_space_write_region_stream_8 957 958 #define ASAN_BUS_READ_FUNC(bytes, bits) \ 959 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 960 bus_size_t, uint##bits##_t *, bus_size_t); \ 961 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 962 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 963 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 964 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 965 bus_size_t count) \ 966 { \ 967 kasan_shadow_check((uintptr_t)buf, \ 968 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 969 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 970 } \ 971 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 972 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 973 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 974 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 975 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 976 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 977 bus_size_t count) \ 978 { \ 979 kasan_shadow_check((uintptr_t)buf, \ 980 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 981 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 982 } \ 983 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 984 bus_size_t, uint##bits##_t *, bus_size_t); \ 985 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \ 986 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 987 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 988 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 989 bus_size_t count) \ 990 { \ 991 kasan_shadow_check((uintptr_t)buf, \ 992 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 993 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 994 } \ 995 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 996 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 997 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 998 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 999 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 1000 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 1001 bus_size_t count) \ 1002 { \ 1003 kasan_shadow_check((uintptr_t)buf, \ 1004 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 1005 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 1006 } 1007 1008 #define ASAN_BUS_WRITE_FUNC(bytes, bits) \ 1009 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 1010 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1011 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1012 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1013 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1014 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1015 bus_size_t count) \ 1016 { \ 1017 kasan_shadow_check((uintptr_t)buf, \ 1018 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1019 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1020 } \ 1021 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1022 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1023 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1024 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1025 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1026 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1027 bus_size_t count) \ 1028 { \ 1029 kasan_shadow_check((uintptr_t)buf, \ 1030 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1031 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1032 } \ 1033 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1034 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1035 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1036 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1037 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1038 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1039 bus_size_t count) \ 1040 { \ 1041 kasan_shadow_check((uintptr_t)buf, \ 1042 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1043 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1044 } \ 1045 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1046 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1047 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1048 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1049 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1050 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1051 bus_size_t count) \ 1052 { \ 1053 kasan_shadow_check((uintptr_t)buf, \ 1054 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1055 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1056 } 1057 1058 ASAN_BUS_READ_FUNC(1, 8) 1059 ASAN_BUS_READ_FUNC(2, 16) 1060 ASAN_BUS_READ_FUNC(4, 32) 1061 ASAN_BUS_READ_FUNC(8, 64) 1062 1063 ASAN_BUS_WRITE_FUNC(1, 8) 1064 ASAN_BUS_WRITE_FUNC(2, 16) 1065 ASAN_BUS_WRITE_FUNC(4, 32) 1066 ASAN_BUS_WRITE_FUNC(8, 64) 1067 1068 #endif /* __HAVE_KASAN_INSTR_BUS */ 1069 1070 /* -------------------------------------------------------------------------- */ 1071 1072 #ifdef __HAVE_KASAN_INSTR_DMA 1073 1074 #include <sys/mbuf.h> 1075 1076 static void 1077 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1078 bool write, uintptr_t pc) 1079 { 1080 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc); 1081 } 1082 1083 static void 1084 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1085 bool write, uintptr_t pc) 1086 { 1087 bus_addr_t minlen; 1088 1089 for (; m != NULL && len != 0; m = m->m_next) { 1090 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc); 1091 1092 if (offset >= m->m_len) { 1093 offset -= m->m_len; 1094 continue; 1095 } 1096 1097 minlen = MIN(len, m->m_len - offset); 1098 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset), 1099 minlen, write, pc); 1100 1101 offset = 0; 1102 len -= minlen; 1103 } 1104 } 1105 1106 static void 1107 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1108 bool write, uintptr_t pc) 1109 { 1110 bus_size_t minlen, resid; 1111 struct iovec *iov; 1112 int i; 1113 1114 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc); 1115 1116 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1117 return; 1118 1119 resid = uio->uio_resid; 1120 iov = uio->uio_iov; 1121 1122 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1123 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]), 1124 false, pc); 1125 minlen = MIN(resid, iov[i].iov_len); 1126 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen, 1127 write, pc); 1128 resid -= minlen; 1129 } 1130 } 1131 1132 void 1133 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1134 { 1135 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0; 1136 1137 switch (map->dm_buftype) { 1138 case KASAN_DMA_LINEAR: 1139 kasan_dma_sync_linear(map->dm_buf, offset, len, write, 1140 __RET_ADDR); 1141 break; 1142 case KASAN_DMA_MBUF: 1143 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write, 1144 __RET_ADDR); 1145 break; 1146 case KASAN_DMA_UIO: 1147 kasan_dma_sync_uio(map->dm_buf, offset, len, write, 1148 __RET_ADDR); 1149 break; 1150 case KASAN_DMA_RAW: 1151 break; 1152 default: 1153 panic("%s: impossible", __func__); 1154 } 1155 } 1156 1157 void 1158 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1159 { 1160 map->dm_buf = buf; 1161 map->dm_buflen = buflen; 1162 map->dm_buftype = type; 1163 } 1164 1165 #endif /* __HAVE_KASAN_INSTR_DMA */ 1166 1167 /* -------------------------------------------------------------------------- */ 1168 1169 void __asan_register_globals(struct __asan_global *, size_t); 1170 void __asan_unregister_globals(struct __asan_global *, size_t); 1171 1172 void 1173 __asan_register_globals(struct __asan_global *globals, size_t n) 1174 { 1175 size_t i; 1176 1177 for (i = 0; i < n; i++) { 1178 kasan_mark(globals[i].beg, globals[i].size, 1179 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 1180 } 1181 } 1182 1183 void 1184 __asan_unregister_globals(struct __asan_global *globals, size_t n) 1185 { 1186 /* never called */ 1187 } 1188 1189 #define ASAN_LOAD_STORE(size) \ 1190 void __asan_load##size(unsigned long); \ 1191 void __asan_load##size(unsigned long addr) \ 1192 { \ 1193 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1194 } \ 1195 void __asan_load##size##_noabort(unsigned long); \ 1196 void __asan_load##size##_noabort(unsigned long addr) \ 1197 { \ 1198 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1199 } \ 1200 void __asan_store##size(unsigned long); \ 1201 void __asan_store##size(unsigned long addr) \ 1202 { \ 1203 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1204 } \ 1205 void __asan_store##size##_noabort(unsigned long); \ 1206 void __asan_store##size##_noabort(unsigned long addr) \ 1207 { \ 1208 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1209 } 1210 1211 ASAN_LOAD_STORE(1); 1212 ASAN_LOAD_STORE(2); 1213 ASAN_LOAD_STORE(4); 1214 ASAN_LOAD_STORE(8); 1215 ASAN_LOAD_STORE(16); 1216 1217 void __asan_loadN(unsigned long, size_t); 1218 void __asan_loadN_noabort(unsigned long, size_t); 1219 void __asan_storeN(unsigned long, size_t); 1220 void __asan_storeN_noabort(unsigned long, size_t); 1221 void __asan_handle_no_return(void); 1222 1223 void 1224 __asan_loadN(unsigned long addr, size_t size) 1225 { 1226 kasan_shadow_check(addr, size, false, __RET_ADDR); 1227 } 1228 1229 void 1230 __asan_loadN_noabort(unsigned long addr, size_t size) 1231 { 1232 kasan_shadow_check(addr, size, false, __RET_ADDR); 1233 } 1234 1235 void 1236 __asan_storeN(unsigned long addr, size_t size) 1237 { 1238 kasan_shadow_check(addr, size, true, __RET_ADDR); 1239 } 1240 1241 void 1242 __asan_storeN_noabort(unsigned long addr, size_t size) 1243 { 1244 kasan_shadow_check(addr, size, true, __RET_ADDR); 1245 } 1246 1247 void 1248 __asan_handle_no_return(void) 1249 { 1250 /* nothing */ 1251 } 1252 1253 #define ASAN_SET_SHADOW(byte) \ 1254 void __asan_set_shadow_##byte(void *, size_t); \ 1255 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1256 { \ 1257 __builtin_memset((void *)addr, 0x##byte, size); \ 1258 } 1259 1260 ASAN_SET_SHADOW(00); 1261 ASAN_SET_SHADOW(f1); 1262 ASAN_SET_SHADOW(f2); 1263 ASAN_SET_SHADOW(f3); 1264 ASAN_SET_SHADOW(f5); 1265 ASAN_SET_SHADOW(f8); 1266 1267 void __asan_poison_stack_memory(const void *, size_t); 1268 void __asan_unpoison_stack_memory(const void *, size_t); 1269 1270 void 1271 __asan_poison_stack_memory(const void *addr, size_t size) 1272 { 1273 size = roundup(size, KASAN_SHADOW_SCALE_SIZE); 1274 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1275 } 1276 1277 void 1278 __asan_unpoison_stack_memory(const void *addr, size_t size) 1279 { 1280 kasan_shadow_Nbyte_markvalid(addr, size); 1281 } 1282 1283 void __asan_alloca_poison(const void *, size_t); 1284 void __asan_allocas_unpoison(const void *, const void *); 1285 1286 void __asan_alloca_poison(const void *addr, size_t size) 1287 { 1288 const void *l, *r; 1289 1290 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0); 1291 1292 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1293 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1294 1295 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1296 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1297 KASAN_STACK_MID); 1298 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1299 } 1300 1301 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1302 { 1303 size_t size; 1304 1305 if (__predict_false(!stkbegin)) 1306 return; 1307 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1308 return; 1309 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1310 1311 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1312 } 1313
Indexes created Sun Sep 21 18:09:52 GMT 2025