subr_csan.c revision 1.12
1 /* $NetBSD: subr_csan.c,v 1.12 2021/09/07 16:56:13 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KCSAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.12 2021/09/07 16:56:13 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/csan.h> 42 #include <sys/cpu.h> 43 44 #ifdef KCSAN_PANIC 45 #define REPORT panic 46 #else 47 #define REPORT printf 48 #endif 49 50 typedef struct { 51 uintptr_t addr; 52 uint32_t size; 53 bool write:1; 54 bool atomic:1; 55 uintptr_t pc; 56 } csan_cell_t; 57 58 typedef struct { 59 bool inited; 60 uint32_t cnt; 61 csan_cell_t cell; 62 } csan_cpu_t; 63 64 static csan_cpu_t kcsan_cpus[MAXCPUS]; 65 static bool kcsan_enabled __read_mostly; 66 67 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 68 69 #define KCSAN_NACCESSES 1024 70 #define KCSAN_DELAY 10 /* 10 microseconds */ 71 72 /* -------------------------------------------------------------------------- */ 73 74 /* The MD code. */ 75 #include <machine/csan.h> 76 77 /* -------------------------------------------------------------------------- */ 78 79 void 80 kcsan_init(void) 81 { 82 kcsan_enabled = true; 83 } 84 85 void 86 kcsan_cpu_init(struct cpu_info *ci) 87 { 88 kcsan_cpus[cpu_index(ci)].inited = true; 89 } 90 91 /* -------------------------------------------------------------------------- */ 92 93 static inline void 94 kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) 95 { 96 const char *newsym, *oldsym; 97 98 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { 99 newsym = "Unknown"; 100 } 101 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { 102 oldsym = "Unknown"; 103 } 104 REPORT("CSan: Racy Access " 105 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " 106 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", 107 newcpu, 108 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 109 (void *)new->addr, new->size, (void *)new->pc, newsym, 110 oldcpu, 111 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 112 (void *)old->addr, old->size, (void *)old->pc, oldsym); 113 kcsan_md_unwind(); 114 } 115 116 static inline bool 117 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 118 { 119 if (new->write && !new->atomic) 120 return false; 121 if (old->write && !old->atomic) 122 return false; 123 return true; 124 } 125 126 static inline void 127 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 128 { 129 csan_cell_t old, new; 130 csan_cpu_t *cpu; 131 uint64_t intr; 132 size_t i; 133 134 if (__predict_false(!kcsan_enabled)) 135 return; 136 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) 137 return; 138 139 new.addr = addr; 140 new.size = size; 141 new.write = write; 142 new.atomic = atomic; 143 new.pc = pc; 144 145 for (i = 0; i < ncpu; i++) { 146 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 147 148 if (old.addr + old.size <= new.addr) 149 continue; 150 if (new.addr + new.size <= old.addr) 151 continue; 152 if (__predict_true(!old.write && !new.write)) 153 continue; 154 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 155 continue; 156 157 kcsan_report(&new, cpu_number(), &old, i); 158 break; 159 } 160 161 if (__predict_false(!kcsan_md_is_avail())) 162 return; 163 164 kcsan_md_disable_intrs(&intr); 165 166 cpu = &kcsan_cpus[cpu_number()]; 167 if (__predict_false(!cpu->inited)) 168 goto out; 169 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 170 if (__predict_true(cpu->cnt != 0)) 171 goto out; 172 173 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 174 kcsan_md_delay(KCSAN_DELAY); 175 __builtin_memset(&cpu->cell, 0, sizeof(new)); 176 177 out: 178 kcsan_md_enable_intrs(&intr); 179 } 180 181 #define CSAN_READ(size) \ 182 void __tsan_read##size(uintptr_t); \ 183 void __tsan_read##size(uintptr_t addr) \ 184 { \ 185 kcsan_access(addr, size, false, false, __RET_ADDR); \ 186 } 187 188 CSAN_READ(1) 189 CSAN_READ(2) 190 CSAN_READ(4) 191 CSAN_READ(8) 192 CSAN_READ(16) 193 194 #define CSAN_WRITE(size) \ 195 void __tsan_write##size(uintptr_t); \ 196 void __tsan_write##size(uintptr_t addr) \ 197 { \ 198 kcsan_access(addr, size, true, false, __RET_ADDR); \ 199 } 200 201 CSAN_WRITE(1) 202 CSAN_WRITE(2) 203 CSAN_WRITE(4) 204 CSAN_WRITE(8) 205 CSAN_WRITE(16) 206 207 void __tsan_read_range(uintptr_t, size_t); 208 void __tsan_write_range(uintptr_t, size_t); 209 210 void 211 __tsan_read_range(uintptr_t addr, size_t size) 212 { 213 kcsan_access(addr, size, false, false, __RET_ADDR); 214 } 215 216 void 217 __tsan_write_range(uintptr_t addr, size_t size) 218 { 219 kcsan_access(addr, size, true, false, __RET_ADDR); 220 } 221 222 void __tsan_init(void); 223 void __tsan_func_entry(void *); 224 void __tsan_func_exit(void); 225 226 void 227 __tsan_init(void) 228 { 229 } 230 231 void 232 __tsan_func_entry(void *call_pc) 233 { 234 } 235 236 void 237 __tsan_func_exit(void) 238 { 239 } 240 241 /* -------------------------------------------------------------------------- */ 242 243 void * 244 kcsan_memcpy(void *dst, const void *src, size_t len) 245 { 246 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 247 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 248 return __builtin_memcpy(dst, src, len); 249 } 250 251 int 252 kcsan_memcmp(const void *b1, const void *b2, size_t len) 253 { 254 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 255 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 256 return __builtin_memcmp(b1, b2, len); 257 } 258 259 void * 260 kcsan_memset(void *b, int c, size_t len) 261 { 262 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 263 return __builtin_memset(b, c, len); 264 } 265 266 void * 267 kcsan_memmove(void *dst, const void *src, size_t len) 268 { 269 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 270 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 271 return __builtin_memmove(dst, src, len); 272 } 273 274 char * 275 kcsan_strcpy(char *dst, const char *src) 276 { 277 char *save = dst; 278 279 while (1) { 280 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 281 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 282 *dst = *src; 283 if (*src == '\0') 284 break; 285 src++, dst++; 286 } 287 288 return save; 289 } 290 291 int 292 kcsan_strcmp(const char *s1, const char *s2) 293 { 294 while (1) { 295 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 296 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 297 if (*s1 != *s2) 298 break; 299 if (*s1 == '\0') 300 return 0; 301 s1++, s2++; 302 } 303 304 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 305 } 306 307 size_t 308 kcsan_strlen(const char *str) 309 { 310 const char *s; 311 312 s = str; 313 while (1) { 314 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 315 if (*s == '\0') 316 break; 317 s++; 318 } 319 320 return (s - str); 321 } 322 323 #undef kcopy 324 #undef copyinstr 325 #undef copyoutstr 326 #undef copyin 327 #undef copyout 328 329 int kcsan_kcopy(const void *, void *, size_t); 330 int kcsan_copyinstr(const void *, void *, size_t, size_t *); 331 int kcsan_copyoutstr(const void *, void *, size_t, size_t *); 332 int kcsan_copyin(const void *, void *, size_t); 333 int kcsan_copyout(const void *, void *, size_t); 334 int kcopy(const void *, void *, size_t); 335 int copyinstr(const void *, void *, size_t, size_t *); 336 int copyoutstr(const void *, void *, size_t, size_t *); 337 int copyin(const void *, void *, size_t); 338 int copyout(const void *, void *, size_t); 339 340 int 341 kcsan_kcopy(const void *src, void *dst, size_t len) 342 { 343 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 344 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 345 return kcopy(src, dst, len); 346 } 347 348 int 349 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 350 { 351 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 352 return copyin(uaddr, kaddr, len); 353 } 354 355 int 356 kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 357 { 358 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 359 return copyout(kaddr, uaddr, len); 360 } 361 362 int 363 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 364 { 365 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 366 return copyinstr(uaddr, kaddr, len, done); 367 } 368 369 int 370 kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 371 { 372 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 373 return copyoutstr(kaddr, uaddr, len, done); 374 } 375 376 /* -------------------------------------------------------------------------- */ 377 378 #undef atomic_add_32 379 #undef atomic_add_int 380 #undef atomic_add_long 381 #undef atomic_add_ptr 382 #undef atomic_add_64 383 #undef atomic_add_32_nv 384 #undef atomic_add_int_nv 385 #undef atomic_add_long_nv 386 #undef atomic_add_ptr_nv 387 #undef atomic_add_64_nv 388 #undef atomic_and_32 389 #undef atomic_and_uint 390 #undef atomic_and_ulong 391 #undef atomic_and_64 392 #undef atomic_and_32_nv 393 #undef atomic_and_uint_nv 394 #undef atomic_and_ulong_nv 395 #undef atomic_and_64_nv 396 #undef atomic_or_32 397 #undef atomic_or_uint 398 #undef atomic_or_ulong 399 #undef atomic_or_64 400 #undef atomic_or_32_nv 401 #undef atomic_or_uint_nv 402 #undef atomic_or_ulong_nv 403 #undef atomic_or_64_nv 404 #undef atomic_cas_32 405 #undef atomic_cas_uint 406 #undef atomic_cas_ulong 407 #undef atomic_cas_ptr 408 #undef atomic_cas_64 409 #undef atomic_cas_32_ni 410 #undef atomic_cas_uint_ni 411 #undef atomic_cas_ulong_ni 412 #undef atomic_cas_ptr_ni 413 #undef atomic_cas_64_ni 414 #undef atomic_swap_32 415 #undef atomic_swap_uint 416 #undef atomic_swap_ulong 417 #undef atomic_swap_ptr 418 #undef atomic_swap_64 419 #undef atomic_dec_32 420 #undef atomic_dec_uint 421 #undef atomic_dec_ulong 422 #undef atomic_dec_ptr 423 #undef atomic_dec_64 424 #undef atomic_dec_32_nv 425 #undef atomic_dec_uint_nv 426 #undef atomic_dec_ulong_nv 427 #undef atomic_dec_ptr_nv 428 #undef atomic_dec_64_nv 429 #undef atomic_inc_32 430 #undef atomic_inc_uint 431 #undef atomic_inc_ulong 432 #undef atomic_inc_ptr 433 #undef atomic_inc_64 434 #undef atomic_inc_32_nv 435 #undef atomic_inc_uint_nv 436 #undef atomic_inc_ulong_nv 437 #undef atomic_inc_ptr_nv 438 #undef atomic_inc_64_nv 439 440 #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 441 void atomic_add_##name(volatile targ1 *, targ2); \ 442 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 443 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 444 { \ 445 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 446 __RET_ADDR); \ 447 atomic_add_##name(ptr, val); \ 448 } \ 449 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 450 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 451 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 452 { \ 453 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 454 __RET_ADDR); \ 455 return atomic_add_##name##_nv(ptr, val); \ 456 } 457 458 #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 459 void atomic_and_##name(volatile targ1 *, targ2); \ 460 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 461 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 462 { \ 463 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 464 __RET_ADDR); \ 465 atomic_and_##name(ptr, val); \ 466 } \ 467 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 468 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 469 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 470 { \ 471 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 472 __RET_ADDR); \ 473 return atomic_and_##name##_nv(ptr, val); \ 474 } 475 476 #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 477 void atomic_or_##name(volatile targ1 *, targ2); \ 478 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 479 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 480 { \ 481 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 482 __RET_ADDR); \ 483 atomic_or_##name(ptr, val); \ 484 } \ 485 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 486 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 487 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 488 { \ 489 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 490 __RET_ADDR); \ 491 return atomic_or_##name##_nv(ptr, val); \ 492 } 493 494 #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 495 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 496 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 497 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 498 { \ 499 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 500 __RET_ADDR); \ 501 return atomic_cas_##name(ptr, exp, new); \ 502 } \ 503 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 504 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 505 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 506 { \ 507 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 508 __RET_ADDR); \ 509 return atomic_cas_##name##_ni(ptr, exp, new); \ 510 } 511 512 #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 513 tret atomic_swap_##name(volatile targ1 *, targ2); \ 514 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ 515 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 516 { \ 517 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 518 __RET_ADDR); \ 519 return atomic_swap_##name(ptr, val); \ 520 } 521 522 #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 523 void atomic_dec_##name(volatile targ1 *); \ 524 void kcsan_atomic_dec_##name(volatile targ1 *); \ 525 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ 526 { \ 527 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 528 __RET_ADDR); \ 529 atomic_dec_##name(ptr); \ 530 } \ 531 tret atomic_dec_##name##_nv(volatile targ1 *); \ 532 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ 533 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 534 { \ 535 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 536 __RET_ADDR); \ 537 return atomic_dec_##name##_nv(ptr); \ 538 } 539 540 #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 541 void atomic_inc_##name(volatile targ1 *); \ 542 void kcsan_atomic_inc_##name(volatile targ1 *); \ 543 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ 544 { \ 545 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 546 __RET_ADDR); \ 547 atomic_inc_##name(ptr); \ 548 } \ 549 tret atomic_inc_##name##_nv(volatile targ1 *); \ 550 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ 551 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 552 { \ 553 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 554 __RET_ADDR); \ 555 return atomic_inc_##name##_nv(ptr); \ 556 } 557 558 CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 559 CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 560 CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 561 CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 562 CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 563 564 CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 565 CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 566 CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 567 CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 568 569 CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 570 CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 571 CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 572 CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 573 574 CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 575 CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 576 CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 577 CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 578 CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 579 580 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 581 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 582 CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 583 CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 584 CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 585 586 CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 587 CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 588 CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 589 CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 590 CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 591 592 CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 593 CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 594 CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 595 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 596 CSAN_ATOMIC_FUNC_INC(ptr, void *, void); 597 598 void 599 kcsan_atomic_load(const volatile void *p, void *v, int size) 600 { 601 kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR); 602 switch (size) { 603 case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break; 604 case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break; 605 case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break; 606 case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break; 607 } 608 } 609 610 void 611 kcsan_atomic_store(volatile void *p, const void *v, int size) 612 { 613 kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR); 614 switch (size) { 615 case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break; 616 case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break; 617 case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break; 618 case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break; 619 } 620 } 621 622 /* -------------------------------------------------------------------------- */ 623 624 #include <sys/bus.h> 625 626 #undef bus_space_read_multi_1 627 #undef bus_space_read_multi_2 628 #undef bus_space_read_multi_4 629 #undef bus_space_read_multi_8 630 #undef bus_space_read_multi_stream_1 631 #undef bus_space_read_multi_stream_2 632 #undef bus_space_read_multi_stream_4 633 #undef bus_space_read_multi_stream_8 634 #undef bus_space_read_region_1 635 #undef bus_space_read_region_2 636 #undef bus_space_read_region_4 637 #undef bus_space_read_region_8 638 #undef bus_space_read_region_stream_1 639 #undef bus_space_read_region_stream_2 640 #undef bus_space_read_region_stream_4 641 #undef bus_space_read_region_stream_8 642 #undef bus_space_write_multi_1 643 #undef bus_space_write_multi_2 644 #undef bus_space_write_multi_4 645 #undef bus_space_write_multi_8 646 #undef bus_space_write_multi_stream_1 647 #undef bus_space_write_multi_stream_2 648 #undef bus_space_write_multi_stream_4 649 #undef bus_space_write_multi_stream_8 650 #undef bus_space_write_region_1 651 #undef bus_space_write_region_2 652 #undef bus_space_write_region_4 653 #undef bus_space_write_region_8 654 #undef bus_space_write_region_stream_1 655 #undef bus_space_write_region_stream_2 656 #undef bus_space_write_region_stream_4 657 #undef bus_space_write_region_stream_8 658 659 #define CSAN_BUS_READ_FUNC(bytes, bits) \ 660 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 661 bus_size_t, uint##bits##_t *, bus_size_t); \ 662 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 663 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 664 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 665 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 666 bus_size_t count) \ 667 { \ 668 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 669 false, false, __RET_ADDR); \ 670 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 671 } \ 672 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 673 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 674 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 675 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 676 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 677 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 678 bus_size_t count) \ 679 { \ 680 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 681 false, false, __RET_ADDR); \ 682 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 683 } \ 684 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 685 bus_size_t, uint##bits##_t *, bus_size_t); \ 686 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 687 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 688 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 689 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 690 bus_size_t count) \ 691 { \ 692 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 693 false, false, __RET_ADDR); \ 694 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 695 } \ 696 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 697 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 698 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 699 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 700 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 701 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 702 bus_size_t count) \ 703 { \ 704 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 705 false, false, __RET_ADDR); \ 706 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 707 } 708 709 #define CSAN_BUS_WRITE_FUNC(bytes, bits) \ 710 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 711 bus_size_t, const uint##bits##_t *, bus_size_t); \ 712 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 713 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 714 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 715 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 716 bus_size_t count) \ 717 { \ 718 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 719 true, false, __RET_ADDR); \ 720 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 721 } \ 722 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 723 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 724 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 725 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 726 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 727 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 728 bus_size_t count) \ 729 { \ 730 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 731 true, false, __RET_ADDR); \ 732 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 733 } \ 734 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 735 bus_size_t, const uint##bits##_t *, bus_size_t); \ 736 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 737 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 738 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 739 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 740 bus_size_t count) \ 741 { \ 742 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 743 true, false, __RET_ADDR); \ 744 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 745 } \ 746 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 747 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 748 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 749 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 750 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 751 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 752 bus_size_t count) \ 753 { \ 754 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 755 true, false, __RET_ADDR); \ 756 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 757 } 758 759 CSAN_BUS_READ_FUNC(1, 8) 760 CSAN_BUS_READ_FUNC(2, 16) 761 CSAN_BUS_READ_FUNC(4, 32) 762 CSAN_BUS_READ_FUNC(8, 64) 763 764 CSAN_BUS_WRITE_FUNC(1, 8) 765 CSAN_BUS_WRITE_FUNC(2, 16) 766 CSAN_BUS_WRITE_FUNC(4, 32) 767 CSAN_BUS_WRITE_FUNC(8, 64) 768
Indexes created Thu Oct 02 14:10:14 GMT 2025