1 /* $NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KCSAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.14 2022/07/30 14:13:27 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/csan.h> 42 #include <sys/cpu.h> 43 #include <sys/pserialize.h> 44 45 #ifdef KCSAN_PANIC 46 #define REPORT panic 47 #else 48 #define REPORT printf 49 #endif 50 51 typedef struct { 52 uintptr_t addr; 53 uint32_t size; 54 bool write:1; 55 bool atomic:1; 56 uintptr_t pc; 57 } csan_cell_t; 58 59 typedef struct { 60 bool inited; 61 uint32_t cnt; 62 csan_cell_t cell; 63 } csan_cpu_t; 64 65 static csan_cpu_t kcsan_cpus[MAXCPUS]; 66 static bool kcsan_enabled __read_mostly; 67 68 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 69 70 #define KCSAN_NACCESSES 1024 71 #define KCSAN_DELAY 10 /* 10 microseconds */ 72 73 /* -------------------------------------------------------------------------- */ 74 75 /* The MD code. */ 76 #include <machine/csan.h> 77 78 /* -------------------------------------------------------------------------- */ 79 80 void 81 kcsan_init(void) 82 { 83 kcsan_enabled = true; 84 } 85 86 void 87 kcsan_cpu_init(struct cpu_info *ci) 88 { 89 kcsan_cpus[cpu_index(ci)].inited = true; 90 } 91 92 /* -------------------------------------------------------------------------- */ 93 94 static inline void 95 kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) 96 { 97 const char *newsym, *oldsym; 98 int s; 99 100 s = pserialize_read_enter(); 101 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { 102 newsym = "Unknown"; 103 } 104 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { 105 oldsym = "Unknown"; 106 } 107 REPORT("CSan: Racy Access " 108 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " 109 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", 110 newcpu, 111 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 112 (void *)new->addr, new->size, (void *)new->pc, newsym, 113 oldcpu, 114 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 115 (void *)old->addr, old->size, (void *)old->pc, oldsym); 116 pserialize_read_exit(s); 117 kcsan_md_unwind(); 118 } 119 120 static inline bool 121 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 122 { 123 if (new->write && !new->atomic) 124 return false; 125 if (old->write && !old->atomic) 126 return false; 127 return true; 128 } 129 130 static inline void 131 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 132 { 133 csan_cell_t old, new; 134 csan_cpu_t *cpu; 135 uint64_t intr; 136 size_t i; 137 138 if (__predict_false(!kcsan_enabled)) 139 return; 140 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) 141 return; 142 143 new.addr = addr; 144 new.size = size; 145 new.write = write; 146 new.atomic = atomic; 147 new.pc = pc; 148 149 for (i = 0; i < ncpu; i++) { 150 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 151 152 if (old.addr + old.size <= new.addr) 153 continue; 154 if (new.addr + new.size <= old.addr) 155 continue; 156 if (__predict_true(!old.write && !new.write)) 157 continue; 158 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 159 continue; 160 161 kcsan_report(&new, cpu_number(), &old, i); 162 break; 163 } 164 165 if (__predict_false(!kcsan_md_is_avail())) 166 return; 167 168 kcsan_md_disable_intrs(&intr); 169 170 cpu = &kcsan_cpus[cpu_number()]; 171 if (__predict_false(!cpu->inited)) 172 goto out; 173 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 174 if (__predict_true(cpu->cnt != 0)) 175 goto out; 176 177 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 178 kcsan_md_delay(KCSAN_DELAY); 179 __builtin_memset(&cpu->cell, 0, sizeof(new)); 180 181 out: 182 kcsan_md_enable_intrs(&intr); 183 } 184 185 #define CSAN_READ(size) \ 186 void __tsan_read##size(uintptr_t); \ 187 void __tsan_read##size(uintptr_t addr) \ 188 { \ 189 kcsan_access(addr, size, false, false, __RET_ADDR); \ 190 } 191 192 CSAN_READ(1) 193 CSAN_READ(2) 194 CSAN_READ(4) 195 CSAN_READ(8) 196 CSAN_READ(16) 197 198 #define CSAN_WRITE(size) \ 199 void __tsan_write##size(uintptr_t); \ 200 void __tsan_write##size(uintptr_t addr) \ 201 { \ 202 kcsan_access(addr, size, true, false, __RET_ADDR); \ 203 } 204 205 CSAN_WRITE(1) 206 CSAN_WRITE(2) 207 CSAN_WRITE(4) 208 CSAN_WRITE(8) 209 CSAN_WRITE(16) 210 211 void __tsan_read_range(uintptr_t, size_t); 212 void __tsan_write_range(uintptr_t, size_t); 213 214 void 215 __tsan_read_range(uintptr_t addr, size_t size) 216 { 217 kcsan_access(addr, size, false, false, __RET_ADDR); 218 } 219 220 void 221 __tsan_write_range(uintptr_t addr, size_t size) 222 { 223 kcsan_access(addr, size, true, false, __RET_ADDR); 224 } 225 226 void __tsan_init(void); 227 void __tsan_func_entry(void *); 228 void __tsan_func_exit(void); 229 230 void 231 __tsan_init(void) 232 { 233 } 234 235 void 236 __tsan_func_entry(void *call_pc) 237 { 238 } 239 240 void 241 __tsan_func_exit(void) 242 { 243 } 244 245 /* -------------------------------------------------------------------------- */ 246 247 void * 248 kcsan_memcpy(void *dst, const void *src, size_t len) 249 { 250 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 251 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 252 return __builtin_memcpy(dst, src, len); 253 } 254 255 int 256 kcsan_memcmp(const void *b1, const void *b2, size_t len) 257 { 258 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 259 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 260 return __builtin_memcmp(b1, b2, len); 261 } 262 263 void * 264 kcsan_memset(void *b, int c, size_t len) 265 { 266 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 267 return __builtin_memset(b, c, len); 268 } 269 270 void * 271 kcsan_memmove(void *dst, const void *src, size_t len) 272 { 273 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 274 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 275 return __builtin_memmove(dst, src, len); 276 } 277 278 char * 279 kcsan_strcpy(char *dst, const char *src) 280 { 281 char *save = dst; 282 283 while (1) { 284 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 285 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 286 *dst = *src; 287 if (*src == '\0') 288 break; 289 src++, dst++; 290 } 291 292 return save; 293 } 294 295 int 296 kcsan_strcmp(const char *s1, const char *s2) 297 { 298 while (1) { 299 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 300 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 301 if (*s1 != *s2) 302 break; 303 if (*s1 == '\0') 304 return 0; 305 s1++, s2++; 306 } 307 308 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 309 } 310 311 size_t 312 kcsan_strlen(const char *str) 313 { 314 const char *s; 315 316 s = str; 317 while (1) { 318 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 319 if (*s == '\0') 320 break; 321 s++; 322 } 323 324 return (s - str); 325 } 326 327 #undef kcopy 328 #undef copyinstr 329 #undef copyoutstr 330 #undef copyin 331 #undef copyout 332 333 int kcsan_kcopy(const void *, void *, size_t); 334 int kcsan_copyinstr(const void *, void *, size_t, size_t *); 335 int kcsan_copyoutstr(const void *, void *, size_t, size_t *); 336 int kcsan_copyin(const void *, void *, size_t); 337 int kcsan_copyout(const void *, void *, size_t); 338 int kcopy(const void *, void *, size_t); 339 int copyinstr(const void *, void *, size_t, size_t *); 340 int copyoutstr(const void *, void *, size_t, size_t *); 341 int copyin(const void *, void *, size_t); 342 int copyout(const void *, void *, size_t); 343 344 int 345 kcsan_kcopy(const void *src, void *dst, size_t len) 346 { 347 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 348 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 349 return kcopy(src, dst, len); 350 } 351 352 int 353 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 354 { 355 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 356 return copyin(uaddr, kaddr, len); 357 } 358 359 int 360 kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 361 { 362 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 363 return copyout(kaddr, uaddr, len); 364 } 365 366 int 367 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 368 { 369 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 370 return copyinstr(uaddr, kaddr, len, done); 371 } 372 373 int 374 kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 375 { 376 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 377 return copyoutstr(kaddr, uaddr, len, done); 378 } 379 380 /* -------------------------------------------------------------------------- */ 381 382 #undef atomic_add_32 383 #undef atomic_add_int 384 #undef atomic_add_long 385 #undef atomic_add_ptr 386 #undef atomic_add_64 387 #undef atomic_add_32_nv 388 #undef atomic_add_int_nv 389 #undef atomic_add_long_nv 390 #undef atomic_add_ptr_nv 391 #undef atomic_add_64_nv 392 #undef atomic_and_32 393 #undef atomic_and_uint 394 #undef atomic_and_ulong 395 #undef atomic_and_64 396 #undef atomic_and_32_nv 397 #undef atomic_and_uint_nv 398 #undef atomic_and_ulong_nv 399 #undef atomic_and_64_nv 400 #undef atomic_or_32 401 #undef atomic_or_uint 402 #undef atomic_or_ulong 403 #undef atomic_or_64 404 #undef atomic_or_32_nv 405 #undef atomic_or_uint_nv 406 #undef atomic_or_ulong_nv 407 #undef atomic_or_64_nv 408 #undef atomic_cas_32 409 #undef atomic_cas_uint 410 #undef atomic_cas_ulong 411 #undef atomic_cas_ptr 412 #undef atomic_cas_64 413 #undef atomic_cas_32_ni 414 #undef atomic_cas_uint_ni 415 #undef atomic_cas_ulong_ni 416 #undef atomic_cas_ptr_ni 417 #undef atomic_cas_64_ni 418 #undef atomic_swap_32 419 #undef atomic_swap_uint 420 #undef atomic_swap_ulong 421 #undef atomic_swap_ptr 422 #undef atomic_swap_64 423 #undef atomic_dec_32 424 #undef atomic_dec_uint 425 #undef atomic_dec_ulong 426 #undef atomic_dec_ptr 427 #undef atomic_dec_64 428 #undef atomic_dec_32_nv 429 #undef atomic_dec_uint_nv 430 #undef atomic_dec_ulong_nv 431 #undef atomic_dec_ptr_nv 432 #undef atomic_dec_64_nv 433 #undef atomic_inc_32 434 #undef atomic_inc_uint 435 #undef atomic_inc_ulong 436 #undef atomic_inc_ptr 437 #undef atomic_inc_64 438 #undef atomic_inc_32_nv 439 #undef atomic_inc_uint_nv 440 #undef atomic_inc_ulong_nv 441 #undef atomic_inc_ptr_nv 442 #undef atomic_inc_64_nv 443 444 #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 445 void atomic_add_##name(volatile targ1 *, targ2); \ 446 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 447 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 448 { \ 449 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 450 __RET_ADDR); \ 451 atomic_add_##name(ptr, val); \ 452 } \ 453 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 454 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 455 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 456 { \ 457 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 458 __RET_ADDR); \ 459 return atomic_add_##name##_nv(ptr, val); \ 460 } 461 462 #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 463 void atomic_and_##name(volatile targ1 *, targ2); \ 464 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 465 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 466 { \ 467 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 468 __RET_ADDR); \ 469 atomic_and_##name(ptr, val); \ 470 } \ 471 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 472 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 473 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 474 { \ 475 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 476 __RET_ADDR); \ 477 return atomic_and_##name##_nv(ptr, val); \ 478 } 479 480 #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 481 void atomic_or_##name(volatile targ1 *, targ2); \ 482 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 483 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 484 { \ 485 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 486 __RET_ADDR); \ 487 atomic_or_##name(ptr, val); \ 488 } \ 489 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 490 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 491 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 492 { \ 493 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 494 __RET_ADDR); \ 495 return atomic_or_##name##_nv(ptr, val); \ 496 } 497 498 #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 499 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 500 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 501 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 502 { \ 503 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 504 __RET_ADDR); \ 505 return atomic_cas_##name(ptr, exp, new); \ 506 } \ 507 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 508 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 509 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 510 { \ 511 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 512 __RET_ADDR); \ 513 return atomic_cas_##name##_ni(ptr, exp, new); \ 514 } 515 516 #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 517 tret atomic_swap_##name(volatile targ1 *, targ2); \ 518 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ 519 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 520 { \ 521 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 522 __RET_ADDR); \ 523 return atomic_swap_##name(ptr, val); \ 524 } 525 526 #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 527 void atomic_dec_##name(volatile targ1 *); \ 528 void kcsan_atomic_dec_##name(volatile targ1 *); \ 529 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ 530 { \ 531 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 532 __RET_ADDR); \ 533 atomic_dec_##name(ptr); \ 534 } \ 535 tret atomic_dec_##name##_nv(volatile targ1 *); \ 536 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ 537 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 538 { \ 539 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 540 __RET_ADDR); \ 541 return atomic_dec_##name##_nv(ptr); \ 542 } 543 544 #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 545 void atomic_inc_##name(volatile targ1 *); \ 546 void kcsan_atomic_inc_##name(volatile targ1 *); \ 547 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ 548 { \ 549 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 550 __RET_ADDR); \ 551 atomic_inc_##name(ptr); \ 552 } \ 553 tret atomic_inc_##name##_nv(volatile targ1 *); \ 554 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ 555 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 556 { \ 557 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 558 __RET_ADDR); \ 559 return atomic_inc_##name##_nv(ptr); \ 560 } 561 562 CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 563 CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 564 CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 565 CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 566 CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 567 568 CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 569 CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 570 CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 571 CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 572 573 CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 574 CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 575 CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 576 CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 577 578 CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 579 CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 580 CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 581 CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 582 CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 583 584 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 585 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 586 CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 587 CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 588 CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 589 590 CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 591 CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 592 CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 593 CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 594 CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 595 596 CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 597 CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 598 CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 599 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 600 CSAN_ATOMIC_FUNC_INC(ptr, void *, void); 601 602 void 603 kcsan_atomic_load(const volatile void *p, void *v, int size) 604 { 605 kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR); 606 switch (size) { 607 case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break; 608 case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break; 609 case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break; 610 case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break; 611 } 612 } 613 614 void 615 kcsan_atomic_store(volatile void *p, const void *v, int size) 616 { 617 kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR); 618 #ifdef __HAVE_HASHLOCKED_ATOMICS 619 __do_atomic_store(p, v, size); 620 #else 621 switch (size) { 622 case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break; 623 case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break; 624 case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break; 625 case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break; 626 } 627 #endif 628 } 629 630 /* -------------------------------------------------------------------------- */ 631 632 #include <sys/bus.h> 633 634 #undef bus_space_read_multi_1 635 #undef bus_space_read_multi_2 636 #undef bus_space_read_multi_4 637 #undef bus_space_read_multi_8 638 #undef bus_space_read_multi_stream_1 639 #undef bus_space_read_multi_stream_2 640 #undef bus_space_read_multi_stream_4 641 #undef bus_space_read_multi_stream_8 642 #undef bus_space_read_region_1 643 #undef bus_space_read_region_2 644 #undef bus_space_read_region_4 645 #undef bus_space_read_region_8 646 #undef bus_space_read_region_stream_1 647 #undef bus_space_read_region_stream_2 648 #undef bus_space_read_region_stream_4 649 #undef bus_space_read_region_stream_8 650 #undef bus_space_write_multi_1 651 #undef bus_space_write_multi_2 652 #undef bus_space_write_multi_4 653 #undef bus_space_write_multi_8 654 #undef bus_space_write_multi_stream_1 655 #undef bus_space_write_multi_stream_2 656 #undef bus_space_write_multi_stream_4 657 #undef bus_space_write_multi_stream_8 658 #undef bus_space_write_region_1 659 #undef bus_space_write_region_2 660 #undef bus_space_write_region_4 661 #undef bus_space_write_region_8 662 #undef bus_space_write_region_stream_1 663 #undef bus_space_write_region_stream_2 664 #undef bus_space_write_region_stream_4 665 #undef bus_space_write_region_stream_8 666 667 #define CSAN_BUS_READ_FUNC(bytes, bits) \ 668 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 669 bus_size_t, uint##bits##_t *, bus_size_t); \ 670 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 671 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 672 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 673 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 674 bus_size_t count) \ 675 { \ 676 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 677 false, false, __RET_ADDR); \ 678 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 679 } \ 680 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 681 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 682 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 683 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 684 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 685 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 686 bus_size_t count) \ 687 { \ 688 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 689 false, false, __RET_ADDR); \ 690 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 691 } \ 692 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 693 bus_size_t, uint##bits##_t *, bus_size_t); \ 694 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 695 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 696 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 697 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 698 bus_size_t count) \ 699 { \ 700 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 701 false, false, __RET_ADDR); \ 702 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 703 } \ 704 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 705 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 706 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 707 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 708 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 709 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 710 bus_size_t count) \ 711 { \ 712 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 713 false, false, __RET_ADDR); \ 714 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 715 } 716 717 #define CSAN_BUS_WRITE_FUNC(bytes, bits) \ 718 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 719 bus_size_t, const uint##bits##_t *, bus_size_t); \ 720 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 721 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 722 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 723 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 724 bus_size_t count) \ 725 { \ 726 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 727 true, false, __RET_ADDR); \ 728 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 729 } \ 730 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 731 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 732 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 733 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 734 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 735 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 736 bus_size_t count) \ 737 { \ 738 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 739 true, false, __RET_ADDR); \ 740 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 741 } \ 742 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 743 bus_size_t, const uint##bits##_t *, bus_size_t); \ 744 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 745 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 746 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 747 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 748 bus_size_t count) \ 749 { \ 750 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 751 true, false, __RET_ADDR); \ 752 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 753 } \ 754 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 755 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 756 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 757 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 758 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 759 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 760 bus_size_t count) \ 761 { \ 762 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 763 true, false, __RET_ADDR); \ 764 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 765 } 766 767 CSAN_BUS_READ_FUNC(1, 8) 768 CSAN_BUS_READ_FUNC(2, 16) 769 CSAN_BUS_READ_FUNC(4, 32) 770 CSAN_BUS_READ_FUNC(8, 64) 771 772 CSAN_BUS_WRITE_FUNC(1, 8) 773 CSAN_BUS_WRITE_FUNC(2, 16) 774 CSAN_BUS_WRITE_FUNC(4, 32) 775 CSAN_BUS_WRITE_FUNC(8, 64) 776
Indexes created Sat Sep 20 22:09:52 GMT 2025