subr_csan.c revision 1.7.2.1
1 /* $NetBSD: subr_csan.c,v 1.7.2.1 2020/04/20 11:29:10 bouyer Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.7.2.1 2020/04/20 11:29:10 bouyer Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/csan.h> 43 #include <sys/cpu.h> 44 45 #ifdef KCSAN_PANIC 46 #define REPORT panic 47 #else 48 #define REPORT printf 49 #endif 50 51 typedef struct { 52 uintptr_t addr; 53 uint32_t size; 54 bool write:1; 55 bool atomic:1; 56 uintptr_t pc; 57 } csan_cell_t; 58 59 typedef struct { 60 bool inited; 61 uint32_t cnt; 62 csan_cell_t cell; 63 } csan_cpu_t; 64 65 static csan_cpu_t kcsan_cpus[MAXCPUS]; 66 static bool kcsan_enabled __read_mostly; 67 68 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 69 70 #define KCSAN_NACCESSES 1024 71 #define KCSAN_DELAY 10 /* 10 microseconds */ 72 73 /* -------------------------------------------------------------------------- */ 74 75 /* The MD code. */ 76 #include <machine/csan.h> 77 78 /* -------------------------------------------------------------------------- */ 79 80 void 81 kcsan_init(void) 82 { 83 kcsan_enabled = true; 84 } 85 86 void 87 kcsan_cpu_init(struct cpu_info *ci) 88 { 89 kcsan_cpus[cpu_index(ci)].inited = true; 90 } 91 92 /* -------------------------------------------------------------------------- */ 93 94 static inline void 95 kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) 96 { 97 const char *newsym, *oldsym; 98 99 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { 100 newsym = "Unknown"; 101 } 102 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { 103 oldsym = "Unknown"; 104 } 105 REPORT("CSan: Racy Access " 106 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " 107 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", 108 newcpu, 109 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 110 (void *)new->addr, new->size, (void *)new->pc, newsym, 111 oldcpu, 112 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 113 (void *)old->addr, old->size, (void *)old->pc, oldsym); 114 kcsan_md_unwind(); 115 } 116 117 static inline bool 118 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 119 { 120 if (new->write && !new->atomic) 121 return false; 122 if (old->write && !old->atomic) 123 return false; 124 return true; 125 } 126 127 static inline void 128 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 129 { 130 csan_cell_t old, new; 131 csan_cpu_t *cpu; 132 uint64_t intr; 133 size_t i; 134 135 if (__predict_false(!kcsan_enabled)) 136 return; 137 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) 138 return; 139 140 new.addr = addr; 141 new.size = size; 142 new.write = write; 143 new.atomic = atomic; 144 new.pc = pc; 145 146 for (i = 0; i < ncpu; i++) { 147 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 148 149 if (old.addr + old.size <= new.addr) 150 continue; 151 if (new.addr + new.size <= old.addr) 152 continue; 153 if (__predict_true(!old.write && !new.write)) 154 continue; 155 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 156 continue; 157 158 kcsan_report(&new, cpu_number(), &old, i); 159 break; 160 } 161 162 if (__predict_false(!kcsan_md_is_avail())) 163 return; 164 165 kcsan_md_disable_intrs(&intr); 166 167 cpu = &kcsan_cpus[cpu_number()]; 168 if (__predict_false(!cpu->inited)) 169 goto out; 170 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 171 if (__predict_true(cpu->cnt != 0)) 172 goto out; 173 174 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 175 kcsan_md_delay(KCSAN_DELAY); 176 __builtin_memset(&cpu->cell, 0, sizeof(new)); 177 178 out: 179 kcsan_md_enable_intrs(&intr); 180 } 181 182 #define CSAN_READ(size) \ 183 void __tsan_read##size(uintptr_t); \ 184 void __tsan_read##size(uintptr_t addr) \ 185 { \ 186 kcsan_access(addr, size, false, false, __RET_ADDR); \ 187 } 188 189 CSAN_READ(1) 190 CSAN_READ(2) 191 CSAN_READ(4) 192 CSAN_READ(8) 193 CSAN_READ(16) 194 195 #define CSAN_WRITE(size) \ 196 void __tsan_write##size(uintptr_t); \ 197 void __tsan_write##size(uintptr_t addr) \ 198 { \ 199 kcsan_access(addr, size, true, false, __RET_ADDR); \ 200 } 201 202 CSAN_WRITE(1) 203 CSAN_WRITE(2) 204 CSAN_WRITE(4) 205 CSAN_WRITE(8) 206 CSAN_WRITE(16) 207 208 void __tsan_read_range(uintptr_t, size_t); 209 void __tsan_write_range(uintptr_t, size_t); 210 211 void 212 __tsan_read_range(uintptr_t addr, size_t size) 213 { 214 kcsan_access(addr, size, false, false, __RET_ADDR); 215 } 216 217 void 218 __tsan_write_range(uintptr_t addr, size_t size) 219 { 220 kcsan_access(addr, size, true, false, __RET_ADDR); 221 } 222 223 void __tsan_init(void); 224 void __tsan_func_entry(void *); 225 void __tsan_func_exit(void); 226 227 void 228 __tsan_init(void) 229 { 230 } 231 232 void 233 __tsan_func_entry(void *call_pc) 234 { 235 } 236 237 void 238 __tsan_func_exit(void) 239 { 240 } 241 242 /* -------------------------------------------------------------------------- */ 243 244 void * 245 kcsan_memcpy(void *dst, const void *src, size_t len) 246 { 247 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 248 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 249 return __builtin_memcpy(dst, src, len); 250 } 251 252 int 253 kcsan_memcmp(const void *b1, const void *b2, size_t len) 254 { 255 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 256 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 257 return __builtin_memcmp(b1, b2, len); 258 } 259 260 void * 261 kcsan_memset(void *b, int c, size_t len) 262 { 263 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 264 return __builtin_memset(b, c, len); 265 } 266 267 void * 268 kcsan_memmove(void *dst, const void *src, size_t len) 269 { 270 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 271 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 272 return __builtin_memmove(dst, src, len); 273 } 274 275 char * 276 kcsan_strcpy(char *dst, const char *src) 277 { 278 char *save = dst; 279 280 while (1) { 281 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 282 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 283 *dst = *src; 284 if (*src == '\0') 285 break; 286 src++, dst++; 287 } 288 289 return save; 290 } 291 292 int 293 kcsan_strcmp(const char *s1, const char *s2) 294 { 295 while (1) { 296 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 297 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 298 if (*s1 != *s2) 299 break; 300 if (*s1 == '\0') 301 return 0; 302 s1++, s2++; 303 } 304 305 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 306 } 307 308 size_t 309 kcsan_strlen(const char *str) 310 { 311 const char *s; 312 313 s = str; 314 while (1) { 315 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 316 if (*s == '\0') 317 break; 318 s++; 319 } 320 321 return (s - str); 322 } 323 324 #undef kcopy 325 #undef copystr 326 #undef copyinstr 327 #undef copyoutstr 328 #undef copyin 329 #undef copyout 330 331 int kcsan_kcopy(const void *, void *, size_t); 332 int kcsan_copystr(const void *, void *, size_t, size_t *); 333 int kcsan_copyinstr(const void *, void *, size_t, size_t *); 334 int kcsan_copyoutstr(const void *, void *, size_t, size_t *); 335 int kcsan_copyin(const void *, void *, size_t); 336 int kcsan_copyout(const void *, void *, size_t); 337 int kcopy(const void *, void *, size_t); 338 int copystr(const void *, void *, size_t, size_t *); 339 int copyinstr(const void *, void *, size_t, size_t *); 340 int copyoutstr(const void *, void *, size_t, size_t *); 341 int copyin(const void *, void *, size_t); 342 int copyout(const void *, void *, size_t); 343 344 int 345 kcsan_kcopy(const void *src, void *dst, size_t len) 346 { 347 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 348 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 349 return kcopy(src, dst, len); 350 } 351 352 int 353 kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 354 { 355 kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR); 356 return copystr(kfaddr, kdaddr, len, done); 357 } 358 359 int 360 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 361 { 362 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 363 return copyin(uaddr, kaddr, len); 364 } 365 366 int 367 kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 368 { 369 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 370 return copyout(kaddr, uaddr, len); 371 } 372 373 int 374 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 375 { 376 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 377 return copyinstr(uaddr, kaddr, len, done); 378 } 379 380 int 381 kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 382 { 383 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 384 return copyoutstr(kaddr, uaddr, len, done); 385 } 386 387 /* -------------------------------------------------------------------------- */ 388 389 #undef atomic_add_32 390 #undef atomic_add_int 391 #undef atomic_add_long 392 #undef atomic_add_ptr 393 #undef atomic_add_64 394 #undef atomic_add_32_nv 395 #undef atomic_add_int_nv 396 #undef atomic_add_long_nv 397 #undef atomic_add_ptr_nv 398 #undef atomic_add_64_nv 399 #undef atomic_and_32 400 #undef atomic_and_uint 401 #undef atomic_and_ulong 402 #undef atomic_and_64 403 #undef atomic_and_32_nv 404 #undef atomic_and_uint_nv 405 #undef atomic_and_ulong_nv 406 #undef atomic_and_64_nv 407 #undef atomic_or_32 408 #undef atomic_or_uint 409 #undef atomic_or_ulong 410 #undef atomic_or_64 411 #undef atomic_or_32_nv 412 #undef atomic_or_uint_nv 413 #undef atomic_or_ulong_nv 414 #undef atomic_or_64_nv 415 #undef atomic_cas_32 416 #undef atomic_cas_uint 417 #undef atomic_cas_ulong 418 #undef atomic_cas_ptr 419 #undef atomic_cas_64 420 #undef atomic_cas_32_ni 421 #undef atomic_cas_uint_ni 422 #undef atomic_cas_ulong_ni 423 #undef atomic_cas_ptr_ni 424 #undef atomic_cas_64_ni 425 #undef atomic_swap_32 426 #undef atomic_swap_uint 427 #undef atomic_swap_ulong 428 #undef atomic_swap_ptr 429 #undef atomic_swap_64 430 #undef atomic_dec_32 431 #undef atomic_dec_uint 432 #undef atomic_dec_ulong 433 #undef atomic_dec_ptr 434 #undef atomic_dec_64 435 #undef atomic_dec_32_nv 436 #undef atomic_dec_uint_nv 437 #undef atomic_dec_ulong_nv 438 #undef atomic_dec_ptr_nv 439 #undef atomic_dec_64_nv 440 #undef atomic_inc_32 441 #undef atomic_inc_uint 442 #undef atomic_inc_ulong 443 #undef atomic_inc_ptr 444 #undef atomic_inc_64 445 #undef atomic_inc_32_nv 446 #undef atomic_inc_uint_nv 447 #undef atomic_inc_ulong_nv 448 #undef atomic_inc_ptr_nv 449 #undef atomic_inc_64_nv 450 451 #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 452 void atomic_add_##name(volatile targ1 *, targ2); \ 453 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 454 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 455 { \ 456 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 457 __RET_ADDR); \ 458 atomic_add_##name(ptr, val); \ 459 } \ 460 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 461 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 462 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 463 { \ 464 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 465 __RET_ADDR); \ 466 return atomic_add_##name##_nv(ptr, val); \ 467 } 468 469 #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 470 void atomic_and_##name(volatile targ1 *, targ2); \ 471 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 472 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 473 { \ 474 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 475 __RET_ADDR); \ 476 atomic_and_##name(ptr, val); \ 477 } \ 478 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 479 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 480 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 481 { \ 482 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 483 __RET_ADDR); \ 484 return atomic_and_##name##_nv(ptr, val); \ 485 } 486 487 #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 488 void atomic_or_##name(volatile targ1 *, targ2); \ 489 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 490 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 491 { \ 492 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 493 __RET_ADDR); \ 494 atomic_or_##name(ptr, val); \ 495 } \ 496 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 497 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 498 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 499 { \ 500 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 501 __RET_ADDR); \ 502 return atomic_or_##name##_nv(ptr, val); \ 503 } 504 505 #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 506 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 507 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 508 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 509 { \ 510 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 511 __RET_ADDR); \ 512 return atomic_cas_##name(ptr, exp, new); \ 513 } \ 514 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 515 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 516 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 517 { \ 518 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 519 __RET_ADDR); \ 520 return atomic_cas_##name##_ni(ptr, exp, new); \ 521 } 522 523 #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 524 tret atomic_swap_##name(volatile targ1 *, targ2); \ 525 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ 526 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 527 { \ 528 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 529 __RET_ADDR); \ 530 return atomic_swap_##name(ptr, val); \ 531 } 532 533 #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 534 void atomic_dec_##name(volatile targ1 *); \ 535 void kcsan_atomic_dec_##name(volatile targ1 *); \ 536 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ 537 { \ 538 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 539 __RET_ADDR); \ 540 atomic_dec_##name(ptr); \ 541 } \ 542 tret atomic_dec_##name##_nv(volatile targ1 *); \ 543 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ 544 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 545 { \ 546 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 547 __RET_ADDR); \ 548 return atomic_dec_##name##_nv(ptr); \ 549 } 550 551 #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 552 void atomic_inc_##name(volatile targ1 *); \ 553 void kcsan_atomic_inc_##name(volatile targ1 *); \ 554 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ 555 { \ 556 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 557 __RET_ADDR); \ 558 atomic_inc_##name(ptr); \ 559 } \ 560 tret atomic_inc_##name##_nv(volatile targ1 *); \ 561 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ 562 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 563 { \ 564 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 565 __RET_ADDR); \ 566 return atomic_inc_##name##_nv(ptr); \ 567 } 568 569 CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 570 CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 571 CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 572 CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 573 CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 574 575 CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 576 CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 577 CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 578 CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 579 580 CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 581 CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 582 CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 583 CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 584 585 CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 586 CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 587 CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 588 CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 589 CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 590 591 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 592 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 593 CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 594 CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 595 CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 596 597 CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 598 CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 599 CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 600 CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 601 CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 602 603 CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 604 CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 605 CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 606 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 607 CSAN_ATOMIC_FUNC_INC(ptr, void *, void); 608 609 void 610 kcsan_atomic_load(const volatile void *p, void *v, int size) 611 { 612 kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR); 613 switch (size) { 614 case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break; 615 case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break; 616 case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break; 617 case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break; 618 } 619 } 620 621 void 622 kcsan_atomic_store(volatile void *p, const void *v, int size) 623 { 624 kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR); 625 switch (size) { 626 case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break; 627 case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break; 628 case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break; 629 case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break; 630 } 631 } 632 633 /* -------------------------------------------------------------------------- */ 634 635 #include <sys/bus.h> 636 637 #undef bus_space_read_multi_1 638 #undef bus_space_read_multi_2 639 #undef bus_space_read_multi_4 640 #undef bus_space_read_multi_8 641 #undef bus_space_read_multi_stream_1 642 #undef bus_space_read_multi_stream_2 643 #undef bus_space_read_multi_stream_4 644 #undef bus_space_read_multi_stream_8 645 #undef bus_space_read_region_1 646 #undef bus_space_read_region_2 647 #undef bus_space_read_region_4 648 #undef bus_space_read_region_8 649 #undef bus_space_read_region_stream_1 650 #undef bus_space_read_region_stream_2 651 #undef bus_space_read_region_stream_4 652 #undef bus_space_read_region_stream_8 653 #undef bus_space_write_multi_1 654 #undef bus_space_write_multi_2 655 #undef bus_space_write_multi_4 656 #undef bus_space_write_multi_8 657 #undef bus_space_write_multi_stream_1 658 #undef bus_space_write_multi_stream_2 659 #undef bus_space_write_multi_stream_4 660 #undef bus_space_write_multi_stream_8 661 #undef bus_space_write_region_1 662 #undef bus_space_write_region_2 663 #undef bus_space_write_region_4 664 #undef bus_space_write_region_8 665 #undef bus_space_write_region_stream_1 666 #undef bus_space_write_region_stream_2 667 #undef bus_space_write_region_stream_4 668 #undef bus_space_write_region_stream_8 669 670 #define CSAN_BUS_READ_FUNC(bytes, bits) \ 671 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 672 bus_size_t, uint##bits##_t *, bus_size_t); \ 673 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 674 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 675 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 676 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 677 bus_size_t count) \ 678 { \ 679 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 680 false, false, __RET_ADDR); \ 681 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 682 } \ 683 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 684 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 685 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 686 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 687 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 688 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 689 bus_size_t count) \ 690 { \ 691 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 692 false, false, __RET_ADDR); \ 693 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 694 } \ 695 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 696 bus_size_t, uint##bits##_t *, bus_size_t); \ 697 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 698 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 699 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 700 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 701 bus_size_t count) \ 702 { \ 703 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 704 false, false, __RET_ADDR); \ 705 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 706 } \ 707 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 708 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 709 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 710 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 711 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 712 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 713 bus_size_t count) \ 714 { \ 715 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 716 false, false, __RET_ADDR); \ 717 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 718 } 719 720 #define CSAN_BUS_WRITE_FUNC(bytes, bits) \ 721 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 722 bus_size_t, const uint##bits##_t *, bus_size_t); \ 723 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 724 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 725 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 726 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 727 bus_size_t count) \ 728 { \ 729 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 730 true, false, __RET_ADDR); \ 731 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 732 } \ 733 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 734 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 735 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 736 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 737 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 738 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 739 bus_size_t count) \ 740 { \ 741 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 742 true, false, __RET_ADDR); \ 743 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 744 } \ 745 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 746 bus_size_t, const uint##bits##_t *, bus_size_t); \ 747 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 748 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 749 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 750 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 751 bus_size_t count) \ 752 { \ 753 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 754 true, false, __RET_ADDR); \ 755 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 756 } \ 757 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 758 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 759 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 760 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 761 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 762 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 763 bus_size_t count) \ 764 { \ 765 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 766 true, false, __RET_ADDR); \ 767 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 768 } 769 770 CSAN_BUS_READ_FUNC(1, 8) 771 CSAN_BUS_READ_FUNC(2, 16) 772 CSAN_BUS_READ_FUNC(4, 32) 773 CSAN_BUS_READ_FUNC(8, 64) 774 775 CSAN_BUS_WRITE_FUNC(1, 8) 776 CSAN_BUS_WRITE_FUNC(2, 16) 777 CSAN_BUS_WRITE_FUNC(4, 32) 778 CSAN_BUS_WRITE_FUNC(8, 64) 779
Indexes created Thu Oct 02 14:10:14 GMT 2025