subr_csan.c revision 1.3
1 /* $NetBSD: subr_csan.c,v 1.3 2019/11/08 12:36:10 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.3 2019/11/08 12:36:10 maxv Exp $"); 34 35 #include "opt_kcsan.h" 36 37 #include <sys/param.h> 38 #include <sys/device.h> 39 #include <sys/kernel.h> 40 #include <sys/param.h> 41 #include <sys/conf.h> 42 #include <sys/systm.h> 43 #include <sys/types.h> 44 #include <sys/csan.h> 45 #include <sys/cpu.h> 46 47 #ifdef KCSAN_PANIC 48 #define REPORT panic 49 #else 50 #define REPORT printf 51 #endif 52 53 typedef struct { 54 uintptr_t addr; 55 uint32_t size; 56 bool write:1; 57 bool atomic:1; 58 uintptr_t pc; 59 } csan_cell_t; 60 61 typedef struct { 62 bool inited; 63 uint32_t cnt; 64 csan_cell_t cell; 65 } csan_cpu_t; 66 67 static csan_cpu_t kcsan_cpus[MAXCPUS]; 68 static bool kcsan_enabled __read_mostly; 69 70 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 71 72 #define KCSAN_NACCESSES 1024 73 #define KCSAN_DELAY 10 /* 10 microseconds */ 74 75 /* -------------------------------------------------------------------------- */ 76 77 /* The MD code. */ 78 #include <machine/csan.h> 79 80 /* -------------------------------------------------------------------------- */ 81 82 void 83 kcsan_init(void) 84 { 85 kcsan_enabled = true; 86 } 87 88 void 89 kcsan_cpu_init(struct cpu_info *ci) 90 { 91 kcsan_cpus[cpu_index(ci)].inited = true; 92 } 93 94 /* -------------------------------------------------------------------------- */ 95 96 static inline void 97 kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) 98 { 99 const char *newsym, *oldsym; 100 101 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { 102 newsym = "Unknown"; 103 } 104 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { 105 oldsym = "Unknown"; 106 } 107 REPORT("CSan: Racy Access " 108 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " 109 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", 110 newcpu, 111 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 112 (void *)new->addr, new->size, (void *)new->pc, newsym, 113 oldcpu, 114 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 115 (void *)old->addr, old->size, (void *)old->pc, oldsym); 116 kcsan_md_unwind(); 117 } 118 119 static inline bool 120 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 121 { 122 if (new->write && !new->atomic) 123 return false; 124 if (old->write && !old->atomic) 125 return false; 126 return true; 127 } 128 129 static inline void 130 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 131 { 132 csan_cell_t old, new; 133 csan_cpu_t *cpu; 134 uint64_t intr; 135 size_t i; 136 137 if (__predict_false(!kcsan_enabled)) 138 return; 139 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) 140 return; 141 142 new.addr = addr; 143 new.size = size; 144 new.write = write; 145 new.atomic = atomic; 146 new.pc = pc; 147 148 for (i = 0; i < ncpu; i++) { 149 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 150 151 if (old.addr + old.size <= new.addr) 152 continue; 153 if (new.addr + new.size <= old.addr) 154 continue; 155 if (__predict_true(!old.write && !new.write)) 156 continue; 157 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 158 continue; 159 160 kcsan_report(&new, cpu_number(), &old, i); 161 break; 162 } 163 164 if (__predict_false(!kcsan_md_is_avail())) 165 return; 166 167 kcsan_md_disable_intrs(&intr); 168 169 cpu = &kcsan_cpus[cpu_number()]; 170 if (__predict_false(!cpu->inited)) 171 goto out; 172 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 173 if (__predict_true(cpu->cnt != 0)) 174 goto out; 175 176 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 177 kcsan_md_delay(KCSAN_DELAY); 178 __builtin_memset(&cpu->cell, 0, sizeof(new)); 179 180 out: 181 kcsan_md_enable_intrs(&intr); 182 } 183 184 #define CSAN_READ(size) \ 185 void __tsan_read##size(uintptr_t); \ 186 void __tsan_read##size(uintptr_t addr) \ 187 { \ 188 kcsan_access(addr, size, false, false, __RET_ADDR); \ 189 } 190 191 CSAN_READ(1) 192 CSAN_READ(2) 193 CSAN_READ(4) 194 CSAN_READ(8) 195 CSAN_READ(16) 196 197 #define CSAN_WRITE(size) \ 198 void __tsan_write##size(uintptr_t); \ 199 void __tsan_write##size(uintptr_t addr) \ 200 { \ 201 kcsan_access(addr, size, true, false, __RET_ADDR); \ 202 } 203 204 CSAN_WRITE(1) 205 CSAN_WRITE(2) 206 CSAN_WRITE(4) 207 CSAN_WRITE(8) 208 CSAN_WRITE(16) 209 210 void __tsan_read_range(uintptr_t, size_t); 211 void __tsan_write_range(uintptr_t, size_t); 212 213 void 214 __tsan_read_range(uintptr_t addr, size_t size) 215 { 216 kcsan_access(addr, size, false, false, __RET_ADDR); 217 } 218 219 void 220 __tsan_write_range(uintptr_t addr, size_t size) 221 { 222 kcsan_access(addr, size, true, false, __RET_ADDR); 223 } 224 225 void __tsan_init(void); 226 void __tsan_func_entry(void *); 227 void __tsan_func_exit(void); 228 229 void 230 __tsan_init(void) 231 { 232 } 233 234 void 235 __tsan_func_entry(void *call_pc) 236 { 237 } 238 239 void 240 __tsan_func_exit(void) 241 { 242 } 243 244 /* -------------------------------------------------------------------------- */ 245 246 void * 247 kcsan_memcpy(void *dst, const void *src, size_t len) 248 { 249 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 250 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 251 return __builtin_memcpy(dst, src, len); 252 } 253 254 int 255 kcsan_memcmp(const void *b1, const void *b2, size_t len) 256 { 257 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 258 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 259 return __builtin_memcmp(b1, b2, len); 260 } 261 262 void * 263 kcsan_memset(void *b, int c, size_t len) 264 { 265 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 266 return __builtin_memset(b, c, len); 267 } 268 269 void * 270 kcsan_memmove(void *dst, const void *src, size_t len) 271 { 272 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 273 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 274 return __builtin_memmove(dst, src, len); 275 } 276 277 char * 278 kcsan_strcpy(char *dst, const char *src) 279 { 280 char *save = dst; 281 282 while (1) { 283 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 284 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 285 *dst = *src; 286 if (*src == '\0') 287 break; 288 src++, dst++; 289 } 290 291 return save; 292 } 293 294 int 295 kcsan_strcmp(const char *s1, const char *s2) 296 { 297 while (1) { 298 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 299 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 300 if (*s1 != *s2) 301 break; 302 if (*s1 == '\0') 303 return 0; 304 s1++, s2++; 305 } 306 307 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 308 } 309 310 size_t 311 kcsan_strlen(const char *str) 312 { 313 const char *s; 314 315 s = str; 316 while (1) { 317 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 318 if (*s == '\0') 319 break; 320 s++; 321 } 322 323 return (s - str); 324 } 325 326 #undef kcopy 327 #undef copystr 328 #undef copyinstr 329 #undef copyoutstr 330 #undef copyin 331 332 int kcsan_kcopy(const void *, void *, size_t); 333 int kcsan_copystr(const void *, void *, size_t, size_t *); 334 int kcsan_copyinstr(const void *, void *, size_t, size_t *); 335 int kcsan_copyoutstr(const void *, void *, size_t, size_t *); 336 int kcsan_copyin(const void *, void *, size_t); 337 int kcopy(const void *, void *, size_t); 338 int copystr(const void *, void *, size_t, size_t *); 339 int copyinstr(const void *, void *, size_t, size_t *); 340 int copyoutstr(const void *, void *, size_t, size_t *); 341 int copyin(const void *, void *, size_t); 342 343 int 344 kcsan_kcopy(const void *src, void *dst, size_t len) 345 { 346 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 347 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 348 return kcopy(src, dst, len); 349 } 350 351 int 352 kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) 353 { 354 kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR); 355 return copystr(kfaddr, kdaddr, len, done); 356 } 357 358 int 359 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 360 { 361 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 362 return copyin(uaddr, kaddr, len); 363 } 364 365 int 366 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 367 { 368 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 369 return copyinstr(uaddr, kaddr, len, done); 370 } 371 372 int 373 kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 374 { 375 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 376 return copyoutstr(kaddr, uaddr, len, done); 377 } 378 379 /* -------------------------------------------------------------------------- */ 380 381 #undef atomic_add_32 382 #undef atomic_add_int 383 #undef atomic_add_long 384 #undef atomic_add_ptr 385 #undef atomic_add_64 386 #undef atomic_add_32_nv 387 #undef atomic_add_int_nv 388 #undef atomic_add_long_nv 389 #undef atomic_add_ptr_nv 390 #undef atomic_add_64_nv 391 #undef atomic_and_32 392 #undef atomic_and_uint 393 #undef atomic_and_ulong 394 #undef atomic_and_64 395 #undef atomic_and_32_nv 396 #undef atomic_and_uint_nv 397 #undef atomic_and_ulong_nv 398 #undef atomic_and_64_nv 399 #undef atomic_or_32 400 #undef atomic_or_uint 401 #undef atomic_or_ulong 402 #undef atomic_or_64 403 #undef atomic_or_32_nv 404 #undef atomic_or_uint_nv 405 #undef atomic_or_ulong_nv 406 #undef atomic_or_64_nv 407 #undef atomic_cas_32 408 #undef atomic_cas_uint 409 #undef atomic_cas_ulong 410 #undef atomic_cas_ptr 411 #undef atomic_cas_64 412 #undef atomic_cas_32_ni 413 #undef atomic_cas_uint_ni 414 #undef atomic_cas_ulong_ni 415 #undef atomic_cas_ptr_ni 416 #undef atomic_cas_64_ni 417 #undef atomic_swap_32 418 #undef atomic_swap_uint 419 #undef atomic_swap_ulong 420 #undef atomic_swap_ptr 421 #undef atomic_swap_64 422 #undef atomic_dec_32 423 #undef atomic_dec_uint 424 #undef atomic_dec_ulong 425 #undef atomic_dec_ptr 426 #undef atomic_dec_64 427 #undef atomic_dec_32_nv 428 #undef atomic_dec_uint_nv 429 #undef atomic_dec_ulong_nv 430 #undef atomic_dec_ptr_nv 431 #undef atomic_dec_64_nv 432 #undef atomic_inc_32 433 #undef atomic_inc_uint 434 #undef atomic_inc_ulong 435 #undef atomic_inc_ptr 436 #undef atomic_inc_64 437 #undef atomic_inc_32_nv 438 #undef atomic_inc_uint_nv 439 #undef atomic_inc_ulong_nv 440 #undef atomic_inc_ptr_nv 441 #undef atomic_inc_64_nv 442 443 #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 444 void atomic_add_##name(volatile targ1 *, targ2); \ 445 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 446 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 447 { \ 448 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 449 __RET_ADDR); \ 450 atomic_add_##name(ptr, val); \ 451 } \ 452 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 453 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 454 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 455 { \ 456 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 457 __RET_ADDR); \ 458 return atomic_add_##name##_nv(ptr, val); \ 459 } 460 461 #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 462 void atomic_and_##name(volatile targ1 *, targ2); \ 463 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 464 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 465 { \ 466 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 467 __RET_ADDR); \ 468 atomic_and_##name(ptr, val); \ 469 } \ 470 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 471 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 472 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 473 { \ 474 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 475 __RET_ADDR); \ 476 return atomic_and_##name##_nv(ptr, val); \ 477 } 478 479 #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 480 void atomic_or_##name(volatile targ1 *, targ2); \ 481 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 482 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 483 { \ 484 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 485 __RET_ADDR); \ 486 atomic_or_##name(ptr, val); \ 487 } \ 488 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 489 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 490 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 491 { \ 492 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 493 __RET_ADDR); \ 494 return atomic_or_##name##_nv(ptr, val); \ 495 } 496 497 #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 498 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 499 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 500 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 501 { \ 502 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 503 __RET_ADDR); \ 504 return atomic_cas_##name(ptr, exp, new); \ 505 } \ 506 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 507 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 508 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 509 { \ 510 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 511 __RET_ADDR); \ 512 return atomic_cas_##name##_ni(ptr, exp, new); \ 513 } 514 515 #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 516 tret atomic_swap_##name(volatile targ1 *, targ2); \ 517 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ 518 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 519 { \ 520 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 521 __RET_ADDR); \ 522 return atomic_swap_##name(ptr, val); \ 523 } 524 525 #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 526 void atomic_dec_##name(volatile targ1 *); \ 527 void kcsan_atomic_dec_##name(volatile targ1 *); \ 528 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ 529 { \ 530 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 531 __RET_ADDR); \ 532 atomic_dec_##name(ptr); \ 533 } \ 534 tret atomic_dec_##name##_nv(volatile targ1 *); \ 535 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ 536 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 537 { \ 538 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 539 __RET_ADDR); \ 540 return atomic_dec_##name##_nv(ptr); \ 541 } 542 543 #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 544 void atomic_inc_##name(volatile targ1 *); \ 545 void kcsan_atomic_inc_##name(volatile targ1 *); \ 546 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ 547 { \ 548 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 549 __RET_ADDR); \ 550 atomic_inc_##name(ptr); \ 551 } \ 552 tret atomic_inc_##name##_nv(volatile targ1 *); \ 553 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ 554 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 555 { \ 556 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 557 __RET_ADDR); \ 558 return atomic_inc_##name##_nv(ptr); \ 559 } 560 561 CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 562 CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 563 CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 564 CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 565 CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 566 567 CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 568 CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 569 CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 570 CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 571 572 CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 573 CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 574 CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 575 CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 576 577 CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 578 CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 579 CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 580 CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 581 CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 582 583 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 584 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 585 CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 586 CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 587 CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 588 589 CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 590 CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 591 CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 592 CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 593 CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 594 595 CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 596 CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 597 CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 598 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 599 CSAN_ATOMIC_FUNC_INC(ptr, void *, void); 600 601 /* -------------------------------------------------------------------------- */ 602 603 #include <sys/bus.h> 604 605 #undef bus_space_read_multi_1 606 #undef bus_space_read_multi_2 607 #undef bus_space_read_multi_4 608 #undef bus_space_read_multi_8 609 #undef bus_space_read_multi_stream_1 610 #undef bus_space_read_multi_stream_2 611 #undef bus_space_read_multi_stream_4 612 #undef bus_space_read_multi_stream_8 613 #undef bus_space_read_region_1 614 #undef bus_space_read_region_2 615 #undef bus_space_read_region_4 616 #undef bus_space_read_region_8 617 #undef bus_space_read_region_stream_1 618 #undef bus_space_read_region_stream_2 619 #undef bus_space_read_region_stream_4 620 #undef bus_space_read_region_stream_8 621 #undef bus_space_write_multi_1 622 #undef bus_space_write_multi_2 623 #undef bus_space_write_multi_4 624 #undef bus_space_write_multi_8 625 #undef bus_space_write_multi_stream_1 626 #undef bus_space_write_multi_stream_2 627 #undef bus_space_write_multi_stream_4 628 #undef bus_space_write_multi_stream_8 629 #undef bus_space_write_region_1 630 #undef bus_space_write_region_2 631 #undef bus_space_write_region_4 632 #undef bus_space_write_region_8 633 #undef bus_space_write_region_stream_1 634 #undef bus_space_write_region_stream_2 635 #undef bus_space_write_region_stream_4 636 #undef bus_space_write_region_stream_8 637 638 #define CSAN_BUS_READ_FUNC(bytes, bits) \ 639 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 640 bus_size_t, uint##bits##_t *, bus_size_t); \ 641 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 642 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 643 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 644 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 645 bus_size_t count) \ 646 { \ 647 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 648 false, false, __RET_ADDR); \ 649 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 650 } \ 651 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 652 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 653 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 654 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 655 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 656 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 657 bus_size_t count) \ 658 { \ 659 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 660 false, false, __RET_ADDR); \ 661 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 662 } \ 663 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 664 bus_size_t, uint##bits##_t *, bus_size_t); \ 665 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 666 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 667 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 668 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 669 bus_size_t count) \ 670 { \ 671 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 672 false, false, __RET_ADDR); \ 673 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 674 } \ 675 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 676 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 677 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 678 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 679 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 680 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 681 bus_size_t count) \ 682 { \ 683 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 684 false, false, __RET_ADDR); \ 685 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 686 } 687 688 #define CSAN_BUS_WRITE_FUNC(bytes, bits) \ 689 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 690 bus_size_t, const uint##bits##_t *, bus_size_t); \ 691 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 692 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 693 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 694 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 695 bus_size_t count) \ 696 { \ 697 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 698 true, false, __RET_ADDR); \ 699 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 700 } \ 701 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 702 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 703 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 704 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 705 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 706 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 707 bus_size_t count) \ 708 { \ 709 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 710 true, false, __RET_ADDR); \ 711 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 712 } \ 713 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 714 bus_size_t, const uint##bits##_t *, bus_size_t); \ 715 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 716 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 717 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 718 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 719 bus_size_t count) \ 720 { \ 721 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 722 true, false, __RET_ADDR); \ 723 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 724 } \ 725 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 726 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 727 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 728 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 729 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 730 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 731 bus_size_t count) \ 732 { \ 733 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 734 true, false, __RET_ADDR); \ 735 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 736 } 737 738 CSAN_BUS_READ_FUNC(1, 8) 739 CSAN_BUS_READ_FUNC(2, 16) 740 CSAN_BUS_READ_FUNC(4, 32) 741 CSAN_BUS_READ_FUNC(8, 64) 742 743 CSAN_BUS_WRITE_FUNC(1, 8) 744 CSAN_BUS_WRITE_FUNC(2, 16) 745 CSAN_BUS_WRITE_FUNC(4, 32) 746 CSAN_BUS_WRITE_FUNC(8, 64) 747
Indexes created Tue Sep 23 07:09:52 GMT 2025