subr_csan.c revision 1.8 1 1.8 maxv /* $NetBSD: subr_csan.c,v 1.8 2020/04/15 17:28:26 maxv Exp $ */
2 1.1 maxv
3 1.1 maxv /*
4 1.1 maxv * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 1.1 maxv * All rights reserved.
6 1.1 maxv *
7 1.1 maxv * This code is derived from software contributed to The NetBSD Foundation
8 1.1 maxv * by Maxime Villard.
9 1.1 maxv *
10 1.1 maxv * Redistribution and use in source and binary forms, with or without
11 1.1 maxv * modification, are permitted provided that the following conditions
12 1.1 maxv * are met:
13 1.1 maxv * 1. Redistributions of source code must retain the above copyright
14 1.1 maxv * notice, this list of conditions and the following disclaimer.
15 1.1 maxv * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 maxv * notice, this list of conditions and the following disclaimer in the
17 1.1 maxv * documentation and/or other materials provided with the distribution.
18 1.1 maxv *
19 1.1 maxv * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.1 maxv * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.1 maxv * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.1 maxv * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.1 maxv * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.1 maxv * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.1 maxv * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.1 maxv * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.1 maxv * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.1 maxv * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 maxv * POSSIBILITY OF SUCH DAMAGE.
30 1.1 maxv */
31 1.1 maxv
32 1.1 maxv #include <sys/cdefs.h>
33 1.8 maxv __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.8 2020/04/15 17:28:26 maxv Exp $");
34 1.1 maxv
35 1.1 maxv #include <sys/param.h>
36 1.1 maxv #include <sys/device.h>
37 1.1 maxv #include <sys/kernel.h>
38 1.1 maxv #include <sys/param.h>
39 1.1 maxv #include <sys/conf.h>
40 1.1 maxv #include <sys/systm.h>
41 1.1 maxv #include <sys/types.h>
42 1.1 maxv #include <sys/csan.h>
43 1.1 maxv #include <sys/cpu.h>
44 1.1 maxv
45 1.1 maxv #ifdef KCSAN_PANIC
46 1.1 maxv #define REPORT panic
47 1.1 maxv #else
48 1.1 maxv #define REPORT printf
49 1.1 maxv #endif
50 1.1 maxv
51 1.1 maxv typedef struct {
52 1.1 maxv uintptr_t addr;
53 1.1 maxv uint32_t size;
54 1.1 maxv bool write:1;
55 1.1 maxv bool atomic:1;
56 1.1 maxv uintptr_t pc;
57 1.1 maxv } csan_cell_t;
58 1.1 maxv
59 1.1 maxv typedef struct {
60 1.1 maxv bool inited;
61 1.1 maxv uint32_t cnt;
62 1.1 maxv csan_cell_t cell;
63 1.1 maxv } csan_cpu_t;
64 1.1 maxv
65 1.1 maxv static csan_cpu_t kcsan_cpus[MAXCPUS];
66 1.1 maxv static bool kcsan_enabled __read_mostly;
67 1.1 maxv
68 1.1 maxv #define __RET_ADDR (uintptr_t)__builtin_return_address(0)
69 1.1 maxv
70 1.2 maxv #define KCSAN_NACCESSES 1024
71 1.2 maxv #define KCSAN_DELAY 10 /* 10 microseconds */
72 1.1 maxv
73 1.1 maxv /* -------------------------------------------------------------------------- */
74 1.1 maxv
75 1.1 maxv /* The MD code. */
76 1.1 maxv #include <machine/csan.h>
77 1.1 maxv
78 1.1 maxv /* -------------------------------------------------------------------------- */
79 1.1 maxv
80 1.1 maxv void
81 1.1 maxv kcsan_init(void)
82 1.1 maxv {
83 1.1 maxv kcsan_enabled = true;
84 1.1 maxv }
85 1.1 maxv
86 1.1 maxv void
87 1.1 maxv kcsan_cpu_init(struct cpu_info *ci)
88 1.1 maxv {
89 1.1 maxv kcsan_cpus[cpu_index(ci)].inited = true;
90 1.1 maxv }
91 1.1 maxv
92 1.1 maxv /* -------------------------------------------------------------------------- */
93 1.1 maxv
94 1.1 maxv static inline void
95 1.1 maxv kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu)
96 1.1 maxv {
97 1.1 maxv const char *newsym, *oldsym;
98 1.1 maxv
99 1.1 maxv if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) {
100 1.1 maxv newsym = "Unknown";
101 1.1 maxv }
102 1.1 maxv if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) {
103 1.1 maxv oldsym = "Unknown";
104 1.1 maxv }
105 1.1 maxv REPORT("CSan: Racy Access "
106 1.1 maxv "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] "
107 1.1 maxv "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n",
108 1.1 maxv newcpu,
109 1.1 maxv (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
110 1.1 maxv (void *)new->addr, new->size, (void *)new->pc, newsym,
111 1.1 maxv oldcpu,
112 1.1 maxv (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
113 1.1 maxv (void *)old->addr, old->size, (void *)old->pc, oldsym);
114 1.1 maxv kcsan_md_unwind();
115 1.1 maxv }
116 1.1 maxv
117 1.1 maxv static inline bool
118 1.1 maxv kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
119 1.1 maxv {
120 1.1 maxv if (new->write && !new->atomic)
121 1.1 maxv return false;
122 1.1 maxv if (old->write && !old->atomic)
123 1.1 maxv return false;
124 1.1 maxv return true;
125 1.1 maxv }
126 1.1 maxv
127 1.1 maxv static inline void
128 1.1 maxv kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
129 1.1 maxv {
130 1.1 maxv csan_cell_t old, new;
131 1.1 maxv csan_cpu_t *cpu;
132 1.1 maxv uint64_t intr;
133 1.1 maxv size_t i;
134 1.1 maxv
135 1.1 maxv if (__predict_false(!kcsan_enabled))
136 1.1 maxv return;
137 1.3 maxv if (__predict_false(kcsan_md_unsupported((vaddr_t)addr)))
138 1.3 maxv return;
139 1.1 maxv
140 1.1 maxv new.addr = addr;
141 1.1 maxv new.size = size;
142 1.1 maxv new.write = write;
143 1.1 maxv new.atomic = atomic;
144 1.1 maxv new.pc = pc;
145 1.1 maxv
146 1.1 maxv for (i = 0; i < ncpu; i++) {
147 1.1 maxv __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
148 1.1 maxv
149 1.1 maxv if (old.addr + old.size <= new.addr)
150 1.1 maxv continue;
151 1.1 maxv if (new.addr + new.size <= old.addr)
152 1.1 maxv continue;
153 1.1 maxv if (__predict_true(!old.write && !new.write))
154 1.1 maxv continue;
155 1.1 maxv if (__predict_true(kcsan_access_is_atomic(&new, &old)))
156 1.1 maxv continue;
157 1.1 maxv
158 1.1 maxv kcsan_report(&new, cpu_number(), &old, i);
159 1.1 maxv break;
160 1.1 maxv }
161 1.1 maxv
162 1.1 maxv if (__predict_false(!kcsan_md_is_avail()))
163 1.1 maxv return;
164 1.1 maxv
165 1.1 maxv kcsan_md_disable_intrs(&intr);
166 1.1 maxv
167 1.1 maxv cpu = &kcsan_cpus[cpu_number()];
168 1.1 maxv if (__predict_false(!cpu->inited))
169 1.1 maxv goto out;
170 1.1 maxv cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
171 1.1 maxv if (__predict_true(cpu->cnt != 0))
172 1.1 maxv goto out;
173 1.1 maxv
174 1.1 maxv __builtin_memcpy(&cpu->cell, &new, sizeof(new));
175 1.1 maxv kcsan_md_delay(KCSAN_DELAY);
176 1.1 maxv __builtin_memset(&cpu->cell, 0, sizeof(new));
177 1.1 maxv
178 1.1 maxv out:
179 1.1 maxv kcsan_md_enable_intrs(&intr);
180 1.1 maxv }
181 1.1 maxv
182 1.1 maxv #define CSAN_READ(size) \
183 1.1 maxv void __tsan_read##size(uintptr_t); \
184 1.1 maxv void __tsan_read##size(uintptr_t addr) \
185 1.1 maxv { \
186 1.1 maxv kcsan_access(addr, size, false, false, __RET_ADDR); \
187 1.1 maxv }
188 1.1 maxv
189 1.1 maxv CSAN_READ(1)
190 1.1 maxv CSAN_READ(2)
191 1.1 maxv CSAN_READ(4)
192 1.1 maxv CSAN_READ(8)
193 1.1 maxv CSAN_READ(16)
194 1.1 maxv
195 1.1 maxv #define CSAN_WRITE(size) \
196 1.1 maxv void __tsan_write##size(uintptr_t); \
197 1.1 maxv void __tsan_write##size(uintptr_t addr) \
198 1.1 maxv { \
199 1.1 maxv kcsan_access(addr, size, true, false, __RET_ADDR); \
200 1.1 maxv }
201 1.1 maxv
202 1.1 maxv CSAN_WRITE(1)
203 1.1 maxv CSAN_WRITE(2)
204 1.1 maxv CSAN_WRITE(4)
205 1.1 maxv CSAN_WRITE(8)
206 1.1 maxv CSAN_WRITE(16)
207 1.1 maxv
208 1.1 maxv void __tsan_read_range(uintptr_t, size_t);
209 1.1 maxv void __tsan_write_range(uintptr_t, size_t);
210 1.1 maxv
211 1.1 maxv void
212 1.1 maxv __tsan_read_range(uintptr_t addr, size_t size)
213 1.1 maxv {
214 1.1 maxv kcsan_access(addr, size, false, false, __RET_ADDR);
215 1.1 maxv }
216 1.1 maxv
217 1.1 maxv void
218 1.1 maxv __tsan_write_range(uintptr_t addr, size_t size)
219 1.1 maxv {
220 1.1 maxv kcsan_access(addr, size, true, false, __RET_ADDR);
221 1.1 maxv }
222 1.1 maxv
223 1.1 maxv void __tsan_init(void);
224 1.1 maxv void __tsan_func_entry(void *);
225 1.1 maxv void __tsan_func_exit(void);
226 1.1 maxv
227 1.1 maxv void
228 1.1 maxv __tsan_init(void)
229 1.1 maxv {
230 1.1 maxv }
231 1.1 maxv
232 1.1 maxv void
233 1.1 maxv __tsan_func_entry(void *call_pc)
234 1.1 maxv {
235 1.1 maxv }
236 1.1 maxv
237 1.1 maxv void
238 1.1 maxv __tsan_func_exit(void)
239 1.1 maxv {
240 1.1 maxv }
241 1.1 maxv
242 1.1 maxv /* -------------------------------------------------------------------------- */
243 1.1 maxv
244 1.1 maxv void *
245 1.1 maxv kcsan_memcpy(void *dst, const void *src, size_t len)
246 1.1 maxv {
247 1.1 maxv kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
248 1.1 maxv kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
249 1.1 maxv return __builtin_memcpy(dst, src, len);
250 1.1 maxv }
251 1.1 maxv
252 1.1 maxv int
253 1.1 maxv kcsan_memcmp(const void *b1, const void *b2, size_t len)
254 1.1 maxv {
255 1.1 maxv kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
256 1.1 maxv kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
257 1.1 maxv return __builtin_memcmp(b1, b2, len);
258 1.1 maxv }
259 1.1 maxv
260 1.1 maxv void *
261 1.1 maxv kcsan_memset(void *b, int c, size_t len)
262 1.1 maxv {
263 1.1 maxv kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
264 1.1 maxv return __builtin_memset(b, c, len);
265 1.1 maxv }
266 1.1 maxv
267 1.1 maxv void *
268 1.1 maxv kcsan_memmove(void *dst, const void *src, size_t len)
269 1.1 maxv {
270 1.1 maxv kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
271 1.1 maxv kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
272 1.1 maxv return __builtin_memmove(dst, src, len);
273 1.1 maxv }
274 1.1 maxv
275 1.1 maxv char *
276 1.1 maxv kcsan_strcpy(char *dst, const char *src)
277 1.1 maxv {
278 1.1 maxv char *save = dst;
279 1.1 maxv
280 1.1 maxv while (1) {
281 1.1 maxv kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
282 1.1 maxv kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
283 1.1 maxv *dst = *src;
284 1.1 maxv if (*src == '\0')
285 1.1 maxv break;
286 1.1 maxv src++, dst++;
287 1.1 maxv }
288 1.1 maxv
289 1.1 maxv return save;
290 1.1 maxv }
291 1.1 maxv
292 1.1 maxv int
293 1.1 maxv kcsan_strcmp(const char *s1, const char *s2)
294 1.1 maxv {
295 1.1 maxv while (1) {
296 1.1 maxv kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
297 1.1 maxv kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
298 1.1 maxv if (*s1 != *s2)
299 1.1 maxv break;
300 1.1 maxv if (*s1 == '\0')
301 1.1 maxv return 0;
302 1.1 maxv s1++, s2++;
303 1.1 maxv }
304 1.1 maxv
305 1.1 maxv return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
306 1.1 maxv }
307 1.1 maxv
308 1.1 maxv size_t
309 1.1 maxv kcsan_strlen(const char *str)
310 1.1 maxv {
311 1.1 maxv const char *s;
312 1.1 maxv
313 1.1 maxv s = str;
314 1.1 maxv while (1) {
315 1.1 maxv kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
316 1.1 maxv if (*s == '\0')
317 1.1 maxv break;
318 1.1 maxv s++;
319 1.1 maxv }
320 1.1 maxv
321 1.1 maxv return (s - str);
322 1.1 maxv }
323 1.1 maxv
324 1.1 maxv #undef kcopy
325 1.1 maxv #undef copystr
326 1.1 maxv #undef copyinstr
327 1.1 maxv #undef copyoutstr
328 1.1 maxv #undef copyin
329 1.5 maxv #undef copyout
330 1.1 maxv
331 1.1 maxv int kcsan_kcopy(const void *, void *, size_t);
332 1.1 maxv int kcsan_copystr(const void *, void *, size_t, size_t *);
333 1.1 maxv int kcsan_copyinstr(const void *, void *, size_t, size_t *);
334 1.1 maxv int kcsan_copyoutstr(const void *, void *, size_t, size_t *);
335 1.1 maxv int kcsan_copyin(const void *, void *, size_t);
336 1.5 maxv int kcsan_copyout(const void *, void *, size_t);
337 1.1 maxv int kcopy(const void *, void *, size_t);
338 1.1 maxv int copystr(const void *, void *, size_t, size_t *);
339 1.1 maxv int copyinstr(const void *, void *, size_t, size_t *);
340 1.1 maxv int copyoutstr(const void *, void *, size_t, size_t *);
341 1.1 maxv int copyin(const void *, void *, size_t);
342 1.5 maxv int copyout(const void *, void *, size_t);
343 1.1 maxv
344 1.1 maxv int
345 1.1 maxv kcsan_kcopy(const void *src, void *dst, size_t len)
346 1.1 maxv {
347 1.1 maxv kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
348 1.1 maxv kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
349 1.1 maxv return kcopy(src, dst, len);
350 1.1 maxv }
351 1.1 maxv
352 1.1 maxv int
353 1.1 maxv kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
354 1.1 maxv {
355 1.1 maxv kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
356 1.1 maxv return copystr(kfaddr, kdaddr, len, done);
357 1.1 maxv }
358 1.1 maxv
359 1.1 maxv int
360 1.1 maxv kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
361 1.1 maxv {
362 1.1 maxv kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
363 1.1 maxv return copyin(uaddr, kaddr, len);
364 1.1 maxv }
365 1.1 maxv
366 1.1 maxv int
367 1.5 maxv kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
368 1.5 maxv {
369 1.5 maxv kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
370 1.5 maxv return copyout(kaddr, uaddr, len);
371 1.5 maxv }
372 1.5 maxv
373 1.5 maxv int
374 1.1 maxv kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
375 1.1 maxv {
376 1.1 maxv kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
377 1.1 maxv return copyinstr(uaddr, kaddr, len, done);
378 1.1 maxv }
379 1.1 maxv
380 1.1 maxv int
381 1.1 maxv kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
382 1.1 maxv {
383 1.1 maxv kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
384 1.1 maxv return copyoutstr(kaddr, uaddr, len, done);
385 1.1 maxv }
386 1.1 maxv
387 1.1 maxv /* -------------------------------------------------------------------------- */
388 1.1 maxv
389 1.1 maxv #undef atomic_add_32
390 1.1 maxv #undef atomic_add_int
391 1.1 maxv #undef atomic_add_long
392 1.1 maxv #undef atomic_add_ptr
393 1.1 maxv #undef atomic_add_64
394 1.1 maxv #undef atomic_add_32_nv
395 1.1 maxv #undef atomic_add_int_nv
396 1.1 maxv #undef atomic_add_long_nv
397 1.1 maxv #undef atomic_add_ptr_nv
398 1.1 maxv #undef atomic_add_64_nv
399 1.1 maxv #undef atomic_and_32
400 1.1 maxv #undef atomic_and_uint
401 1.1 maxv #undef atomic_and_ulong
402 1.1 maxv #undef atomic_and_64
403 1.1 maxv #undef atomic_and_32_nv
404 1.1 maxv #undef atomic_and_uint_nv
405 1.1 maxv #undef atomic_and_ulong_nv
406 1.1 maxv #undef atomic_and_64_nv
407 1.1 maxv #undef atomic_or_32
408 1.1 maxv #undef atomic_or_uint
409 1.1 maxv #undef atomic_or_ulong
410 1.1 maxv #undef atomic_or_64
411 1.1 maxv #undef atomic_or_32_nv
412 1.1 maxv #undef atomic_or_uint_nv
413 1.1 maxv #undef atomic_or_ulong_nv
414 1.1 maxv #undef atomic_or_64_nv
415 1.1 maxv #undef atomic_cas_32
416 1.1 maxv #undef atomic_cas_uint
417 1.1 maxv #undef atomic_cas_ulong
418 1.1 maxv #undef atomic_cas_ptr
419 1.1 maxv #undef atomic_cas_64
420 1.1 maxv #undef atomic_cas_32_ni
421 1.1 maxv #undef atomic_cas_uint_ni
422 1.1 maxv #undef atomic_cas_ulong_ni
423 1.1 maxv #undef atomic_cas_ptr_ni
424 1.1 maxv #undef atomic_cas_64_ni
425 1.1 maxv #undef atomic_swap_32
426 1.1 maxv #undef atomic_swap_uint
427 1.1 maxv #undef atomic_swap_ulong
428 1.1 maxv #undef atomic_swap_ptr
429 1.1 maxv #undef atomic_swap_64
430 1.1 maxv #undef atomic_dec_32
431 1.1 maxv #undef atomic_dec_uint
432 1.1 maxv #undef atomic_dec_ulong
433 1.1 maxv #undef atomic_dec_ptr
434 1.1 maxv #undef atomic_dec_64
435 1.1 maxv #undef atomic_dec_32_nv
436 1.1 maxv #undef atomic_dec_uint_nv
437 1.1 maxv #undef atomic_dec_ulong_nv
438 1.1 maxv #undef atomic_dec_ptr_nv
439 1.1 maxv #undef atomic_dec_64_nv
440 1.1 maxv #undef atomic_inc_32
441 1.1 maxv #undef atomic_inc_uint
442 1.1 maxv #undef atomic_inc_ulong
443 1.1 maxv #undef atomic_inc_ptr
444 1.1 maxv #undef atomic_inc_64
445 1.1 maxv #undef atomic_inc_32_nv
446 1.1 maxv #undef atomic_inc_uint_nv
447 1.1 maxv #undef atomic_inc_ulong_nv
448 1.1 maxv #undef atomic_inc_ptr_nv
449 1.1 maxv #undef atomic_inc_64_nv
450 1.1 maxv
451 1.1 maxv #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
452 1.1 maxv void atomic_add_##name(volatile targ1 *, targ2); \
453 1.1 maxv void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
454 1.1 maxv void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
455 1.1 maxv { \
456 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
457 1.1 maxv __RET_ADDR); \
458 1.1 maxv atomic_add_##name(ptr, val); \
459 1.1 maxv } \
460 1.1 maxv tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
461 1.1 maxv tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
462 1.1 maxv tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
463 1.1 maxv { \
464 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
465 1.1 maxv __RET_ADDR); \
466 1.1 maxv return atomic_add_##name##_nv(ptr, val); \
467 1.1 maxv }
468 1.1 maxv
469 1.1 maxv #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
470 1.1 maxv void atomic_and_##name(volatile targ1 *, targ2); \
471 1.1 maxv void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
472 1.1 maxv void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
473 1.1 maxv { \
474 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
475 1.1 maxv __RET_ADDR); \
476 1.1 maxv atomic_and_##name(ptr, val); \
477 1.1 maxv } \
478 1.1 maxv tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
479 1.1 maxv tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
480 1.1 maxv tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
481 1.1 maxv { \
482 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
483 1.1 maxv __RET_ADDR); \
484 1.1 maxv return atomic_and_##name##_nv(ptr, val); \
485 1.1 maxv }
486 1.1 maxv
487 1.1 maxv #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
488 1.1 maxv void atomic_or_##name(volatile targ1 *, targ2); \
489 1.1 maxv void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
490 1.1 maxv void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
491 1.1 maxv { \
492 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
493 1.1 maxv __RET_ADDR); \
494 1.1 maxv atomic_or_##name(ptr, val); \
495 1.1 maxv } \
496 1.1 maxv tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
497 1.1 maxv tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
498 1.1 maxv tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
499 1.1 maxv { \
500 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
501 1.1 maxv __RET_ADDR); \
502 1.1 maxv return atomic_or_##name##_nv(ptr, val); \
503 1.1 maxv }
504 1.1 maxv
505 1.1 maxv #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
506 1.1 maxv tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
507 1.1 maxv tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
508 1.1 maxv tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
509 1.1 maxv { \
510 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
511 1.1 maxv __RET_ADDR); \
512 1.1 maxv return atomic_cas_##name(ptr, exp, new); \
513 1.1 maxv } \
514 1.1 maxv tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
515 1.1 maxv tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
516 1.1 maxv tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
517 1.1 maxv { \
518 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
519 1.1 maxv __RET_ADDR); \
520 1.1 maxv return atomic_cas_##name##_ni(ptr, exp, new); \
521 1.1 maxv }
522 1.1 maxv
523 1.1 maxv #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
524 1.1 maxv tret atomic_swap_##name(volatile targ1 *, targ2); \
525 1.1 maxv tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \
526 1.1 maxv tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
527 1.1 maxv { \
528 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
529 1.1 maxv __RET_ADDR); \
530 1.1 maxv return atomic_swap_##name(ptr, val); \
531 1.1 maxv }
532 1.1 maxv
533 1.1 maxv #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
534 1.1 maxv void atomic_dec_##name(volatile targ1 *); \
535 1.1 maxv void kcsan_atomic_dec_##name(volatile targ1 *); \
536 1.1 maxv void kcsan_atomic_dec_##name(volatile targ1 *ptr) \
537 1.1 maxv { \
538 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
539 1.1 maxv __RET_ADDR); \
540 1.1 maxv atomic_dec_##name(ptr); \
541 1.1 maxv } \
542 1.1 maxv tret atomic_dec_##name##_nv(volatile targ1 *); \
543 1.1 maxv tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \
544 1.1 maxv tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
545 1.1 maxv { \
546 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
547 1.1 maxv __RET_ADDR); \
548 1.1 maxv return atomic_dec_##name##_nv(ptr); \
549 1.1 maxv }
550 1.1 maxv
551 1.1 maxv #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
552 1.1 maxv void atomic_inc_##name(volatile targ1 *); \
553 1.1 maxv void kcsan_atomic_inc_##name(volatile targ1 *); \
554 1.1 maxv void kcsan_atomic_inc_##name(volatile targ1 *ptr) \
555 1.1 maxv { \
556 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
557 1.1 maxv __RET_ADDR); \
558 1.1 maxv atomic_inc_##name(ptr); \
559 1.1 maxv } \
560 1.1 maxv tret atomic_inc_##name##_nv(volatile targ1 *); \
561 1.1 maxv tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \
562 1.1 maxv tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
563 1.1 maxv { \
564 1.1 maxv kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
565 1.1 maxv __RET_ADDR); \
566 1.1 maxv return atomic_inc_##name##_nv(ptr); \
567 1.1 maxv }
568 1.1 maxv
569 1.1 maxv CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
570 1.1 maxv CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
571 1.1 maxv CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
572 1.1 maxv CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
573 1.1 maxv CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
574 1.1 maxv
575 1.1 maxv CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
576 1.1 maxv CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
577 1.1 maxv CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
578 1.1 maxv CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
579 1.1 maxv
580 1.1 maxv CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
581 1.1 maxv CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
582 1.1 maxv CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
583 1.1 maxv CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
584 1.1 maxv
585 1.1 maxv CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
586 1.1 maxv CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
587 1.1 maxv CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
588 1.1 maxv CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
589 1.1 maxv CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
590 1.1 maxv
591 1.1 maxv CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
592 1.1 maxv CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
593 1.1 maxv CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
594 1.1 maxv CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
595 1.1 maxv CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
596 1.1 maxv
597 1.1 maxv CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
598 1.1 maxv CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
599 1.1 maxv CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
600 1.1 maxv CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
601 1.1 maxv CSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
602 1.1 maxv
603 1.1 maxv CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
604 1.1 maxv CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
605 1.1 maxv CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
606 1.1 maxv CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
607 1.1 maxv CSAN_ATOMIC_FUNC_INC(ptr, void *, void);
608 1.1 maxv
609 1.6 maxv void
610 1.6 maxv kcsan_atomic_load(const volatile void *p, void *v, int size)
611 1.6 maxv {
612 1.8 maxv kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR);
613 1.6 maxv switch (size) {
614 1.6 maxv case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break;
615 1.6 maxv case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break;
616 1.6 maxv case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break;
617 1.6 maxv case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break;
618 1.6 maxv }
619 1.6 maxv }
620 1.6 maxv
621 1.6 maxv void
622 1.6 maxv kcsan_atomic_store(volatile void *p, const void *v, int size)
623 1.6 maxv {
624 1.8 maxv kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR);
625 1.6 maxv switch (size) {
626 1.6 maxv case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break;
627 1.6 maxv case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break;
628 1.6 maxv case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break;
629 1.6 maxv case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break;
630 1.6 maxv }
631 1.6 maxv }
632 1.6 maxv
633 1.1 maxv /* -------------------------------------------------------------------------- */
634 1.1 maxv
635 1.1 maxv #include <sys/bus.h>
636 1.1 maxv
637 1.1 maxv #undef bus_space_read_multi_1
638 1.1 maxv #undef bus_space_read_multi_2
639 1.1 maxv #undef bus_space_read_multi_4
640 1.1 maxv #undef bus_space_read_multi_8
641 1.1 maxv #undef bus_space_read_multi_stream_1
642 1.1 maxv #undef bus_space_read_multi_stream_2
643 1.1 maxv #undef bus_space_read_multi_stream_4
644 1.1 maxv #undef bus_space_read_multi_stream_8
645 1.1 maxv #undef bus_space_read_region_1
646 1.1 maxv #undef bus_space_read_region_2
647 1.1 maxv #undef bus_space_read_region_4
648 1.1 maxv #undef bus_space_read_region_8
649 1.1 maxv #undef bus_space_read_region_stream_1
650 1.1 maxv #undef bus_space_read_region_stream_2
651 1.1 maxv #undef bus_space_read_region_stream_4
652 1.1 maxv #undef bus_space_read_region_stream_8
653 1.1 maxv #undef bus_space_write_multi_1
654 1.1 maxv #undef bus_space_write_multi_2
655 1.1 maxv #undef bus_space_write_multi_4
656 1.1 maxv #undef bus_space_write_multi_8
657 1.1 maxv #undef bus_space_write_multi_stream_1
658 1.1 maxv #undef bus_space_write_multi_stream_2
659 1.1 maxv #undef bus_space_write_multi_stream_4
660 1.1 maxv #undef bus_space_write_multi_stream_8
661 1.1 maxv #undef bus_space_write_region_1
662 1.1 maxv #undef bus_space_write_region_2
663 1.1 maxv #undef bus_space_write_region_4
664 1.1 maxv #undef bus_space_write_region_8
665 1.1 maxv #undef bus_space_write_region_stream_1
666 1.1 maxv #undef bus_space_write_region_stream_2
667 1.1 maxv #undef bus_space_write_region_stream_4
668 1.1 maxv #undef bus_space_write_region_stream_8
669 1.1 maxv
670 1.1 maxv #define CSAN_BUS_READ_FUNC(bytes, bits) \
671 1.1 maxv void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
672 1.1 maxv bus_size_t, uint##bits##_t *, bus_size_t); \
673 1.1 maxv void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
674 1.1 maxv bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
675 1.1 maxv void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
676 1.1 maxv bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
677 1.1 maxv bus_size_t count) \
678 1.1 maxv { \
679 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
680 1.1 maxv false, false, __RET_ADDR); \
681 1.1 maxv bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
682 1.1 maxv } \
683 1.1 maxv void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
684 1.1 maxv bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
685 1.1 maxv void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
686 1.1 maxv bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
687 1.1 maxv void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
688 1.1 maxv bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
689 1.1 maxv bus_size_t count) \
690 1.1 maxv { \
691 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
692 1.1 maxv false, false, __RET_ADDR); \
693 1.1 maxv bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
694 1.1 maxv } \
695 1.1 maxv void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
696 1.1 maxv bus_size_t, uint##bits##_t *, bus_size_t); \
697 1.1 maxv void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \
698 1.1 maxv bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
699 1.1 maxv void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
700 1.1 maxv bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
701 1.1 maxv bus_size_t count) \
702 1.1 maxv { \
703 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
704 1.1 maxv false, false, __RET_ADDR); \
705 1.1 maxv bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
706 1.1 maxv } \
707 1.1 maxv void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
708 1.1 maxv bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
709 1.1 maxv void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
710 1.1 maxv bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
711 1.1 maxv void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
712 1.1 maxv bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
713 1.1 maxv bus_size_t count) \
714 1.1 maxv { \
715 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
716 1.1 maxv false, false, __RET_ADDR); \
717 1.1 maxv bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
718 1.1 maxv }
719 1.1 maxv
720 1.1 maxv #define CSAN_BUS_WRITE_FUNC(bytes, bits) \
721 1.1 maxv void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
722 1.1 maxv bus_size_t, const uint##bits##_t *, bus_size_t); \
723 1.1 maxv void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
724 1.1 maxv bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
725 1.1 maxv void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
726 1.1 maxv bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
727 1.1 maxv bus_size_t count) \
728 1.1 maxv { \
729 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
730 1.1 maxv true, false, __RET_ADDR); \
731 1.1 maxv bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
732 1.1 maxv } \
733 1.1 maxv void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
734 1.1 maxv bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
735 1.1 maxv void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
736 1.1 maxv bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
737 1.1 maxv void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
738 1.1 maxv bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
739 1.1 maxv bus_size_t count) \
740 1.1 maxv { \
741 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
742 1.1 maxv true, false, __RET_ADDR); \
743 1.1 maxv bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
744 1.1 maxv } \
745 1.1 maxv void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
746 1.1 maxv bus_size_t, const uint##bits##_t *, bus_size_t); \
747 1.1 maxv void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \
748 1.1 maxv bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
749 1.1 maxv void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
750 1.1 maxv bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
751 1.1 maxv bus_size_t count) \
752 1.1 maxv { \
753 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
754 1.1 maxv true, false, __RET_ADDR); \
755 1.1 maxv bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
756 1.1 maxv } \
757 1.1 maxv void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
758 1.1 maxv bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
759 1.1 maxv void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
760 1.1 maxv bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
761 1.1 maxv void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
762 1.1 maxv bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
763 1.1 maxv bus_size_t count) \
764 1.1 maxv { \
765 1.1 maxv kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
766 1.1 maxv true, false, __RET_ADDR); \
767 1.1 maxv bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
768 1.1 maxv }
769 1.1 maxv
770 1.1 maxv CSAN_BUS_READ_FUNC(1, 8)
771 1.1 maxv CSAN_BUS_READ_FUNC(2, 16)
772 1.1 maxv CSAN_BUS_READ_FUNC(4, 32)
773 1.1 maxv CSAN_BUS_READ_FUNC(8, 64)
774 1.1 maxv
775 1.1 maxv CSAN_BUS_WRITE_FUNC(1, 8)
776 1.1 maxv CSAN_BUS_WRITE_FUNC(2, 16)
777 1.1 maxv CSAN_BUS_WRITE_FUNC(4, 32)
778 1.1 maxv CSAN_BUS_WRITE_FUNC(8, 64)
779