subr_asan.c revision 1.9.2.4 1 1.9.2.3 martin /* $NetBSD: subr_asan.c,v 1.9.2.4 2020/04/21 18:42:42 martin Exp $ */
2 1.9.2.2 christos
3 1.9.2.2 christos /*
4 1.9.2.3 martin * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
5 1.9.2.2 christos * All rights reserved.
6 1.9.2.2 christos *
7 1.9.2.2 christos * This code is derived from software contributed to The NetBSD Foundation
8 1.9.2.2 christos * by Maxime Villard.
9 1.9.2.2 christos *
10 1.9.2.2 christos * Redistribution and use in source and binary forms, with or without
11 1.9.2.2 christos * modification, are permitted provided that the following conditions
12 1.9.2.2 christos * are met:
13 1.9.2.2 christos * 1. Redistributions of source code must retain the above copyright
14 1.9.2.2 christos * notice, this list of conditions and the following disclaimer.
15 1.9.2.2 christos * 2. Redistributions in binary form must reproduce the above copyright
16 1.9.2.2 christos * notice, this list of conditions and the following disclaimer in the
17 1.9.2.2 christos * documentation and/or other materials provided with the distribution.
18 1.9.2.2 christos *
19 1.9.2.2 christos * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.9.2.2 christos * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.9.2.2 christos * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.9.2.2 christos * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.9.2.2 christos * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.9.2.2 christos * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.9.2.2 christos * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.9.2.2 christos * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.9.2.2 christos * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.9.2.2 christos * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.9.2.2 christos * POSSIBILITY OF SUCH DAMAGE.
30 1.9.2.2 christos */
31 1.9.2.2 christos
32 1.9.2.2 christos #include <sys/cdefs.h>
33 1.9.2.3 martin __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.9.2.4 2020/04/21 18:42:42 martin Exp $");
34 1.9.2.2 christos
35 1.9.2.2 christos #include <sys/param.h>
36 1.9.2.2 christos #include <sys/device.h>
37 1.9.2.2 christos #include <sys/kernel.h>
38 1.9.2.2 christos #include <sys/param.h>
39 1.9.2.2 christos #include <sys/conf.h>
40 1.9.2.2 christos #include <sys/systm.h>
41 1.9.2.2 christos #include <sys/types.h>
42 1.9.2.2 christos #include <sys/asan.h>
43 1.9.2.2 christos
44 1.9.2.2 christos #include <uvm/uvm.h>
45 1.9.2.2 christos
46 1.9.2.3 martin #ifdef KASAN_PANIC
47 1.9.2.3 martin #define REPORT panic
48 1.9.2.3 martin #else
49 1.9.2.3 martin #define REPORT printf
50 1.9.2.3 martin #endif
51 1.9.2.3 martin
52 1.9.2.2 christos /* ASAN constants. Part of the compiler ABI. */
53 1.9.2.2 christos #define KASAN_SHADOW_SCALE_SHIFT 3
54 1.9.2.2 christos #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
55 1.9.2.2 christos #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1)
56 1.9.2.3 martin #define KASAN_ALLOCA_SCALE_SIZE 32
57 1.9.2.2 christos
58 1.9.2.2 christos /* The MD code. */
59 1.9.2.2 christos #include <machine/asan.h>
60 1.9.2.2 christos
61 1.9.2.2 christos /* ASAN ABI version. */
62 1.9.2.2 christos #if defined(__clang__) && (__clang_major__ - 0 >= 6)
63 1.9.2.2 christos #define ASAN_ABI_VERSION 8
64 1.9.2.2 christos #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
65 1.9.2.2 christos #define ASAN_ABI_VERSION 8
66 1.9.2.2 christos #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
67 1.9.2.2 christos #define ASAN_ABI_VERSION 6
68 1.9.2.2 christos #else
69 1.9.2.2 christos #error "Unsupported compiler version"
70 1.9.2.2 christos #endif
71 1.9.2.2 christos
72 1.9.2.2 christos #define __RET_ADDR (unsigned long)__builtin_return_address(0)
73 1.9.2.2 christos
74 1.9.2.2 christos /* Global variable descriptor. Part of the compiler ABI. */
75 1.9.2.2 christos struct __asan_global_source_location {
76 1.9.2.2 christos const char *filename;
77 1.9.2.2 christos int line_no;
78 1.9.2.2 christos int column_no;
79 1.9.2.2 christos };
80 1.9.2.2 christos struct __asan_global {
81 1.9.2.2 christos const void *beg; /* address of the global variable */
82 1.9.2.2 christos size_t size; /* size of the global variable */
83 1.9.2.2 christos size_t size_with_redzone; /* size with the redzone */
84 1.9.2.2 christos const void *name; /* name of the variable */
85 1.9.2.2 christos const void *module_name; /* name of the module where the var is declared */
86 1.9.2.2 christos unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
87 1.9.2.2 christos struct __asan_global_source_location *location;
88 1.9.2.2 christos #if ASAN_ABI_VERSION >= 7
89 1.9.2.2 christos uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
90 1.9.2.2 christos #endif
91 1.9.2.2 christos };
92 1.9.2.2 christos
93 1.9.2.2 christos static bool kasan_enabled __read_mostly = false;
94 1.9.2.2 christos
95 1.9.2.2 christos /* -------------------------------------------------------------------------- */
96 1.9.2.2 christos
97 1.9.2.2 christos void
98 1.9.2.2 christos kasan_shadow_map(void *addr, size_t size)
99 1.9.2.2 christos {
100 1.9.2.2 christos size_t sz, npages, i;
101 1.9.2.2 christos vaddr_t sva, eva;
102 1.9.2.2 christos
103 1.9.2.2 christos KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
104 1.9.2.2 christos
105 1.9.2.2 christos sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE;
106 1.9.2.2 christos
107 1.9.2.2 christos sva = (vaddr_t)kasan_md_addr_to_shad(addr);
108 1.9.2.2 christos eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz;
109 1.9.2.2 christos
110 1.9.2.2 christos sva = rounddown(sva, PAGE_SIZE);
111 1.9.2.2 christos eva = roundup(eva, PAGE_SIZE);
112 1.9.2.2 christos
113 1.9.2.2 christos npages = (eva - sva) / PAGE_SIZE;
114 1.9.2.2 christos
115 1.9.2.2 christos KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END);
116 1.9.2.2 christos
117 1.9.2.2 christos for (i = 0; i < npages; i++) {
118 1.9.2.2 christos kasan_md_shadow_map_page(sva + i * PAGE_SIZE);
119 1.9.2.2 christos }
120 1.9.2.2 christos }
121 1.9.2.2 christos
122 1.9.2.2 christos static void
123 1.9.2.2 christos kasan_ctors(void)
124 1.9.2.2 christos {
125 1.9.2.2 christos extern uint64_t __CTOR_LIST__, __CTOR_END__;
126 1.9.2.2 christos size_t nentries, i;
127 1.9.2.2 christos uint64_t *ptr;
128 1.9.2.2 christos
129 1.9.2.2 christos nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) /
130 1.9.2.2 christos sizeof(uintptr_t);
131 1.9.2.2 christos
132 1.9.2.2 christos ptr = &__CTOR_LIST__;
133 1.9.2.2 christos for (i = 0; i < nentries; i++) {
134 1.9.2.2 christos void (*func)(void);
135 1.9.2.2 christos
136 1.9.2.2 christos func = (void *)(*ptr);
137 1.9.2.2 christos (*func)();
138 1.9.2.2 christos
139 1.9.2.2 christos ptr++;
140 1.9.2.2 christos }
141 1.9.2.2 christos }
142 1.9.2.2 christos
143 1.9.2.2 christos void
144 1.9.2.2 christos kasan_early_init(void *stack)
145 1.9.2.2 christos {
146 1.9.2.2 christos kasan_md_early_init(stack);
147 1.9.2.2 christos }
148 1.9.2.2 christos
149 1.9.2.2 christos void
150 1.9.2.2 christos kasan_init(void)
151 1.9.2.2 christos {
152 1.9.2.2 christos /* MD initialization. */
153 1.9.2.2 christos kasan_md_init();
154 1.9.2.2 christos
155 1.9.2.2 christos /* Now officially enabled. */
156 1.9.2.2 christos kasan_enabled = true;
157 1.9.2.2 christos
158 1.9.2.2 christos /* Call the ASAN constructors. */
159 1.9.2.2 christos kasan_ctors();
160 1.9.2.2 christos }
161 1.9.2.2 christos
162 1.9.2.2 christos static inline const char *
163 1.9.2.2 christos kasan_code_name(uint8_t code)
164 1.9.2.2 christos {
165 1.9.2.2 christos switch (code) {
166 1.9.2.2 christos case KASAN_GENERIC_REDZONE:
167 1.9.2.2 christos return "GenericRedZone";
168 1.9.2.2 christos case KASAN_MALLOC_REDZONE:
169 1.9.2.2 christos return "MallocRedZone";
170 1.9.2.2 christos case KASAN_KMEM_REDZONE:
171 1.9.2.2 christos return "KmemRedZone";
172 1.9.2.2 christos case KASAN_POOL_REDZONE:
173 1.9.2.2 christos return "PoolRedZone";
174 1.9.2.2 christos case KASAN_POOL_FREED:
175 1.9.2.2 christos return "PoolUseAfterFree";
176 1.9.2.2 christos case 1 ... 7:
177 1.9.2.2 christos return "RedZonePartial";
178 1.9.2.2 christos case KASAN_STACK_LEFT:
179 1.9.2.2 christos return "StackLeft";
180 1.9.2.3 martin case KASAN_STACK_MID:
181 1.9.2.3 martin return "StackMiddle";
182 1.9.2.2 christos case KASAN_STACK_RIGHT:
183 1.9.2.2 christos return "StackRight";
184 1.9.2.3 martin case KASAN_USE_AFTER_RET:
185 1.9.2.3 martin return "UseAfterRet";
186 1.9.2.2 christos case KASAN_USE_AFTER_SCOPE:
187 1.9.2.2 christos return "UseAfterScope";
188 1.9.2.2 christos default:
189 1.9.2.2 christos return "Unknown";
190 1.9.2.2 christos }
191 1.9.2.2 christos }
192 1.9.2.2 christos
193 1.9.2.2 christos static void
194 1.9.2.2 christos kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
195 1.9.2.2 christos uint8_t code)
196 1.9.2.2 christos {
197 1.9.2.3 martin REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s,"
198 1.9.2.2 christos " %s]\n",
199 1.9.2.2 christos (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""),
200 1.9.2.2 christos (write ? "write" : "read"), kasan_code_name(code));
201 1.9.2.2 christos kasan_md_unwind();
202 1.9.2.2 christos }
203 1.9.2.2 christos
204 1.9.2.2 christos static __always_inline void
205 1.9.2.2 christos kasan_shadow_1byte_markvalid(unsigned long addr)
206 1.9.2.2 christos {
207 1.9.2.2 christos int8_t *byte = kasan_md_addr_to_shad((void *)addr);
208 1.9.2.2 christos int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
209 1.9.2.2 christos
210 1.9.2.2 christos *byte = last;
211 1.9.2.2 christos }
212 1.9.2.2 christos
213 1.9.2.2 christos static __always_inline void
214 1.9.2.2 christos kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
215 1.9.2.2 christos {
216 1.9.2.2 christos size_t i;
217 1.9.2.2 christos
218 1.9.2.2 christos for (i = 0; i < size; i++) {
219 1.9.2.2 christos kasan_shadow_1byte_markvalid((unsigned long)addr+i);
220 1.9.2.2 christos }
221 1.9.2.2 christos }
222 1.9.2.2 christos
223 1.9.2.2 christos static __always_inline void
224 1.9.2.2 christos kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
225 1.9.2.2 christos {
226 1.9.2.2 christos void *shad;
227 1.9.2.2 christos
228 1.9.2.2 christos if (__predict_false(size == 0))
229 1.9.2.2 christos return;
230 1.9.2.2 christos if (__predict_false(kasan_md_unsupported((vaddr_t)addr)))
231 1.9.2.2 christos return;
232 1.9.2.2 christos
233 1.9.2.2 christos KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
234 1.9.2.2 christos KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
235 1.9.2.2 christos
236 1.9.2.2 christos shad = (void *)kasan_md_addr_to_shad(addr);
237 1.9.2.2 christos size = size >> KASAN_SHADOW_SCALE_SHIFT;
238 1.9.2.2 christos
239 1.9.2.2 christos __builtin_memset(shad, code, size);
240 1.9.2.2 christos }
241 1.9.2.2 christos
242 1.9.2.2 christos void
243 1.9.2.2 christos kasan_add_redzone(size_t *size)
244 1.9.2.2 christos {
245 1.9.2.2 christos *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE);
246 1.9.2.2 christos *size += KASAN_SHADOW_SCALE_SIZE;
247 1.9.2.2 christos }
248 1.9.2.2 christos
249 1.9.2.2 christos void
250 1.9.2.2 christos kasan_softint(struct lwp *l)
251 1.9.2.2 christos {
252 1.9.2.2 christos const void *stk = (const void *)uvm_lwp_getuarea(l);
253 1.9.2.2 christos
254 1.9.2.2 christos kasan_shadow_Nbyte_fill(stk, USPACE, 0);
255 1.9.2.2 christos }
256 1.9.2.2 christos
257 1.9.2.2 christos /*
258 1.9.2.2 christos * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
259 1.9.2.2 christos * and the rest as invalid. There are generally two use cases:
260 1.9.2.2 christos *
261 1.9.2.2 christos * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
262 1.9.2.2 christos * the redzone at the end of the buffer as invalid.
263 1.9.2.2 christos *
264 1.9.2.2 christos * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
265 1.9.2.2 christos */
266 1.9.2.2 christos void
267 1.9.2.2 christos kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code)
268 1.9.2.2 christos {
269 1.9.2.2 christos size_t i, n, redz;
270 1.9.2.2 christos int8_t *shad;
271 1.9.2.2 christos
272 1.9.2.2 christos KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
273 1.9.2.2 christos redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE);
274 1.9.2.2 christos KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0);
275 1.9.2.2 christos shad = kasan_md_addr_to_shad(addr);
276 1.9.2.2 christos
277 1.9.2.2 christos /* Chunks of 8 bytes, valid. */
278 1.9.2.2 christos n = size / KASAN_SHADOW_SCALE_SIZE;
279 1.9.2.2 christos for (i = 0; i < n; i++) {
280 1.9.2.2 christos *shad++ = 0;
281 1.9.2.2 christos }
282 1.9.2.2 christos
283 1.9.2.2 christos /* Possibly one chunk, mid. */
284 1.9.2.2 christos if ((size & KASAN_SHADOW_MASK) != 0) {
285 1.9.2.2 christos *shad++ = (size & KASAN_SHADOW_MASK);
286 1.9.2.2 christos }
287 1.9.2.2 christos
288 1.9.2.2 christos /* Chunks of 8 bytes, invalid. */
289 1.9.2.2 christos n = redz / KASAN_SHADOW_SCALE_SIZE;
290 1.9.2.2 christos for (i = 0; i < n; i++) {
291 1.9.2.2 christos *shad++ = code;
292 1.9.2.2 christos }
293 1.9.2.2 christos }
294 1.9.2.2 christos
295 1.9.2.2 christos /* -------------------------------------------------------------------------- */
296 1.9.2.2 christos
297 1.9.2.2 christos #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
298 1.9.2.2 christos (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
299 1.9.2.2 christos ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
300 1.9.2.2 christos
301 1.9.2.2 christos static __always_inline bool
302 1.9.2.2 christos kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
303 1.9.2.2 christos {
304 1.9.2.2 christos int8_t *byte = kasan_md_addr_to_shad((void *)addr);
305 1.9.2.2 christos int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
306 1.9.2.2 christos
307 1.9.2.2 christos if (__predict_true(*byte == 0 || last <= *byte)) {
308 1.9.2.2 christos return true;
309 1.9.2.2 christos }
310 1.9.2.2 christos *code = *byte;
311 1.9.2.2 christos return false;
312 1.9.2.2 christos }
313 1.9.2.2 christos
314 1.9.2.2 christos static __always_inline bool
315 1.9.2.2 christos kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
316 1.9.2.2 christos {
317 1.9.2.2 christos int8_t *byte, last;
318 1.9.2.2 christos
319 1.9.2.2 christos if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
320 1.9.2.2 christos return (kasan_shadow_1byte_isvalid(addr, code) &&
321 1.9.2.2 christos kasan_shadow_1byte_isvalid(addr+1, code));
322 1.9.2.2 christos }
323 1.9.2.2 christos
324 1.9.2.2 christos byte = kasan_md_addr_to_shad((void *)addr);
325 1.9.2.2 christos last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
326 1.9.2.2 christos
327 1.9.2.2 christos if (__predict_true(*byte == 0 || last <= *byte)) {
328 1.9.2.2 christos return true;
329 1.9.2.2 christos }
330 1.9.2.2 christos *code = *byte;
331 1.9.2.2 christos return false;
332 1.9.2.2 christos }
333 1.9.2.2 christos
334 1.9.2.2 christos static __always_inline bool
335 1.9.2.2 christos kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
336 1.9.2.2 christos {
337 1.9.2.2 christos int8_t *byte, last;
338 1.9.2.2 christos
339 1.9.2.2 christos if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
340 1.9.2.2 christos return (kasan_shadow_2byte_isvalid(addr, code) &&
341 1.9.2.2 christos kasan_shadow_2byte_isvalid(addr+2, code));
342 1.9.2.2 christos }
343 1.9.2.2 christos
344 1.9.2.2 christos byte = kasan_md_addr_to_shad((void *)addr);
345 1.9.2.2 christos last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
346 1.9.2.2 christos
347 1.9.2.2 christos if (__predict_true(*byte == 0 || last <= *byte)) {
348 1.9.2.2 christos return true;
349 1.9.2.2 christos }
350 1.9.2.2 christos *code = *byte;
351 1.9.2.2 christos return false;
352 1.9.2.2 christos }
353 1.9.2.2 christos
354 1.9.2.2 christos static __always_inline bool
355 1.9.2.2 christos kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
356 1.9.2.2 christos {
357 1.9.2.2 christos int8_t *byte, last;
358 1.9.2.2 christos
359 1.9.2.2 christos if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
360 1.9.2.2 christos return (kasan_shadow_4byte_isvalid(addr, code) &&
361 1.9.2.2 christos kasan_shadow_4byte_isvalid(addr+4, code));
362 1.9.2.2 christos }
363 1.9.2.2 christos
364 1.9.2.2 christos byte = kasan_md_addr_to_shad((void *)addr);
365 1.9.2.2 christos last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
366 1.9.2.2 christos
367 1.9.2.2 christos if (__predict_true(*byte == 0 || last <= *byte)) {
368 1.9.2.2 christos return true;
369 1.9.2.2 christos }
370 1.9.2.2 christos *code = *byte;
371 1.9.2.2 christos return false;
372 1.9.2.2 christos }
373 1.9.2.2 christos
374 1.9.2.2 christos static __always_inline bool
375 1.9.2.2 christos kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
376 1.9.2.2 christos {
377 1.9.2.2 christos size_t i;
378 1.9.2.2 christos
379 1.9.2.2 christos for (i = 0; i < size; i++) {
380 1.9.2.2 christos if (!kasan_shadow_1byte_isvalid(addr+i, code))
381 1.9.2.2 christos return false;
382 1.9.2.2 christos }
383 1.9.2.2 christos
384 1.9.2.2 christos return true;
385 1.9.2.2 christos }
386 1.9.2.2 christos
387 1.9.2.2 christos static __always_inline void
388 1.9.2.2 christos kasan_shadow_check(unsigned long addr, size_t size, bool write,
389 1.9.2.2 christos unsigned long retaddr)
390 1.9.2.2 christos {
391 1.9.2.2 christos uint8_t code;
392 1.9.2.2 christos bool valid;
393 1.9.2.2 christos
394 1.9.2.2 christos if (__predict_false(!kasan_enabled))
395 1.9.2.2 christos return;
396 1.9.2.2 christos if (__predict_false(size == 0))
397 1.9.2.2 christos return;
398 1.9.2.2 christos if (__predict_false(kasan_md_unsupported(addr)))
399 1.9.2.2 christos return;
400 1.9.2.2 christos
401 1.9.2.2 christos if (__builtin_constant_p(size)) {
402 1.9.2.2 christos switch (size) {
403 1.9.2.2 christos case 1:
404 1.9.2.2 christos valid = kasan_shadow_1byte_isvalid(addr, &code);
405 1.9.2.2 christos break;
406 1.9.2.2 christos case 2:
407 1.9.2.2 christos valid = kasan_shadow_2byte_isvalid(addr, &code);
408 1.9.2.2 christos break;
409 1.9.2.2 christos case 4:
410 1.9.2.2 christos valid = kasan_shadow_4byte_isvalid(addr, &code);
411 1.9.2.2 christos break;
412 1.9.2.2 christos case 8:
413 1.9.2.2 christos valid = kasan_shadow_8byte_isvalid(addr, &code);
414 1.9.2.2 christos break;
415 1.9.2.2 christos default:
416 1.9.2.2 christos valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
417 1.9.2.2 christos break;
418 1.9.2.2 christos }
419 1.9.2.2 christos } else {
420 1.9.2.2 christos valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
421 1.9.2.2 christos }
422 1.9.2.2 christos
423 1.9.2.2 christos if (__predict_false(!valid)) {
424 1.9.2.2 christos kasan_report(addr, size, write, retaddr, code);
425 1.9.2.2 christos }
426 1.9.2.2 christos }
427 1.9.2.2 christos
428 1.9.2.2 christos /* -------------------------------------------------------------------------- */
429 1.9.2.2 christos
430 1.9.2.2 christos void *
431 1.9.2.2 christos kasan_memcpy(void *dst, const void *src, size_t len)
432 1.9.2.2 christos {
433 1.9.2.2 christos kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
434 1.9.2.2 christos kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
435 1.9.2.2 christos return __builtin_memcpy(dst, src, len);
436 1.9.2.2 christos }
437 1.9.2.2 christos
438 1.9.2.2 christos int
439 1.9.2.2 christos kasan_memcmp(const void *b1, const void *b2, size_t len)
440 1.9.2.2 christos {
441 1.9.2.2 christos kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
442 1.9.2.2 christos kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
443 1.9.2.2 christos return __builtin_memcmp(b1, b2, len);
444 1.9.2.2 christos }
445 1.9.2.2 christos
446 1.9.2.2 christos void *
447 1.9.2.2 christos kasan_memset(void *b, int c, size_t len)
448 1.9.2.2 christos {
449 1.9.2.2 christos kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
450 1.9.2.2 christos return __builtin_memset(b, c, len);
451 1.9.2.2 christos }
452 1.9.2.2 christos
453 1.9.2.3 martin void *
454 1.9.2.3 martin kasan_memmove(void *dst, const void *src, size_t len)
455 1.9.2.3 martin {
456 1.9.2.3 martin kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
457 1.9.2.3 martin kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
458 1.9.2.3 martin return __builtin_memmove(dst, src, len);
459 1.9.2.3 martin }
460 1.9.2.3 martin
461 1.9.2.2 christos char *
462 1.9.2.2 christos kasan_strcpy(char *dst, const char *src)
463 1.9.2.2 christos {
464 1.9.2.2 christos char *save = dst;
465 1.9.2.2 christos
466 1.9.2.2 christos while (1) {
467 1.9.2.2 christos kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
468 1.9.2.2 christos kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
469 1.9.2.2 christos *dst = *src;
470 1.9.2.2 christos if (*src == '\0')
471 1.9.2.2 christos break;
472 1.9.2.2 christos src++, dst++;
473 1.9.2.2 christos }
474 1.9.2.2 christos
475 1.9.2.2 christos return save;
476 1.9.2.2 christos }
477 1.9.2.2 christos
478 1.9.2.2 christos int
479 1.9.2.2 christos kasan_strcmp(const char *s1, const char *s2)
480 1.9.2.2 christos {
481 1.9.2.2 christos while (1) {
482 1.9.2.2 christos kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
483 1.9.2.2 christos kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
484 1.9.2.2 christos if (*s1 != *s2)
485 1.9.2.2 christos break;
486 1.9.2.2 christos if (*s1 == '\0')
487 1.9.2.2 christos return 0;
488 1.9.2.2 christos s1++, s2++;
489 1.9.2.2 christos }
490 1.9.2.2 christos
491 1.9.2.2 christos return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
492 1.9.2.2 christos }
493 1.9.2.2 christos
494 1.9.2.2 christos size_t
495 1.9.2.2 christos kasan_strlen(const char *str)
496 1.9.2.2 christos {
497 1.9.2.2 christos const char *s;
498 1.9.2.2 christos
499 1.9.2.2 christos s = str;
500 1.9.2.2 christos while (1) {
501 1.9.2.2 christos kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
502 1.9.2.2 christos if (*s == '\0')
503 1.9.2.2 christos break;
504 1.9.2.2 christos s++;
505 1.9.2.2 christos }
506 1.9.2.2 christos
507 1.9.2.2 christos return (s - str);
508 1.9.2.2 christos }
509 1.9.2.2 christos
510 1.9.2.3 martin char *
511 1.9.2.3 martin kasan_strcat(char *dst, const char *src)
512 1.9.2.3 martin {
513 1.9.2.3 martin size_t ldst, lsrc;
514 1.9.2.3 martin
515 1.9.2.3 martin ldst = __builtin_strlen(dst);
516 1.9.2.3 martin lsrc = __builtin_strlen(src);
517 1.9.2.3 martin kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true,
518 1.9.2.3 martin __RET_ADDR);
519 1.9.2.3 martin kasan_shadow_check((unsigned long)src, lsrc + 1, false,
520 1.9.2.3 martin __RET_ADDR);
521 1.9.2.3 martin
522 1.9.2.3 martin return __builtin_strcat(dst, src);
523 1.9.2.3 martin }
524 1.9.2.3 martin
525 1.9.2.3 martin char *
526 1.9.2.3 martin kasan_strchr(const char *s, int c)
527 1.9.2.3 martin {
528 1.9.2.3 martin kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
529 1.9.2.3 martin __RET_ADDR);
530 1.9.2.3 martin return __builtin_strchr(s, c);
531 1.9.2.3 martin }
532 1.9.2.3 martin
533 1.9.2.3 martin char *
534 1.9.2.3 martin kasan_strrchr(const char *s, int c)
535 1.9.2.3 martin {
536 1.9.2.3 martin kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
537 1.9.2.3 martin __RET_ADDR);
538 1.9.2.3 martin return __builtin_strrchr(s, c);
539 1.9.2.3 martin }
540 1.9.2.3 martin
541 1.9.2.2 christos #undef kcopy
542 1.9.2.2 christos #undef copystr
543 1.9.2.2 christos #undef copyinstr
544 1.9.2.2 christos #undef copyoutstr
545 1.9.2.2 christos #undef copyin
546 1.9.2.2 christos
547 1.9.2.2 christos int kasan_kcopy(const void *, void *, size_t);
548 1.9.2.2 christos int kasan_copystr(const void *, void *, size_t, size_t *);
549 1.9.2.2 christos int kasan_copyinstr(const void *, void *, size_t, size_t *);
550 1.9.2.2 christos int kasan_copyoutstr(const void *, void *, size_t, size_t *);
551 1.9.2.2 christos int kasan_copyin(const void *, void *, size_t);
552 1.9.2.2 christos int kcopy(const void *, void *, size_t);
553 1.9.2.2 christos int copystr(const void *, void *, size_t, size_t *);
554 1.9.2.2 christos int copyinstr(const void *, void *, size_t, size_t *);
555 1.9.2.2 christos int copyoutstr(const void *, void *, size_t, size_t *);
556 1.9.2.2 christos int copyin(const void *, void *, size_t);
557 1.9.2.2 christos
558 1.9.2.2 christos int
559 1.9.2.2 christos kasan_kcopy(const void *src, void *dst, size_t len)
560 1.9.2.2 christos {
561 1.9.2.2 christos kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
562 1.9.2.2 christos kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
563 1.9.2.2 christos return kcopy(src, dst, len);
564 1.9.2.2 christos }
565 1.9.2.2 christos
566 1.9.2.2 christos int
567 1.9.2.2 christos kasan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
568 1.9.2.2 christos {
569 1.9.2.2 christos kasan_shadow_check((unsigned long)kdaddr, len, true, __RET_ADDR);
570 1.9.2.2 christos return copystr(kfaddr, kdaddr, len, done);
571 1.9.2.2 christos }
572 1.9.2.2 christos
573 1.9.2.2 christos int
574 1.9.2.2 christos kasan_copyin(const void *uaddr, void *kaddr, size_t len)
575 1.9.2.2 christos {
576 1.9.2.2 christos kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
577 1.9.2.2 christos return copyin(uaddr, kaddr, len);
578 1.9.2.2 christos }
579 1.9.2.2 christos
580 1.9.2.2 christos int
581 1.9.2.2 christos kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
582 1.9.2.2 christos {
583 1.9.2.2 christos kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
584 1.9.2.2 christos return copyinstr(uaddr, kaddr, len, done);
585 1.9.2.2 christos }
586 1.9.2.2 christos
587 1.9.2.2 christos int
588 1.9.2.2 christos kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
589 1.9.2.2 christos {
590 1.9.2.2 christos kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
591 1.9.2.2 christos return copyoutstr(kaddr, uaddr, len, done);
592 1.9.2.2 christos }
593 1.9.2.2 christos
594 1.9.2.2 christos /* -------------------------------------------------------------------------- */
595 1.9.2.2 christos
596 1.9.2.3 martin #undef _ucas_32
597 1.9.2.3 martin #undef _ucas_32_mp
598 1.9.2.3 martin #undef _ucas_64
599 1.9.2.3 martin #undef _ucas_64_mp
600 1.9.2.3 martin #undef _ufetch_8
601 1.9.2.3 martin #undef _ufetch_16
602 1.9.2.3 martin #undef _ufetch_32
603 1.9.2.3 martin #undef _ufetch_64
604 1.9.2.3 martin
605 1.9.2.3 martin int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
606 1.9.2.3 martin int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
607 1.9.2.3 martin int
608 1.9.2.3 martin kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
609 1.9.2.3 martin uint32_t *ret)
610 1.9.2.3 martin {
611 1.9.2.3 martin kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
612 1.9.2.3 martin __RET_ADDR);
613 1.9.2.3 martin return _ucas_32(uaddr, old, new, ret);
614 1.9.2.3 martin }
615 1.9.2.3 martin
616 1.9.2.3 martin #ifdef __HAVE_UCAS_MP
617 1.9.2.3 martin int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
618 1.9.2.3 martin int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
619 1.9.2.3 martin int
620 1.9.2.3 martin kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
621 1.9.2.3 martin uint32_t *ret)
622 1.9.2.3 martin {
623 1.9.2.3 martin kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
624 1.9.2.3 martin __RET_ADDR);
625 1.9.2.3 martin return _ucas_32_mp(uaddr, old, new, ret);
626 1.9.2.3 martin }
627 1.9.2.3 martin #endif
628 1.9.2.3 martin
629 1.9.2.3 martin #ifdef _LP64
630 1.9.2.3 martin int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
631 1.9.2.3 martin int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
632 1.9.2.3 martin int
633 1.9.2.3 martin kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
634 1.9.2.3 martin uint64_t *ret)
635 1.9.2.3 martin {
636 1.9.2.3 martin kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
637 1.9.2.3 martin __RET_ADDR);
638 1.9.2.3 martin return _ucas_64(uaddr, old, new, ret);
639 1.9.2.3 martin }
640 1.9.2.3 martin
641 1.9.2.3 martin #ifdef __HAVE_UCAS_MP
642 1.9.2.3 martin int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
643 1.9.2.3 martin int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
644 1.9.2.3 martin int
645 1.9.2.3 martin kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
646 1.9.2.3 martin uint64_t *ret)
647 1.9.2.3 martin {
648 1.9.2.3 martin kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
649 1.9.2.3 martin __RET_ADDR);
650 1.9.2.3 martin return _ucas_64_mp(uaddr, old, new, ret);
651 1.9.2.3 martin }
652 1.9.2.3 martin #endif
653 1.9.2.3 martin #endif
654 1.9.2.3 martin
655 1.9.2.3 martin int _ufetch_8(const uint8_t *, uint8_t *);
656 1.9.2.3 martin int kasan__ufetch_8(const uint8_t *, uint8_t *);
657 1.9.2.3 martin int
658 1.9.2.3 martin kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
659 1.9.2.3 martin {
660 1.9.2.3 martin kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
661 1.9.2.3 martin __RET_ADDR);
662 1.9.2.3 martin return _ufetch_8(uaddr, valp);
663 1.9.2.3 martin }
664 1.9.2.3 martin
665 1.9.2.3 martin int _ufetch_16(const uint16_t *, uint16_t *);
666 1.9.2.3 martin int kasan__ufetch_16(const uint16_t *, uint16_t *);
667 1.9.2.3 martin int
668 1.9.2.3 martin kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
669 1.9.2.3 martin {
670 1.9.2.3 martin kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
671 1.9.2.3 martin __RET_ADDR);
672 1.9.2.3 martin return _ufetch_16(uaddr, valp);
673 1.9.2.3 martin }
674 1.9.2.3 martin
675 1.9.2.3 martin int _ufetch_32(const uint32_t *, uint32_t *);
676 1.9.2.3 martin int kasan__ufetch_32(const uint32_t *, uint32_t *);
677 1.9.2.3 martin int
678 1.9.2.3 martin kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
679 1.9.2.3 martin {
680 1.9.2.3 martin kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
681 1.9.2.3 martin __RET_ADDR);
682 1.9.2.3 martin return _ufetch_32(uaddr, valp);
683 1.9.2.3 martin }
684 1.9.2.3 martin
685 1.9.2.3 martin #ifdef _LP64
686 1.9.2.3 martin int _ufetch_64(const uint64_t *, uint64_t *);
687 1.9.2.3 martin int kasan__ufetch_64(const uint64_t *, uint64_t *);
688 1.9.2.3 martin int
689 1.9.2.3 martin kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
690 1.9.2.3 martin {
691 1.9.2.3 martin kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
692 1.9.2.3 martin __RET_ADDR);
693 1.9.2.3 martin return _ufetch_64(uaddr, valp);
694 1.9.2.3 martin }
695 1.9.2.3 martin #endif
696 1.9.2.3 martin
697 1.9.2.3 martin /* -------------------------------------------------------------------------- */
698 1.9.2.3 martin
699 1.9.2.3 martin #undef atomic_add_32
700 1.9.2.3 martin #undef atomic_add_int
701 1.9.2.3 martin #undef atomic_add_long
702 1.9.2.3 martin #undef atomic_add_ptr
703 1.9.2.3 martin #undef atomic_add_64
704 1.9.2.3 martin #undef atomic_add_32_nv
705 1.9.2.3 martin #undef atomic_add_int_nv
706 1.9.2.3 martin #undef atomic_add_long_nv
707 1.9.2.3 martin #undef atomic_add_ptr_nv
708 1.9.2.3 martin #undef atomic_add_64_nv
709 1.9.2.3 martin #undef atomic_and_32
710 1.9.2.3 martin #undef atomic_and_uint
711 1.9.2.3 martin #undef atomic_and_ulong
712 1.9.2.3 martin #undef atomic_and_64
713 1.9.2.3 martin #undef atomic_and_32_nv
714 1.9.2.3 martin #undef atomic_and_uint_nv
715 1.9.2.3 martin #undef atomic_and_ulong_nv
716 1.9.2.3 martin #undef atomic_and_64_nv
717 1.9.2.3 martin #undef atomic_or_32
718 1.9.2.3 martin #undef atomic_or_uint
719 1.9.2.3 martin #undef atomic_or_ulong
720 1.9.2.3 martin #undef atomic_or_64
721 1.9.2.3 martin #undef atomic_or_32_nv
722 1.9.2.3 martin #undef atomic_or_uint_nv
723 1.9.2.3 martin #undef atomic_or_ulong_nv
724 1.9.2.3 martin #undef atomic_or_64_nv
725 1.9.2.3 martin #undef atomic_cas_32
726 1.9.2.3 martin #undef atomic_cas_uint
727 1.9.2.3 martin #undef atomic_cas_ulong
728 1.9.2.3 martin #undef atomic_cas_ptr
729 1.9.2.3 martin #undef atomic_cas_64
730 1.9.2.3 martin #undef atomic_cas_32_ni
731 1.9.2.3 martin #undef atomic_cas_uint_ni
732 1.9.2.3 martin #undef atomic_cas_ulong_ni
733 1.9.2.3 martin #undef atomic_cas_ptr_ni
734 1.9.2.3 martin #undef atomic_cas_64_ni
735 1.9.2.3 martin #undef atomic_swap_32
736 1.9.2.3 martin #undef atomic_swap_uint
737 1.9.2.3 martin #undef atomic_swap_ulong
738 1.9.2.3 martin #undef atomic_swap_ptr
739 1.9.2.3 martin #undef atomic_swap_64
740 1.9.2.3 martin #undef atomic_dec_32
741 1.9.2.3 martin #undef atomic_dec_uint
742 1.9.2.3 martin #undef atomic_dec_ulong
743 1.9.2.3 martin #undef atomic_dec_ptr
744 1.9.2.3 martin #undef atomic_dec_64
745 1.9.2.3 martin #undef atomic_dec_32_nv
746 1.9.2.3 martin #undef atomic_dec_uint_nv
747 1.9.2.3 martin #undef atomic_dec_ulong_nv
748 1.9.2.3 martin #undef atomic_dec_ptr_nv
749 1.9.2.3 martin #undef atomic_dec_64_nv
750 1.9.2.3 martin #undef atomic_inc_32
751 1.9.2.3 martin #undef atomic_inc_uint
752 1.9.2.3 martin #undef atomic_inc_ulong
753 1.9.2.3 martin #undef atomic_inc_ptr
754 1.9.2.3 martin #undef atomic_inc_64
755 1.9.2.3 martin #undef atomic_inc_32_nv
756 1.9.2.3 martin #undef atomic_inc_uint_nv
757 1.9.2.3 martin #undef atomic_inc_ulong_nv
758 1.9.2.3 martin #undef atomic_inc_ptr_nv
759 1.9.2.3 martin #undef atomic_inc_64_nv
760 1.9.2.3 martin
761 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
762 1.9.2.3 martin void atomic_add_##name(volatile targ1 *, targ2); \
763 1.9.2.3 martin void kasan_atomic_add_##name(volatile targ1 *, targ2); \
764 1.9.2.3 martin void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
765 1.9.2.3 martin { \
766 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
767 1.9.2.3 martin __RET_ADDR); \
768 1.9.2.3 martin atomic_add_##name(ptr, val); \
769 1.9.2.3 martin } \
770 1.9.2.3 martin tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
771 1.9.2.3 martin tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
772 1.9.2.3 martin tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
773 1.9.2.3 martin { \
774 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
775 1.9.2.3 martin __RET_ADDR); \
776 1.9.2.3 martin return atomic_add_##name##_nv(ptr, val); \
777 1.9.2.3 martin }
778 1.9.2.3 martin
779 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
780 1.9.2.3 martin void atomic_and_##name(volatile targ1 *, targ2); \
781 1.9.2.3 martin void kasan_atomic_and_##name(volatile targ1 *, targ2); \
782 1.9.2.3 martin void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
783 1.9.2.3 martin { \
784 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
785 1.9.2.3 martin __RET_ADDR); \
786 1.9.2.3 martin atomic_and_##name(ptr, val); \
787 1.9.2.3 martin } \
788 1.9.2.3 martin tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
789 1.9.2.3 martin tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
790 1.9.2.3 martin tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
791 1.9.2.3 martin { \
792 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
793 1.9.2.3 martin __RET_ADDR); \
794 1.9.2.3 martin return atomic_and_##name##_nv(ptr, val); \
795 1.9.2.3 martin }
796 1.9.2.3 martin
797 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
798 1.9.2.3 martin void atomic_or_##name(volatile targ1 *, targ2); \
799 1.9.2.3 martin void kasan_atomic_or_##name(volatile targ1 *, targ2); \
800 1.9.2.3 martin void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
801 1.9.2.3 martin { \
802 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
803 1.9.2.3 martin __RET_ADDR); \
804 1.9.2.3 martin atomic_or_##name(ptr, val); \
805 1.9.2.3 martin } \
806 1.9.2.3 martin tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
807 1.9.2.3 martin tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
808 1.9.2.3 martin tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
809 1.9.2.3 martin { \
810 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
811 1.9.2.3 martin __RET_ADDR); \
812 1.9.2.3 martin return atomic_or_##name##_nv(ptr, val); \
813 1.9.2.3 martin }
814 1.9.2.3 martin
815 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
816 1.9.2.3 martin tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
817 1.9.2.3 martin tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
818 1.9.2.3 martin tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
819 1.9.2.3 martin { \
820 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
821 1.9.2.3 martin __RET_ADDR); \
822 1.9.2.3 martin return atomic_cas_##name(ptr, exp, new); \
823 1.9.2.3 martin } \
824 1.9.2.3 martin tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
825 1.9.2.3 martin tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
826 1.9.2.3 martin tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
827 1.9.2.3 martin { \
828 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
829 1.9.2.3 martin __RET_ADDR); \
830 1.9.2.3 martin return atomic_cas_##name##_ni(ptr, exp, new); \
831 1.9.2.3 martin }
832 1.9.2.3 martin
833 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
834 1.9.2.3 martin tret atomic_swap_##name(volatile targ1 *, targ2); \
835 1.9.2.3 martin tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \
836 1.9.2.3 martin tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
837 1.9.2.3 martin { \
838 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
839 1.9.2.3 martin __RET_ADDR); \
840 1.9.2.3 martin return atomic_swap_##name(ptr, val); \
841 1.9.2.3 martin }
842 1.9.2.3 martin
843 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
844 1.9.2.3 martin void atomic_dec_##name(volatile targ1 *); \
845 1.9.2.3 martin void kasan_atomic_dec_##name(volatile targ1 *); \
846 1.9.2.3 martin void kasan_atomic_dec_##name(volatile targ1 *ptr) \
847 1.9.2.3 martin { \
848 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
849 1.9.2.3 martin __RET_ADDR); \
850 1.9.2.3 martin atomic_dec_##name(ptr); \
851 1.9.2.3 martin } \
852 1.9.2.3 martin tret atomic_dec_##name##_nv(volatile targ1 *); \
853 1.9.2.3 martin tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \
854 1.9.2.3 martin tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
855 1.9.2.3 martin { \
856 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
857 1.9.2.3 martin __RET_ADDR); \
858 1.9.2.3 martin return atomic_dec_##name##_nv(ptr); \
859 1.9.2.3 martin }
860 1.9.2.3 martin
861 1.9.2.3 martin #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \
862 1.9.2.3 martin void atomic_inc_##name(volatile targ1 *); \
863 1.9.2.3 martin void kasan_atomic_inc_##name(volatile targ1 *); \
864 1.9.2.3 martin void kasan_atomic_inc_##name(volatile targ1 *ptr) \
865 1.9.2.3 martin { \
866 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
867 1.9.2.3 martin __RET_ADDR); \
868 1.9.2.3 martin atomic_inc_##name(ptr); \
869 1.9.2.3 martin } \
870 1.9.2.3 martin tret atomic_inc_##name##_nv(volatile targ1 *); \
871 1.9.2.3 martin tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \
872 1.9.2.3 martin tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
873 1.9.2.3 martin { \
874 1.9.2.3 martin kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
875 1.9.2.3 martin __RET_ADDR); \
876 1.9.2.3 martin return atomic_inc_##name##_nv(ptr); \
877 1.9.2.3 martin }
878 1.9.2.3 martin
879 1.9.2.3 martin ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
880 1.9.2.3 martin ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
881 1.9.2.3 martin ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
882 1.9.2.3 martin ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
883 1.9.2.3 martin ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
884 1.9.2.3 martin
885 1.9.2.3 martin ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
886 1.9.2.3 martin ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
887 1.9.2.3 martin ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
888 1.9.2.3 martin ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
889 1.9.2.3 martin
890 1.9.2.3 martin ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
891 1.9.2.3 martin ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
892 1.9.2.3 martin ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
893 1.9.2.3 martin ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
894 1.9.2.3 martin
895 1.9.2.3 martin ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
896 1.9.2.3 martin ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
897 1.9.2.3 martin ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
898 1.9.2.3 martin ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
899 1.9.2.3 martin ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
900 1.9.2.3 martin
901 1.9.2.3 martin ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
902 1.9.2.3 martin ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
903 1.9.2.3 martin ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
904 1.9.2.3 martin ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
905 1.9.2.3 martin ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
906 1.9.2.3 martin
907 1.9.2.3 martin ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
908 1.9.2.3 martin ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
909 1.9.2.3 martin ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
910 1.9.2.3 martin ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
911 1.9.2.3 martin ASAN_ATOMIC_FUNC_DEC(ptr, void *, void);
912 1.9.2.3 martin
913 1.9.2.3 martin ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
914 1.9.2.3 martin ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
915 1.9.2.3 martin ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
916 1.9.2.3 martin ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
917 1.9.2.3 martin ASAN_ATOMIC_FUNC_INC(ptr, void *, void);
918 1.9.2.3 martin
919 1.9.2.3 martin /* -------------------------------------------------------------------------- */
920 1.9.2.3 martin
921 1.9.2.3 martin #ifdef __HAVE_KASAN_INSTR_BUS
922 1.9.2.3 martin
923 1.9.2.3 martin #include <sys/bus.h>
924 1.9.2.3 martin
925 1.9.2.3 martin #undef bus_space_read_multi_1
926 1.9.2.3 martin #undef bus_space_read_multi_2
927 1.9.2.3 martin #undef bus_space_read_multi_4
928 1.9.2.3 martin #undef bus_space_read_multi_8
929 1.9.2.3 martin #undef bus_space_read_multi_stream_1
930 1.9.2.3 martin #undef bus_space_read_multi_stream_2
931 1.9.2.3 martin #undef bus_space_read_multi_stream_4
932 1.9.2.3 martin #undef bus_space_read_multi_stream_8
933 1.9.2.3 martin #undef bus_space_read_region_1
934 1.9.2.3 martin #undef bus_space_read_region_2
935 1.9.2.3 martin #undef bus_space_read_region_4
936 1.9.2.3 martin #undef bus_space_read_region_8
937 1.9.2.3 martin #undef bus_space_read_region_stream_1
938 1.9.2.3 martin #undef bus_space_read_region_stream_2
939 1.9.2.3 martin #undef bus_space_read_region_stream_4
940 1.9.2.3 martin #undef bus_space_read_region_stream_8
941 1.9.2.3 martin #undef bus_space_write_multi_1
942 1.9.2.3 martin #undef bus_space_write_multi_2
943 1.9.2.3 martin #undef bus_space_write_multi_4
944 1.9.2.3 martin #undef bus_space_write_multi_8
945 1.9.2.3 martin #undef bus_space_write_multi_stream_1
946 1.9.2.3 martin #undef bus_space_write_multi_stream_2
947 1.9.2.3 martin #undef bus_space_write_multi_stream_4
948 1.9.2.3 martin #undef bus_space_write_multi_stream_8
949 1.9.2.3 martin #undef bus_space_write_region_1
950 1.9.2.3 martin #undef bus_space_write_region_2
951 1.9.2.3 martin #undef bus_space_write_region_4
952 1.9.2.3 martin #undef bus_space_write_region_8
953 1.9.2.3 martin #undef bus_space_write_region_stream_1
954 1.9.2.3 martin #undef bus_space_write_region_stream_2
955 1.9.2.3 martin #undef bus_space_write_region_stream_4
956 1.9.2.3 martin #undef bus_space_write_region_stream_8
957 1.9.2.3 martin
958 1.9.2.3 martin #define ASAN_BUS_READ_FUNC(bytes, bits) \
959 1.9.2.3 martin void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
960 1.9.2.3 martin bus_size_t, uint##bits##_t *, bus_size_t); \
961 1.9.2.3 martin void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \
962 1.9.2.3 martin bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
963 1.9.2.3 martin void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
964 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
965 1.9.2.3 martin bus_size_t count) \
966 1.9.2.3 martin { \
967 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
968 1.9.2.3 martin sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
969 1.9.2.3 martin bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
970 1.9.2.3 martin } \
971 1.9.2.3 martin void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
972 1.9.2.3 martin bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
973 1.9.2.3 martin void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
974 1.9.2.3 martin bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
975 1.9.2.3 martin void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
976 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
977 1.9.2.3 martin bus_size_t count) \
978 1.9.2.3 martin { \
979 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
980 1.9.2.3 martin sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
981 1.9.2.3 martin bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
982 1.9.2.3 martin } \
983 1.9.2.3 martin void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
984 1.9.2.3 martin bus_size_t, uint##bits##_t *, bus_size_t); \
985 1.9.2.3 martin void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \
986 1.9.2.3 martin bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
987 1.9.2.3 martin void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
988 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
989 1.9.2.3 martin bus_size_t count) \
990 1.9.2.3 martin { \
991 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
992 1.9.2.3 martin sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
993 1.9.2.3 martin bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
994 1.9.2.3 martin } \
995 1.9.2.3 martin void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
996 1.9.2.3 martin bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
997 1.9.2.3 martin void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
998 1.9.2.3 martin bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
999 1.9.2.3 martin void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
1000 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
1001 1.9.2.3 martin bus_size_t count) \
1002 1.9.2.3 martin { \
1003 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
1004 1.9.2.3 martin sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
1005 1.9.2.3 martin bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
1006 1.9.2.3 martin }
1007 1.9.2.3 martin
1008 1.9.2.3 martin #define ASAN_BUS_WRITE_FUNC(bytes, bits) \
1009 1.9.2.3 martin void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1010 1.9.2.3 martin bus_size_t, const uint##bits##_t *, bus_size_t); \
1011 1.9.2.3 martin void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \
1012 1.9.2.3 martin bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1013 1.9.2.3 martin void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
1014 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1015 1.9.2.3 martin bus_size_t count) \
1016 1.9.2.3 martin { \
1017 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
1018 1.9.2.3 martin sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1019 1.9.2.3 martin bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
1020 1.9.2.3 martin } \
1021 1.9.2.3 martin void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1022 1.9.2.3 martin bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1023 1.9.2.3 martin void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1024 1.9.2.3 martin bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1025 1.9.2.3 martin void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
1026 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1027 1.9.2.3 martin bus_size_t count) \
1028 1.9.2.3 martin { \
1029 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
1030 1.9.2.3 martin sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1031 1.9.2.3 martin bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
1032 1.9.2.3 martin } \
1033 1.9.2.3 martin void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
1034 1.9.2.3 martin bus_size_t, const uint##bits##_t *, bus_size_t); \
1035 1.9.2.3 martin void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \
1036 1.9.2.3 martin bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1037 1.9.2.3 martin void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
1038 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1039 1.9.2.3 martin bus_size_t count) \
1040 1.9.2.3 martin { \
1041 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
1042 1.9.2.3 martin sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1043 1.9.2.3 martin bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
1044 1.9.2.3 martin } \
1045 1.9.2.3 martin void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1046 1.9.2.3 martin bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1047 1.9.2.3 martin void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1048 1.9.2.3 martin bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1049 1.9.2.3 martin void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
1050 1.9.2.3 martin bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1051 1.9.2.3 martin bus_size_t count) \
1052 1.9.2.3 martin { \
1053 1.9.2.3 martin kasan_shadow_check((uintptr_t)buf, \
1054 1.9.2.3 martin sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1055 1.9.2.3 martin bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
1056 1.9.2.3 martin }
1057 1.9.2.3 martin
1058 1.9.2.3 martin ASAN_BUS_READ_FUNC(1, 8)
1059 1.9.2.3 martin ASAN_BUS_READ_FUNC(2, 16)
1060 1.9.2.3 martin ASAN_BUS_READ_FUNC(4, 32)
1061 1.9.2.3 martin ASAN_BUS_READ_FUNC(8, 64)
1062 1.9.2.3 martin
1063 1.9.2.3 martin ASAN_BUS_WRITE_FUNC(1, 8)
1064 1.9.2.3 martin ASAN_BUS_WRITE_FUNC(2, 16)
1065 1.9.2.3 martin ASAN_BUS_WRITE_FUNC(4, 32)
1066 1.9.2.3 martin ASAN_BUS_WRITE_FUNC(8, 64)
1067 1.9.2.3 martin
1068 1.9.2.3 martin #endif /* __HAVE_KASAN_INSTR_BUS */
1069 1.9.2.3 martin
1070 1.9.2.3 martin /* -------------------------------------------------------------------------- */
1071 1.9.2.3 martin
1072 1.9.2.3 martin #include <sys/mbuf.h>
1073 1.9.2.3 martin
1074 1.9.2.3 martin static void
1075 1.9.2.3 martin kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
1076 1.9.2.3 martin bool write, uintptr_t pc)
1077 1.9.2.3 martin {
1078 1.9.2.3 martin kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc);
1079 1.9.2.3 martin }
1080 1.9.2.3 martin
1081 1.9.2.3 martin static void
1082 1.9.2.3 martin kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
1083 1.9.2.3 martin bool write, uintptr_t pc)
1084 1.9.2.3 martin {
1085 1.9.2.3 martin bus_addr_t minlen;
1086 1.9.2.3 martin
1087 1.9.2.3 martin for (; m != NULL && len != 0; m = m->m_next) {
1088 1.9.2.3 martin kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc);
1089 1.9.2.3 martin
1090 1.9.2.3 martin if (offset >= m->m_len) {
1091 1.9.2.3 martin offset -= m->m_len;
1092 1.9.2.3 martin continue;
1093 1.9.2.3 martin }
1094 1.9.2.3 martin
1095 1.9.2.3 martin minlen = MIN(len, m->m_len - offset);
1096 1.9.2.3 martin kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset),
1097 1.9.2.3 martin minlen, write, pc);
1098 1.9.2.3 martin
1099 1.9.2.3 martin offset = 0;
1100 1.9.2.3 martin len -= minlen;
1101 1.9.2.3 martin }
1102 1.9.2.3 martin }
1103 1.9.2.3 martin
1104 1.9.2.3 martin static void
1105 1.9.2.3 martin kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
1106 1.9.2.3 martin bool write, uintptr_t pc)
1107 1.9.2.3 martin {
1108 1.9.2.3 martin bus_size_t minlen, resid;
1109 1.9.2.3 martin struct iovec *iov;
1110 1.9.2.3 martin int i;
1111 1.9.2.3 martin
1112 1.9.2.3 martin kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc);
1113 1.9.2.3 martin
1114 1.9.2.3 martin if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
1115 1.9.2.3 martin return;
1116 1.9.2.3 martin
1117 1.9.2.3 martin resid = uio->uio_resid;
1118 1.9.2.3 martin iov = uio->uio_iov;
1119 1.9.2.3 martin
1120 1.9.2.3 martin for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1121 1.9.2.3 martin kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]),
1122 1.9.2.3 martin false, pc);
1123 1.9.2.3 martin minlen = MIN(resid, iov[i].iov_len);
1124 1.9.2.3 martin kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen,
1125 1.9.2.3 martin write, pc);
1126 1.9.2.3 martin resid -= minlen;
1127 1.9.2.3 martin }
1128 1.9.2.3 martin }
1129 1.9.2.3 martin
1130 1.9.2.3 martin void
1131 1.9.2.3 martin kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
1132 1.9.2.3 martin {
1133 1.9.2.3 martin bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0;
1134 1.9.2.3 martin
1135 1.9.2.3 martin switch (map->dm_buftype) {
1136 1.9.2.3 martin case KASAN_DMA_LINEAR:
1137 1.9.2.3 martin kasan_dma_sync_linear(map->dm_buf, offset, len, write,
1138 1.9.2.3 martin __RET_ADDR);
1139 1.9.2.3 martin break;
1140 1.9.2.3 martin case KASAN_DMA_MBUF:
1141 1.9.2.3 martin kasan_dma_sync_mbuf(map->dm_buf, offset, len, write,
1142 1.9.2.3 martin __RET_ADDR);
1143 1.9.2.3 martin break;
1144 1.9.2.3 martin case KASAN_DMA_UIO:
1145 1.9.2.3 martin kasan_dma_sync_uio(map->dm_buf, offset, len, write,
1146 1.9.2.3 martin __RET_ADDR);
1147 1.9.2.3 martin break;
1148 1.9.2.3 martin case KASAN_DMA_RAW:
1149 1.9.2.3 martin break;
1150 1.9.2.3 martin default:
1151 1.9.2.3 martin panic("%s: impossible", __func__);
1152 1.9.2.3 martin }
1153 1.9.2.3 martin }
1154 1.9.2.3 martin
1155 1.9.2.3 martin void
1156 1.9.2.3 martin kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
1157 1.9.2.3 martin {
1158 1.9.2.3 martin map->dm_buf = buf;
1159 1.9.2.3 martin map->dm_buflen = buflen;
1160 1.9.2.3 martin map->dm_buftype = type;
1161 1.9.2.3 martin }
1162 1.9.2.3 martin
1163 1.9.2.3 martin /* -------------------------------------------------------------------------- */
1164 1.9.2.3 martin
1165 1.9.2.2 christos void __asan_register_globals(struct __asan_global *, size_t);
1166 1.9.2.2 christos void __asan_unregister_globals(struct __asan_global *, size_t);
1167 1.9.2.2 christos
1168 1.9.2.2 christos void
1169 1.9.2.2 christos __asan_register_globals(struct __asan_global *globals, size_t n)
1170 1.9.2.2 christos {
1171 1.9.2.2 christos size_t i;
1172 1.9.2.2 christos
1173 1.9.2.2 christos for (i = 0; i < n; i++) {
1174 1.9.2.2 christos kasan_mark(globals[i].beg, globals[i].size,
1175 1.9.2.2 christos globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1176 1.9.2.2 christos }
1177 1.9.2.2 christos }
1178 1.9.2.2 christos
1179 1.9.2.2 christos void
1180 1.9.2.2 christos __asan_unregister_globals(struct __asan_global *globals, size_t n)
1181 1.9.2.2 christos {
1182 1.9.2.2 christos /* never called */
1183 1.9.2.2 christos }
1184 1.9.2.2 christos
1185 1.9.2.2 christos #define ASAN_LOAD_STORE(size) \
1186 1.9.2.2 christos void __asan_load##size(unsigned long); \
1187 1.9.2.2 christos void __asan_load##size(unsigned long addr) \
1188 1.9.2.2 christos { \
1189 1.9.2.2 christos kasan_shadow_check(addr, size, false, __RET_ADDR);\
1190 1.9.2.2 christos } \
1191 1.9.2.2 christos void __asan_load##size##_noabort(unsigned long); \
1192 1.9.2.2 christos void __asan_load##size##_noabort(unsigned long addr) \
1193 1.9.2.2 christos { \
1194 1.9.2.2 christos kasan_shadow_check(addr, size, false, __RET_ADDR);\
1195 1.9.2.2 christos } \
1196 1.9.2.2 christos void __asan_store##size(unsigned long); \
1197 1.9.2.2 christos void __asan_store##size(unsigned long addr) \
1198 1.9.2.2 christos { \
1199 1.9.2.2 christos kasan_shadow_check(addr, size, true, __RET_ADDR);\
1200 1.9.2.2 christos } \
1201 1.9.2.2 christos void __asan_store##size##_noabort(unsigned long); \
1202 1.9.2.2 christos void __asan_store##size##_noabort(unsigned long addr) \
1203 1.9.2.2 christos { \
1204 1.9.2.2 christos kasan_shadow_check(addr, size, true, __RET_ADDR);\
1205 1.9.2.2 christos }
1206 1.9.2.2 christos
1207 1.9.2.2 christos ASAN_LOAD_STORE(1);
1208 1.9.2.2 christos ASAN_LOAD_STORE(2);
1209 1.9.2.2 christos ASAN_LOAD_STORE(4);
1210 1.9.2.2 christos ASAN_LOAD_STORE(8);
1211 1.9.2.2 christos ASAN_LOAD_STORE(16);
1212 1.9.2.2 christos
1213 1.9.2.2 christos void __asan_loadN(unsigned long, size_t);
1214 1.9.2.2 christos void __asan_loadN_noabort(unsigned long, size_t);
1215 1.9.2.2 christos void __asan_storeN(unsigned long, size_t);
1216 1.9.2.2 christos void __asan_storeN_noabort(unsigned long, size_t);
1217 1.9.2.2 christos void __asan_handle_no_return(void);
1218 1.9.2.2 christos
1219 1.9.2.2 christos void
1220 1.9.2.2 christos __asan_loadN(unsigned long addr, size_t size)
1221 1.9.2.2 christos {
1222 1.9.2.2 christos kasan_shadow_check(addr, size, false, __RET_ADDR);
1223 1.9.2.2 christos }
1224 1.9.2.2 christos
1225 1.9.2.2 christos void
1226 1.9.2.2 christos __asan_loadN_noabort(unsigned long addr, size_t size)
1227 1.9.2.2 christos {
1228 1.9.2.2 christos kasan_shadow_check(addr, size, false, __RET_ADDR);
1229 1.9.2.2 christos }
1230 1.9.2.2 christos
1231 1.9.2.2 christos void
1232 1.9.2.2 christos __asan_storeN(unsigned long addr, size_t size)
1233 1.9.2.2 christos {
1234 1.9.2.2 christos kasan_shadow_check(addr, size, true, __RET_ADDR);
1235 1.9.2.2 christos }
1236 1.9.2.2 christos
1237 1.9.2.2 christos void
1238 1.9.2.2 christos __asan_storeN_noabort(unsigned long addr, size_t size)
1239 1.9.2.2 christos {
1240 1.9.2.2 christos kasan_shadow_check(addr, size, true, __RET_ADDR);
1241 1.9.2.2 christos }
1242 1.9.2.2 christos
1243 1.9.2.2 christos void
1244 1.9.2.2 christos __asan_handle_no_return(void)
1245 1.9.2.2 christos {
1246 1.9.2.2 christos /* nothing */
1247 1.9.2.2 christos }
1248 1.9.2.2 christos
1249 1.9.2.2 christos #define ASAN_SET_SHADOW(byte) \
1250 1.9.2.2 christos void __asan_set_shadow_##byte(void *, size_t); \
1251 1.9.2.2 christos void __asan_set_shadow_##byte(void *addr, size_t size) \
1252 1.9.2.2 christos { \
1253 1.9.2.2 christos __builtin_memset((void *)addr, 0x##byte, size); \
1254 1.9.2.2 christos }
1255 1.9.2.2 christos
1256 1.9.2.2 christos ASAN_SET_SHADOW(00);
1257 1.9.2.2 christos ASAN_SET_SHADOW(f1);
1258 1.9.2.2 christos ASAN_SET_SHADOW(f2);
1259 1.9.2.2 christos ASAN_SET_SHADOW(f3);
1260 1.9.2.2 christos ASAN_SET_SHADOW(f5);
1261 1.9.2.2 christos ASAN_SET_SHADOW(f8);
1262 1.9.2.2 christos
1263 1.9.2.2 christos void __asan_poison_stack_memory(const void *, size_t);
1264 1.9.2.2 christos void __asan_unpoison_stack_memory(const void *, size_t);
1265 1.9.2.2 christos
1266 1.9.2.3 martin void
1267 1.9.2.3 martin __asan_poison_stack_memory(const void *addr, size_t size)
1268 1.9.2.2 christos {
1269 1.9.2.2 christos size = roundup(size, KASAN_SHADOW_SCALE_SIZE);
1270 1.9.2.2 christos kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1271 1.9.2.2 christos }
1272 1.9.2.2 christos
1273 1.9.2.3 martin void
1274 1.9.2.3 martin __asan_unpoison_stack_memory(const void *addr, size_t size)
1275 1.9.2.2 christos {
1276 1.9.2.2 christos kasan_shadow_Nbyte_markvalid(addr, size);
1277 1.9.2.2 christos }
1278 1.9.2.3 martin
1279 1.9.2.3 martin void __asan_alloca_poison(const void *, size_t);
1280 1.9.2.3 martin void __asan_allocas_unpoison(const void *, const void *);
1281 1.9.2.3 martin
1282 1.9.2.3 martin void __asan_alloca_poison(const void *addr, size_t size)
1283 1.9.2.3 martin {
1284 1.9.2.3 martin const void *l, *r;
1285 1.9.2.3 martin
1286 1.9.2.3 martin KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0);
1287 1.9.2.3 martin
1288 1.9.2.3 martin l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1289 1.9.2.3 martin r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1290 1.9.2.3 martin
1291 1.9.2.3 martin kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1292 1.9.2.3 martin kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1293 1.9.2.3 martin KASAN_STACK_MID);
1294 1.9.2.3 martin kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1295 1.9.2.3 martin }
1296 1.9.2.3 martin
1297 1.9.2.3 martin void __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1298 1.9.2.3 martin {
1299 1.9.2.3 martin size_t size;
1300 1.9.2.3 martin
1301 1.9.2.3 martin if (__predict_false(!stkbegin))
1302 1.9.2.3 martin return;
1303 1.9.2.3 martin if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1304 1.9.2.3 martin return;
1305 1.9.2.3 martin size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1306 1.9.2.3 martin
1307 1.9.2.3 martin kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1308 1.9.2.3 martin }
1309