Home | History | Annotate | Line # | Download | only in kern
subr_asan.c revision 1.9.2.4
      1  1.9.2.3    martin /*	$NetBSD: subr_asan.c,v 1.9.2.4 2020/04/21 18:42:42 martin Exp $	*/
      2  1.9.2.2  christos 
      3  1.9.2.2  christos /*
      4  1.9.2.3    martin  * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
      5  1.9.2.2  christos  * All rights reserved.
      6  1.9.2.2  christos  *
      7  1.9.2.2  christos  * This code is derived from software contributed to The NetBSD Foundation
      8  1.9.2.2  christos  * by Maxime Villard.
      9  1.9.2.2  christos  *
     10  1.9.2.2  christos  * Redistribution and use in source and binary forms, with or without
     11  1.9.2.2  christos  * modification, are permitted provided that the following conditions
     12  1.9.2.2  christos  * are met:
     13  1.9.2.2  christos  * 1. Redistributions of source code must retain the above copyright
     14  1.9.2.2  christos  *    notice, this list of conditions and the following disclaimer.
     15  1.9.2.2  christos  * 2. Redistributions in binary form must reproduce the above copyright
     16  1.9.2.2  christos  *    notice, this list of conditions and the following disclaimer in the
     17  1.9.2.2  christos  *    documentation and/or other materials provided with the distribution.
     18  1.9.2.2  christos  *
     19  1.9.2.2  christos  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  1.9.2.2  christos  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  1.9.2.2  christos  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  1.9.2.2  christos  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  1.9.2.2  christos  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  1.9.2.2  christos  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  1.9.2.2  christos  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  1.9.2.2  christos  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  1.9.2.2  christos  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  1.9.2.2  christos  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  1.9.2.2  christos  * POSSIBILITY OF SUCH DAMAGE.
     30  1.9.2.2  christos  */
     31  1.9.2.2  christos 
     32  1.9.2.2  christos #include <sys/cdefs.h>
     33  1.9.2.3    martin __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.9.2.4 2020/04/21 18:42:42 martin Exp $");
     34  1.9.2.2  christos 
     35  1.9.2.2  christos #include <sys/param.h>
     36  1.9.2.2  christos #include <sys/device.h>
     37  1.9.2.2  christos #include <sys/kernel.h>
     38  1.9.2.2  christos #include <sys/param.h>
     39  1.9.2.2  christos #include <sys/conf.h>
     40  1.9.2.2  christos #include <sys/systm.h>
     41  1.9.2.2  christos #include <sys/types.h>
     42  1.9.2.2  christos #include <sys/asan.h>
     43  1.9.2.2  christos 
     44  1.9.2.2  christos #include <uvm/uvm.h>
     45  1.9.2.2  christos 
     46  1.9.2.3    martin #ifdef KASAN_PANIC
     47  1.9.2.3    martin #define REPORT panic
     48  1.9.2.3    martin #else
     49  1.9.2.3    martin #define REPORT printf
     50  1.9.2.3    martin #endif
     51  1.9.2.3    martin 
     52  1.9.2.2  christos /* ASAN constants. Part of the compiler ABI. */
     53  1.9.2.2  christos #define KASAN_SHADOW_SCALE_SHIFT	3
     54  1.9.2.2  christos #define KASAN_SHADOW_SCALE_SIZE		(1UL << KASAN_SHADOW_SCALE_SHIFT)
     55  1.9.2.2  christos #define KASAN_SHADOW_MASK		(KASAN_SHADOW_SCALE_SIZE - 1)
     56  1.9.2.3    martin #define KASAN_ALLOCA_SCALE_SIZE		32
     57  1.9.2.2  christos 
     58  1.9.2.2  christos /* The MD code. */
     59  1.9.2.2  christos #include <machine/asan.h>
     60  1.9.2.2  christos 
     61  1.9.2.2  christos /* ASAN ABI version. */
     62  1.9.2.2  christos #if defined(__clang__) && (__clang_major__ - 0 >= 6)
     63  1.9.2.2  christos #define ASAN_ABI_VERSION	8
     64  1.9.2.2  christos #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
     65  1.9.2.2  christos #define ASAN_ABI_VERSION	8
     66  1.9.2.2  christos #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
     67  1.9.2.2  christos #define ASAN_ABI_VERSION	6
     68  1.9.2.2  christos #else
     69  1.9.2.2  christos #error "Unsupported compiler version"
     70  1.9.2.2  christos #endif
     71  1.9.2.2  christos 
     72  1.9.2.2  christos #define __RET_ADDR	(unsigned long)__builtin_return_address(0)
     73  1.9.2.2  christos 
     74  1.9.2.2  christos /* Global variable descriptor. Part of the compiler ABI.  */
     75  1.9.2.2  christos struct __asan_global_source_location {
     76  1.9.2.2  christos 	const char *filename;
     77  1.9.2.2  christos 	int line_no;
     78  1.9.2.2  christos 	int column_no;
     79  1.9.2.2  christos };
     80  1.9.2.2  christos struct __asan_global {
     81  1.9.2.2  christos 	const void *beg;		/* address of the global variable */
     82  1.9.2.2  christos 	size_t size;			/* size of the global variable */
     83  1.9.2.2  christos 	size_t size_with_redzone;	/* size with the redzone */
     84  1.9.2.2  christos 	const void *name;		/* name of the variable */
     85  1.9.2.2  christos 	const void *module_name;	/* name of the module where the var is declared */
     86  1.9.2.2  christos 	unsigned long has_dynamic_init;	/* the var has dyn initializer (c++) */
     87  1.9.2.2  christos 	struct __asan_global_source_location *location;
     88  1.9.2.2  christos #if ASAN_ABI_VERSION >= 7
     89  1.9.2.2  christos 	uintptr_t odr_indicator;	/* the address of the ODR indicator symbol */
     90  1.9.2.2  christos #endif
     91  1.9.2.2  christos };
     92  1.9.2.2  christos 
     93  1.9.2.2  christos static bool kasan_enabled __read_mostly = false;
     94  1.9.2.2  christos 
     95  1.9.2.2  christos /* -------------------------------------------------------------------------- */
     96  1.9.2.2  christos 
     97  1.9.2.2  christos void
     98  1.9.2.2  christos kasan_shadow_map(void *addr, size_t size)
     99  1.9.2.2  christos {
    100  1.9.2.2  christos 	size_t sz, npages, i;
    101  1.9.2.2  christos 	vaddr_t sva, eva;
    102  1.9.2.2  christos 
    103  1.9.2.2  christos 	KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
    104  1.9.2.2  christos 
    105  1.9.2.2  christos 	sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE;
    106  1.9.2.2  christos 
    107  1.9.2.2  christos 	sva = (vaddr_t)kasan_md_addr_to_shad(addr);
    108  1.9.2.2  christos 	eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz;
    109  1.9.2.2  christos 
    110  1.9.2.2  christos 	sva = rounddown(sva, PAGE_SIZE);
    111  1.9.2.2  christos 	eva = roundup(eva, PAGE_SIZE);
    112  1.9.2.2  christos 
    113  1.9.2.2  christos 	npages = (eva - sva) / PAGE_SIZE;
    114  1.9.2.2  christos 
    115  1.9.2.2  christos 	KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END);
    116  1.9.2.2  christos 
    117  1.9.2.2  christos 	for (i = 0; i < npages; i++) {
    118  1.9.2.2  christos 		kasan_md_shadow_map_page(sva + i * PAGE_SIZE);
    119  1.9.2.2  christos 	}
    120  1.9.2.2  christos }
    121  1.9.2.2  christos 
    122  1.9.2.2  christos static void
    123  1.9.2.2  christos kasan_ctors(void)
    124  1.9.2.2  christos {
    125  1.9.2.2  christos 	extern uint64_t __CTOR_LIST__, __CTOR_END__;
    126  1.9.2.2  christos 	size_t nentries, i;
    127  1.9.2.2  christos 	uint64_t *ptr;
    128  1.9.2.2  christos 
    129  1.9.2.2  christos 	nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) /
    130  1.9.2.2  christos 	    sizeof(uintptr_t);
    131  1.9.2.2  christos 
    132  1.9.2.2  christos 	ptr = &__CTOR_LIST__;
    133  1.9.2.2  christos 	for (i = 0; i < nentries; i++) {
    134  1.9.2.2  christos 		void (*func)(void);
    135  1.9.2.2  christos 
    136  1.9.2.2  christos 		func = (void *)(*ptr);
    137  1.9.2.2  christos 		(*func)();
    138  1.9.2.2  christos 
    139  1.9.2.2  christos 		ptr++;
    140  1.9.2.2  christos 	}
    141  1.9.2.2  christos }
    142  1.9.2.2  christos 
    143  1.9.2.2  christos void
    144  1.9.2.2  christos kasan_early_init(void *stack)
    145  1.9.2.2  christos {
    146  1.9.2.2  christos 	kasan_md_early_init(stack);
    147  1.9.2.2  christos }
    148  1.9.2.2  christos 
    149  1.9.2.2  christos void
    150  1.9.2.2  christos kasan_init(void)
    151  1.9.2.2  christos {
    152  1.9.2.2  christos 	/* MD initialization. */
    153  1.9.2.2  christos 	kasan_md_init();
    154  1.9.2.2  christos 
    155  1.9.2.2  christos 	/* Now officially enabled. */
    156  1.9.2.2  christos 	kasan_enabled = true;
    157  1.9.2.2  christos 
    158  1.9.2.2  christos 	/* Call the ASAN constructors. */
    159  1.9.2.2  christos 	kasan_ctors();
    160  1.9.2.2  christos }
    161  1.9.2.2  christos 
    162  1.9.2.2  christos static inline const char *
    163  1.9.2.2  christos kasan_code_name(uint8_t code)
    164  1.9.2.2  christos {
    165  1.9.2.2  christos 	switch (code) {
    166  1.9.2.2  christos 	case KASAN_GENERIC_REDZONE:
    167  1.9.2.2  christos 		return "GenericRedZone";
    168  1.9.2.2  christos 	case KASAN_MALLOC_REDZONE:
    169  1.9.2.2  christos 		return "MallocRedZone";
    170  1.9.2.2  christos 	case KASAN_KMEM_REDZONE:
    171  1.9.2.2  christos 		return "KmemRedZone";
    172  1.9.2.2  christos 	case KASAN_POOL_REDZONE:
    173  1.9.2.2  christos 		return "PoolRedZone";
    174  1.9.2.2  christos 	case KASAN_POOL_FREED:
    175  1.9.2.2  christos 		return "PoolUseAfterFree";
    176  1.9.2.2  christos 	case 1 ... 7:
    177  1.9.2.2  christos 		return "RedZonePartial";
    178  1.9.2.2  christos 	case KASAN_STACK_LEFT:
    179  1.9.2.2  christos 		return "StackLeft";
    180  1.9.2.3    martin 	case KASAN_STACK_MID:
    181  1.9.2.3    martin 		return "StackMiddle";
    182  1.9.2.2  christos 	case KASAN_STACK_RIGHT:
    183  1.9.2.2  christos 		return "StackRight";
    184  1.9.2.3    martin 	case KASAN_USE_AFTER_RET:
    185  1.9.2.3    martin 		return "UseAfterRet";
    186  1.9.2.2  christos 	case KASAN_USE_AFTER_SCOPE:
    187  1.9.2.2  christos 		return "UseAfterScope";
    188  1.9.2.2  christos 	default:
    189  1.9.2.2  christos 		return "Unknown";
    190  1.9.2.2  christos 	}
    191  1.9.2.2  christos }
    192  1.9.2.2  christos 
    193  1.9.2.2  christos static void
    194  1.9.2.2  christos kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
    195  1.9.2.2  christos     uint8_t code)
    196  1.9.2.2  christos {
    197  1.9.2.3    martin 	REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s,"
    198  1.9.2.2  christos 	    " %s]\n",
    199  1.9.2.2  christos 	    (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""),
    200  1.9.2.2  christos 	    (write ? "write" : "read"), kasan_code_name(code));
    201  1.9.2.2  christos 	kasan_md_unwind();
    202  1.9.2.2  christos }
    203  1.9.2.2  christos 
    204  1.9.2.2  christos static __always_inline void
    205  1.9.2.2  christos kasan_shadow_1byte_markvalid(unsigned long addr)
    206  1.9.2.2  christos {
    207  1.9.2.2  christos 	int8_t *byte = kasan_md_addr_to_shad((void *)addr);
    208  1.9.2.2  christos 	int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
    209  1.9.2.2  christos 
    210  1.9.2.2  christos 	*byte = last;
    211  1.9.2.2  christos }
    212  1.9.2.2  christos 
    213  1.9.2.2  christos static __always_inline void
    214  1.9.2.2  christos kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
    215  1.9.2.2  christos {
    216  1.9.2.2  christos 	size_t i;
    217  1.9.2.2  christos 
    218  1.9.2.2  christos 	for (i = 0; i < size; i++) {
    219  1.9.2.2  christos 		kasan_shadow_1byte_markvalid((unsigned long)addr+i);
    220  1.9.2.2  christos 	}
    221  1.9.2.2  christos }
    222  1.9.2.2  christos 
    223  1.9.2.2  christos static __always_inline void
    224  1.9.2.2  christos kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
    225  1.9.2.2  christos {
    226  1.9.2.2  christos 	void *shad;
    227  1.9.2.2  christos 
    228  1.9.2.2  christos 	if (__predict_false(size == 0))
    229  1.9.2.2  christos 		return;
    230  1.9.2.2  christos 	if (__predict_false(kasan_md_unsupported((vaddr_t)addr)))
    231  1.9.2.2  christos 		return;
    232  1.9.2.2  christos 
    233  1.9.2.2  christos 	KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
    234  1.9.2.2  christos 	KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
    235  1.9.2.2  christos 
    236  1.9.2.2  christos 	shad = (void *)kasan_md_addr_to_shad(addr);
    237  1.9.2.2  christos 	size = size >> KASAN_SHADOW_SCALE_SHIFT;
    238  1.9.2.2  christos 
    239  1.9.2.2  christos 	__builtin_memset(shad, code, size);
    240  1.9.2.2  christos }
    241  1.9.2.2  christos 
    242  1.9.2.2  christos void
    243  1.9.2.2  christos kasan_add_redzone(size_t *size)
    244  1.9.2.2  christos {
    245  1.9.2.2  christos 	*size = roundup(*size, KASAN_SHADOW_SCALE_SIZE);
    246  1.9.2.2  christos 	*size += KASAN_SHADOW_SCALE_SIZE;
    247  1.9.2.2  christos }
    248  1.9.2.2  christos 
    249  1.9.2.2  christos void
    250  1.9.2.2  christos kasan_softint(struct lwp *l)
    251  1.9.2.2  christos {
    252  1.9.2.2  christos 	const void *stk = (const void *)uvm_lwp_getuarea(l);
    253  1.9.2.2  christos 
    254  1.9.2.2  christos 	kasan_shadow_Nbyte_fill(stk, USPACE, 0);
    255  1.9.2.2  christos }
    256  1.9.2.2  christos 
    257  1.9.2.2  christos /*
    258  1.9.2.2  christos  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
    259  1.9.2.2  christos  * and the rest as invalid. There are generally two use cases:
    260  1.9.2.2  christos  *
    261  1.9.2.2  christos  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
    262  1.9.2.2  christos  *    the redzone at the end of the buffer as invalid.
    263  1.9.2.2  christos  *
    264  1.9.2.2  christos  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
    265  1.9.2.2  christos  */
    266  1.9.2.2  christos void
    267  1.9.2.2  christos kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code)
    268  1.9.2.2  christos {
    269  1.9.2.2  christos 	size_t i, n, redz;
    270  1.9.2.2  christos 	int8_t *shad;
    271  1.9.2.2  christos 
    272  1.9.2.2  christos 	KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
    273  1.9.2.2  christos 	redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE);
    274  1.9.2.2  christos 	KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0);
    275  1.9.2.2  christos 	shad = kasan_md_addr_to_shad(addr);
    276  1.9.2.2  christos 
    277  1.9.2.2  christos 	/* Chunks of 8 bytes, valid. */
    278  1.9.2.2  christos 	n = size / KASAN_SHADOW_SCALE_SIZE;
    279  1.9.2.2  christos 	for (i = 0; i < n; i++) {
    280  1.9.2.2  christos 		*shad++ = 0;
    281  1.9.2.2  christos 	}
    282  1.9.2.2  christos 
    283  1.9.2.2  christos 	/* Possibly one chunk, mid. */
    284  1.9.2.2  christos 	if ((size & KASAN_SHADOW_MASK) != 0) {
    285  1.9.2.2  christos 		*shad++ = (size & KASAN_SHADOW_MASK);
    286  1.9.2.2  christos 	}
    287  1.9.2.2  christos 
    288  1.9.2.2  christos 	/* Chunks of 8 bytes, invalid. */
    289  1.9.2.2  christos 	n = redz / KASAN_SHADOW_SCALE_SIZE;
    290  1.9.2.2  christos 	for (i = 0; i < n; i++) {
    291  1.9.2.2  christos 		*shad++ = code;
    292  1.9.2.2  christos 	}
    293  1.9.2.2  christos }
    294  1.9.2.2  christos 
    295  1.9.2.2  christos /* -------------------------------------------------------------------------- */
    296  1.9.2.2  christos 
    297  1.9.2.2  christos #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) 		\
    298  1.9.2.2  christos 	(addr >> KASAN_SHADOW_SCALE_SHIFT) !=			\
    299  1.9.2.2  christos 	    ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
    300  1.9.2.2  christos 
    301  1.9.2.2  christos static __always_inline bool
    302  1.9.2.2  christos kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
    303  1.9.2.2  christos {
    304  1.9.2.2  christos 	int8_t *byte = kasan_md_addr_to_shad((void *)addr);
    305  1.9.2.2  christos 	int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
    306  1.9.2.2  christos 
    307  1.9.2.2  christos 	if (__predict_true(*byte == 0 || last <= *byte)) {
    308  1.9.2.2  christos 		return true;
    309  1.9.2.2  christos 	}
    310  1.9.2.2  christos 	*code = *byte;
    311  1.9.2.2  christos 	return false;
    312  1.9.2.2  christos }
    313  1.9.2.2  christos 
    314  1.9.2.2  christos static __always_inline bool
    315  1.9.2.2  christos kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
    316  1.9.2.2  christos {
    317  1.9.2.2  christos 	int8_t *byte, last;
    318  1.9.2.2  christos 
    319  1.9.2.2  christos 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
    320  1.9.2.2  christos 		return (kasan_shadow_1byte_isvalid(addr, code) &&
    321  1.9.2.2  christos 		    kasan_shadow_1byte_isvalid(addr+1, code));
    322  1.9.2.2  christos 	}
    323  1.9.2.2  christos 
    324  1.9.2.2  christos 	byte = kasan_md_addr_to_shad((void *)addr);
    325  1.9.2.2  christos 	last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
    326  1.9.2.2  christos 
    327  1.9.2.2  christos 	if (__predict_true(*byte == 0 || last <= *byte)) {
    328  1.9.2.2  christos 		return true;
    329  1.9.2.2  christos 	}
    330  1.9.2.2  christos 	*code = *byte;
    331  1.9.2.2  christos 	return false;
    332  1.9.2.2  christos }
    333  1.9.2.2  christos 
    334  1.9.2.2  christos static __always_inline bool
    335  1.9.2.2  christos kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
    336  1.9.2.2  christos {
    337  1.9.2.2  christos 	int8_t *byte, last;
    338  1.9.2.2  christos 
    339  1.9.2.2  christos 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
    340  1.9.2.2  christos 		return (kasan_shadow_2byte_isvalid(addr, code) &&
    341  1.9.2.2  christos 		    kasan_shadow_2byte_isvalid(addr+2, code));
    342  1.9.2.2  christos 	}
    343  1.9.2.2  christos 
    344  1.9.2.2  christos 	byte = kasan_md_addr_to_shad((void *)addr);
    345  1.9.2.2  christos 	last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
    346  1.9.2.2  christos 
    347  1.9.2.2  christos 	if (__predict_true(*byte == 0 || last <= *byte)) {
    348  1.9.2.2  christos 		return true;
    349  1.9.2.2  christos 	}
    350  1.9.2.2  christos 	*code = *byte;
    351  1.9.2.2  christos 	return false;
    352  1.9.2.2  christos }
    353  1.9.2.2  christos 
    354  1.9.2.2  christos static __always_inline bool
    355  1.9.2.2  christos kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
    356  1.9.2.2  christos {
    357  1.9.2.2  christos 	int8_t *byte, last;
    358  1.9.2.2  christos 
    359  1.9.2.2  christos 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
    360  1.9.2.2  christos 		return (kasan_shadow_4byte_isvalid(addr, code) &&
    361  1.9.2.2  christos 		    kasan_shadow_4byte_isvalid(addr+4, code));
    362  1.9.2.2  christos 	}
    363  1.9.2.2  christos 
    364  1.9.2.2  christos 	byte = kasan_md_addr_to_shad((void *)addr);
    365  1.9.2.2  christos 	last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
    366  1.9.2.2  christos 
    367  1.9.2.2  christos 	if (__predict_true(*byte == 0 || last <= *byte)) {
    368  1.9.2.2  christos 		return true;
    369  1.9.2.2  christos 	}
    370  1.9.2.2  christos 	*code = *byte;
    371  1.9.2.2  christos 	return false;
    372  1.9.2.2  christos }
    373  1.9.2.2  christos 
    374  1.9.2.2  christos static __always_inline bool
    375  1.9.2.2  christos kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
    376  1.9.2.2  christos {
    377  1.9.2.2  christos 	size_t i;
    378  1.9.2.2  christos 
    379  1.9.2.2  christos 	for (i = 0; i < size; i++) {
    380  1.9.2.2  christos 		if (!kasan_shadow_1byte_isvalid(addr+i, code))
    381  1.9.2.2  christos 			return false;
    382  1.9.2.2  christos 	}
    383  1.9.2.2  christos 
    384  1.9.2.2  christos 	return true;
    385  1.9.2.2  christos }
    386  1.9.2.2  christos 
    387  1.9.2.2  christos static __always_inline void
    388  1.9.2.2  christos kasan_shadow_check(unsigned long addr, size_t size, bool write,
    389  1.9.2.2  christos     unsigned long retaddr)
    390  1.9.2.2  christos {
    391  1.9.2.2  christos 	uint8_t code;
    392  1.9.2.2  christos 	bool valid;
    393  1.9.2.2  christos 
    394  1.9.2.2  christos 	if (__predict_false(!kasan_enabled))
    395  1.9.2.2  christos 		return;
    396  1.9.2.2  christos 	if (__predict_false(size == 0))
    397  1.9.2.2  christos 		return;
    398  1.9.2.2  christos 	if (__predict_false(kasan_md_unsupported(addr)))
    399  1.9.2.2  christos 		return;
    400  1.9.2.2  christos 
    401  1.9.2.2  christos 	if (__builtin_constant_p(size)) {
    402  1.9.2.2  christos 		switch (size) {
    403  1.9.2.2  christos 		case 1:
    404  1.9.2.2  christos 			valid = kasan_shadow_1byte_isvalid(addr, &code);
    405  1.9.2.2  christos 			break;
    406  1.9.2.2  christos 		case 2:
    407  1.9.2.2  christos 			valid = kasan_shadow_2byte_isvalid(addr, &code);
    408  1.9.2.2  christos 			break;
    409  1.9.2.2  christos 		case 4:
    410  1.9.2.2  christos 			valid = kasan_shadow_4byte_isvalid(addr, &code);
    411  1.9.2.2  christos 			break;
    412  1.9.2.2  christos 		case 8:
    413  1.9.2.2  christos 			valid = kasan_shadow_8byte_isvalid(addr, &code);
    414  1.9.2.2  christos 			break;
    415  1.9.2.2  christos 		default:
    416  1.9.2.2  christos 			valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
    417  1.9.2.2  christos 			break;
    418  1.9.2.2  christos 		}
    419  1.9.2.2  christos 	} else {
    420  1.9.2.2  christos 		valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
    421  1.9.2.2  christos 	}
    422  1.9.2.2  christos 
    423  1.9.2.2  christos 	if (__predict_false(!valid)) {
    424  1.9.2.2  christos 		kasan_report(addr, size, write, retaddr, code);
    425  1.9.2.2  christos 	}
    426  1.9.2.2  christos }
    427  1.9.2.2  christos 
    428  1.9.2.2  christos /* -------------------------------------------------------------------------- */
    429  1.9.2.2  christos 
    430  1.9.2.2  christos void *
    431  1.9.2.2  christos kasan_memcpy(void *dst, const void *src, size_t len)
    432  1.9.2.2  christos {
    433  1.9.2.2  christos 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
    434  1.9.2.2  christos 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
    435  1.9.2.2  christos 	return __builtin_memcpy(dst, src, len);
    436  1.9.2.2  christos }
    437  1.9.2.2  christos 
    438  1.9.2.2  christos int
    439  1.9.2.2  christos kasan_memcmp(const void *b1, const void *b2, size_t len)
    440  1.9.2.2  christos {
    441  1.9.2.2  christos 	kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
    442  1.9.2.2  christos 	kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
    443  1.9.2.2  christos 	return __builtin_memcmp(b1, b2, len);
    444  1.9.2.2  christos }
    445  1.9.2.2  christos 
    446  1.9.2.2  christos void *
    447  1.9.2.2  christos kasan_memset(void *b, int c, size_t len)
    448  1.9.2.2  christos {
    449  1.9.2.2  christos 	kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
    450  1.9.2.2  christos 	return __builtin_memset(b, c, len);
    451  1.9.2.2  christos }
    452  1.9.2.2  christos 
    453  1.9.2.3    martin void *
    454  1.9.2.3    martin kasan_memmove(void *dst, const void *src, size_t len)
    455  1.9.2.3    martin {
    456  1.9.2.3    martin 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
    457  1.9.2.3    martin 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
    458  1.9.2.3    martin 	return __builtin_memmove(dst, src, len);
    459  1.9.2.3    martin }
    460  1.9.2.3    martin 
    461  1.9.2.2  christos char *
    462  1.9.2.2  christos kasan_strcpy(char *dst, const char *src)
    463  1.9.2.2  christos {
    464  1.9.2.2  christos 	char *save = dst;
    465  1.9.2.2  christos 
    466  1.9.2.2  christos 	while (1) {
    467  1.9.2.2  christos 		kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
    468  1.9.2.2  christos 		kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
    469  1.9.2.2  christos 		*dst = *src;
    470  1.9.2.2  christos 		if (*src == '\0')
    471  1.9.2.2  christos 			break;
    472  1.9.2.2  christos 		src++, dst++;
    473  1.9.2.2  christos 	}
    474  1.9.2.2  christos 
    475  1.9.2.2  christos 	return save;
    476  1.9.2.2  christos }
    477  1.9.2.2  christos 
    478  1.9.2.2  christos int
    479  1.9.2.2  christos kasan_strcmp(const char *s1, const char *s2)
    480  1.9.2.2  christos {
    481  1.9.2.2  christos 	while (1) {
    482  1.9.2.2  christos 		kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
    483  1.9.2.2  christos 		kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
    484  1.9.2.2  christos 		if (*s1 != *s2)
    485  1.9.2.2  christos 			break;
    486  1.9.2.2  christos 		if (*s1 == '\0')
    487  1.9.2.2  christos 			return 0;
    488  1.9.2.2  christos 		s1++, s2++;
    489  1.9.2.2  christos 	}
    490  1.9.2.2  christos 
    491  1.9.2.2  christos 	return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
    492  1.9.2.2  christos }
    493  1.9.2.2  christos 
    494  1.9.2.2  christos size_t
    495  1.9.2.2  christos kasan_strlen(const char *str)
    496  1.9.2.2  christos {
    497  1.9.2.2  christos 	const char *s;
    498  1.9.2.2  christos 
    499  1.9.2.2  christos 	s = str;
    500  1.9.2.2  christos 	while (1) {
    501  1.9.2.2  christos 		kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
    502  1.9.2.2  christos 		if (*s == '\0')
    503  1.9.2.2  christos 			break;
    504  1.9.2.2  christos 		s++;
    505  1.9.2.2  christos 	}
    506  1.9.2.2  christos 
    507  1.9.2.2  christos 	return (s - str);
    508  1.9.2.2  christos }
    509  1.9.2.2  christos 
    510  1.9.2.3    martin char *
    511  1.9.2.3    martin kasan_strcat(char *dst, const char *src)
    512  1.9.2.3    martin {
    513  1.9.2.3    martin 	size_t ldst, lsrc;
    514  1.9.2.3    martin 
    515  1.9.2.3    martin 	ldst = __builtin_strlen(dst);
    516  1.9.2.3    martin 	lsrc = __builtin_strlen(src);
    517  1.9.2.3    martin 	kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true,
    518  1.9.2.3    martin 	    __RET_ADDR);
    519  1.9.2.3    martin 	kasan_shadow_check((unsigned long)src, lsrc + 1, false,
    520  1.9.2.3    martin 	    __RET_ADDR);
    521  1.9.2.3    martin 
    522  1.9.2.3    martin 	return __builtin_strcat(dst, src);
    523  1.9.2.3    martin }
    524  1.9.2.3    martin 
    525  1.9.2.3    martin char *
    526  1.9.2.3    martin kasan_strchr(const char *s, int c)
    527  1.9.2.3    martin {
    528  1.9.2.3    martin 	kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
    529  1.9.2.3    martin 	    __RET_ADDR);
    530  1.9.2.3    martin 	return __builtin_strchr(s, c);
    531  1.9.2.3    martin }
    532  1.9.2.3    martin 
    533  1.9.2.3    martin char *
    534  1.9.2.3    martin kasan_strrchr(const char *s, int c)
    535  1.9.2.3    martin {
    536  1.9.2.3    martin 	kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
    537  1.9.2.3    martin 	    __RET_ADDR);
    538  1.9.2.3    martin 	return __builtin_strrchr(s, c);
    539  1.9.2.3    martin }
    540  1.9.2.3    martin 
    541  1.9.2.2  christos #undef kcopy
    542  1.9.2.2  christos #undef copystr
    543  1.9.2.2  christos #undef copyinstr
    544  1.9.2.2  christos #undef copyoutstr
    545  1.9.2.2  christos #undef copyin
    546  1.9.2.2  christos 
    547  1.9.2.2  christos int	kasan_kcopy(const void *, void *, size_t);
    548  1.9.2.2  christos int	kasan_copystr(const void *, void *, size_t, size_t *);
    549  1.9.2.2  christos int	kasan_copyinstr(const void *, void *, size_t, size_t *);
    550  1.9.2.2  christos int	kasan_copyoutstr(const void *, void *, size_t, size_t *);
    551  1.9.2.2  christos int	kasan_copyin(const void *, void *, size_t);
    552  1.9.2.2  christos int	kcopy(const void *, void *, size_t);
    553  1.9.2.2  christos int	copystr(const void *, void *, size_t, size_t *);
    554  1.9.2.2  christos int	copyinstr(const void *, void *, size_t, size_t *);
    555  1.9.2.2  christos int	copyoutstr(const void *, void *, size_t, size_t *);
    556  1.9.2.2  christos int	copyin(const void *, void *, size_t);
    557  1.9.2.2  christos 
    558  1.9.2.2  christos int
    559  1.9.2.2  christos kasan_kcopy(const void *src, void *dst, size_t len)
    560  1.9.2.2  christos {
    561  1.9.2.2  christos 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
    562  1.9.2.2  christos 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
    563  1.9.2.2  christos 	return kcopy(src, dst, len);
    564  1.9.2.2  christos }
    565  1.9.2.2  christos 
    566  1.9.2.2  christos int
    567  1.9.2.2  christos kasan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
    568  1.9.2.2  christos {
    569  1.9.2.2  christos 	kasan_shadow_check((unsigned long)kdaddr, len, true, __RET_ADDR);
    570  1.9.2.2  christos 	return copystr(kfaddr, kdaddr, len, done);
    571  1.9.2.2  christos }
    572  1.9.2.2  christos 
    573  1.9.2.2  christos int
    574  1.9.2.2  christos kasan_copyin(const void *uaddr, void *kaddr, size_t len)
    575  1.9.2.2  christos {
    576  1.9.2.2  christos 	kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
    577  1.9.2.2  christos 	return copyin(uaddr, kaddr, len);
    578  1.9.2.2  christos }
    579  1.9.2.2  christos 
    580  1.9.2.2  christos int
    581  1.9.2.2  christos kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
    582  1.9.2.2  christos {
    583  1.9.2.2  christos 	kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
    584  1.9.2.2  christos 	return copyinstr(uaddr, kaddr, len, done);
    585  1.9.2.2  christos }
    586  1.9.2.2  christos 
    587  1.9.2.2  christos int
    588  1.9.2.2  christos kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
    589  1.9.2.2  christos {
    590  1.9.2.2  christos 	kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
    591  1.9.2.2  christos 	return copyoutstr(kaddr, uaddr, len, done);
    592  1.9.2.2  christos }
    593  1.9.2.2  christos 
    594  1.9.2.2  christos /* -------------------------------------------------------------------------- */
    595  1.9.2.2  christos 
    596  1.9.2.3    martin #undef _ucas_32
    597  1.9.2.3    martin #undef _ucas_32_mp
    598  1.9.2.3    martin #undef _ucas_64
    599  1.9.2.3    martin #undef _ucas_64_mp
    600  1.9.2.3    martin #undef _ufetch_8
    601  1.9.2.3    martin #undef _ufetch_16
    602  1.9.2.3    martin #undef _ufetch_32
    603  1.9.2.3    martin #undef _ufetch_64
    604  1.9.2.3    martin 
    605  1.9.2.3    martin int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    606  1.9.2.3    martin int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    607  1.9.2.3    martin int
    608  1.9.2.3    martin kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
    609  1.9.2.3    martin     uint32_t *ret)
    610  1.9.2.3    martin {
    611  1.9.2.3    martin 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    612  1.9.2.3    martin 	    __RET_ADDR);
    613  1.9.2.3    martin 	return _ucas_32(uaddr, old, new, ret);
    614  1.9.2.3    martin }
    615  1.9.2.3    martin 
    616  1.9.2.3    martin #ifdef __HAVE_UCAS_MP
    617  1.9.2.3    martin int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    618  1.9.2.3    martin int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    619  1.9.2.3    martin int
    620  1.9.2.3    martin kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
    621  1.9.2.3    martin     uint32_t *ret)
    622  1.9.2.3    martin {
    623  1.9.2.3    martin 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    624  1.9.2.3    martin 	    __RET_ADDR);
    625  1.9.2.3    martin 	return _ucas_32_mp(uaddr, old, new, ret);
    626  1.9.2.3    martin }
    627  1.9.2.3    martin #endif
    628  1.9.2.3    martin 
    629  1.9.2.3    martin #ifdef _LP64
    630  1.9.2.3    martin int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    631  1.9.2.3    martin int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    632  1.9.2.3    martin int
    633  1.9.2.3    martin kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
    634  1.9.2.3    martin     uint64_t *ret)
    635  1.9.2.3    martin {
    636  1.9.2.3    martin 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    637  1.9.2.3    martin 	    __RET_ADDR);
    638  1.9.2.3    martin 	return _ucas_64(uaddr, old, new, ret);
    639  1.9.2.3    martin }
    640  1.9.2.3    martin 
    641  1.9.2.3    martin #ifdef __HAVE_UCAS_MP
    642  1.9.2.3    martin int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    643  1.9.2.3    martin int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    644  1.9.2.3    martin int
    645  1.9.2.3    martin kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
    646  1.9.2.3    martin     uint64_t *ret)
    647  1.9.2.3    martin {
    648  1.9.2.3    martin 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    649  1.9.2.3    martin 	    __RET_ADDR);
    650  1.9.2.3    martin 	return _ucas_64_mp(uaddr, old, new, ret);
    651  1.9.2.3    martin }
    652  1.9.2.3    martin #endif
    653  1.9.2.3    martin #endif
    654  1.9.2.3    martin 
    655  1.9.2.3    martin int _ufetch_8(const uint8_t *, uint8_t *);
    656  1.9.2.3    martin int kasan__ufetch_8(const uint8_t *, uint8_t *);
    657  1.9.2.3    martin int
    658  1.9.2.3    martin kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
    659  1.9.2.3    martin {
    660  1.9.2.3    martin 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    661  1.9.2.3    martin 	    __RET_ADDR);
    662  1.9.2.3    martin 	return _ufetch_8(uaddr, valp);
    663  1.9.2.3    martin }
    664  1.9.2.3    martin 
    665  1.9.2.3    martin int _ufetch_16(const uint16_t *, uint16_t *);
    666  1.9.2.3    martin int kasan__ufetch_16(const uint16_t *, uint16_t *);
    667  1.9.2.3    martin int
    668  1.9.2.3    martin kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
    669  1.9.2.3    martin {
    670  1.9.2.3    martin 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    671  1.9.2.3    martin 	    __RET_ADDR);
    672  1.9.2.3    martin 	return _ufetch_16(uaddr, valp);
    673  1.9.2.3    martin }
    674  1.9.2.3    martin 
    675  1.9.2.3    martin int _ufetch_32(const uint32_t *, uint32_t *);
    676  1.9.2.3    martin int kasan__ufetch_32(const uint32_t *, uint32_t *);
    677  1.9.2.3    martin int
    678  1.9.2.3    martin kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
    679  1.9.2.3    martin {
    680  1.9.2.3    martin 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    681  1.9.2.3    martin 	    __RET_ADDR);
    682  1.9.2.3    martin 	return _ufetch_32(uaddr, valp);
    683  1.9.2.3    martin }
    684  1.9.2.3    martin 
    685  1.9.2.3    martin #ifdef _LP64
    686  1.9.2.3    martin int _ufetch_64(const uint64_t *, uint64_t *);
    687  1.9.2.3    martin int kasan__ufetch_64(const uint64_t *, uint64_t *);
    688  1.9.2.3    martin int
    689  1.9.2.3    martin kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
    690  1.9.2.3    martin {
    691  1.9.2.3    martin 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    692  1.9.2.3    martin 	    __RET_ADDR);
    693  1.9.2.3    martin 	return _ufetch_64(uaddr, valp);
    694  1.9.2.3    martin }
    695  1.9.2.3    martin #endif
    696  1.9.2.3    martin 
    697  1.9.2.3    martin /* -------------------------------------------------------------------------- */
    698  1.9.2.3    martin 
    699  1.9.2.3    martin #undef atomic_add_32
    700  1.9.2.3    martin #undef atomic_add_int
    701  1.9.2.3    martin #undef atomic_add_long
    702  1.9.2.3    martin #undef atomic_add_ptr
    703  1.9.2.3    martin #undef atomic_add_64
    704  1.9.2.3    martin #undef atomic_add_32_nv
    705  1.9.2.3    martin #undef atomic_add_int_nv
    706  1.9.2.3    martin #undef atomic_add_long_nv
    707  1.9.2.3    martin #undef atomic_add_ptr_nv
    708  1.9.2.3    martin #undef atomic_add_64_nv
    709  1.9.2.3    martin #undef atomic_and_32
    710  1.9.2.3    martin #undef atomic_and_uint
    711  1.9.2.3    martin #undef atomic_and_ulong
    712  1.9.2.3    martin #undef atomic_and_64
    713  1.9.2.3    martin #undef atomic_and_32_nv
    714  1.9.2.3    martin #undef atomic_and_uint_nv
    715  1.9.2.3    martin #undef atomic_and_ulong_nv
    716  1.9.2.3    martin #undef atomic_and_64_nv
    717  1.9.2.3    martin #undef atomic_or_32
    718  1.9.2.3    martin #undef atomic_or_uint
    719  1.9.2.3    martin #undef atomic_or_ulong
    720  1.9.2.3    martin #undef atomic_or_64
    721  1.9.2.3    martin #undef atomic_or_32_nv
    722  1.9.2.3    martin #undef atomic_or_uint_nv
    723  1.9.2.3    martin #undef atomic_or_ulong_nv
    724  1.9.2.3    martin #undef atomic_or_64_nv
    725  1.9.2.3    martin #undef atomic_cas_32
    726  1.9.2.3    martin #undef atomic_cas_uint
    727  1.9.2.3    martin #undef atomic_cas_ulong
    728  1.9.2.3    martin #undef atomic_cas_ptr
    729  1.9.2.3    martin #undef atomic_cas_64
    730  1.9.2.3    martin #undef atomic_cas_32_ni
    731  1.9.2.3    martin #undef atomic_cas_uint_ni
    732  1.9.2.3    martin #undef atomic_cas_ulong_ni
    733  1.9.2.3    martin #undef atomic_cas_ptr_ni
    734  1.9.2.3    martin #undef atomic_cas_64_ni
    735  1.9.2.3    martin #undef atomic_swap_32
    736  1.9.2.3    martin #undef atomic_swap_uint
    737  1.9.2.3    martin #undef atomic_swap_ulong
    738  1.9.2.3    martin #undef atomic_swap_ptr
    739  1.9.2.3    martin #undef atomic_swap_64
    740  1.9.2.3    martin #undef atomic_dec_32
    741  1.9.2.3    martin #undef atomic_dec_uint
    742  1.9.2.3    martin #undef atomic_dec_ulong
    743  1.9.2.3    martin #undef atomic_dec_ptr
    744  1.9.2.3    martin #undef atomic_dec_64
    745  1.9.2.3    martin #undef atomic_dec_32_nv
    746  1.9.2.3    martin #undef atomic_dec_uint_nv
    747  1.9.2.3    martin #undef atomic_dec_ulong_nv
    748  1.9.2.3    martin #undef atomic_dec_ptr_nv
    749  1.9.2.3    martin #undef atomic_dec_64_nv
    750  1.9.2.3    martin #undef atomic_inc_32
    751  1.9.2.3    martin #undef atomic_inc_uint
    752  1.9.2.3    martin #undef atomic_inc_ulong
    753  1.9.2.3    martin #undef atomic_inc_ptr
    754  1.9.2.3    martin #undef atomic_inc_64
    755  1.9.2.3    martin #undef atomic_inc_32_nv
    756  1.9.2.3    martin #undef atomic_inc_uint_nv
    757  1.9.2.3    martin #undef atomic_inc_ulong_nv
    758  1.9.2.3    martin #undef atomic_inc_ptr_nv
    759  1.9.2.3    martin #undef atomic_inc_64_nv
    760  1.9.2.3    martin 
    761  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
    762  1.9.2.3    martin 	void atomic_add_##name(volatile targ1 *, targ2); \
    763  1.9.2.3    martin 	void kasan_atomic_add_##name(volatile targ1 *, targ2); \
    764  1.9.2.3    martin 	void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
    765  1.9.2.3    martin 	{ \
    766  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    767  1.9.2.3    martin 		    __RET_ADDR); \
    768  1.9.2.3    martin 		atomic_add_##name(ptr, val); \
    769  1.9.2.3    martin 	} \
    770  1.9.2.3    martin 	tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
    771  1.9.2.3    martin 	tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
    772  1.9.2.3    martin 	tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
    773  1.9.2.3    martin 	{ \
    774  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    775  1.9.2.3    martin 		    __RET_ADDR); \
    776  1.9.2.3    martin 		return atomic_add_##name##_nv(ptr, val); \
    777  1.9.2.3    martin 	}
    778  1.9.2.3    martin 
    779  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
    780  1.9.2.3    martin 	void atomic_and_##name(volatile targ1 *, targ2); \
    781  1.9.2.3    martin 	void kasan_atomic_and_##name(volatile targ1 *, targ2); \
    782  1.9.2.3    martin 	void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
    783  1.9.2.3    martin 	{ \
    784  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    785  1.9.2.3    martin 		    __RET_ADDR); \
    786  1.9.2.3    martin 		atomic_and_##name(ptr, val); \
    787  1.9.2.3    martin 	} \
    788  1.9.2.3    martin 	tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
    789  1.9.2.3    martin 	tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
    790  1.9.2.3    martin 	tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
    791  1.9.2.3    martin 	{ \
    792  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    793  1.9.2.3    martin 		    __RET_ADDR); \
    794  1.9.2.3    martin 		return atomic_and_##name##_nv(ptr, val); \
    795  1.9.2.3    martin 	}
    796  1.9.2.3    martin 
    797  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
    798  1.9.2.3    martin 	void atomic_or_##name(volatile targ1 *, targ2); \
    799  1.9.2.3    martin 	void kasan_atomic_or_##name(volatile targ1 *, targ2); \
    800  1.9.2.3    martin 	void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
    801  1.9.2.3    martin 	{ \
    802  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    803  1.9.2.3    martin 		    __RET_ADDR); \
    804  1.9.2.3    martin 		atomic_or_##name(ptr, val); \
    805  1.9.2.3    martin 	} \
    806  1.9.2.3    martin 	tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
    807  1.9.2.3    martin 	tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
    808  1.9.2.3    martin 	tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
    809  1.9.2.3    martin 	{ \
    810  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    811  1.9.2.3    martin 		    __RET_ADDR); \
    812  1.9.2.3    martin 		return atomic_or_##name##_nv(ptr, val); \
    813  1.9.2.3    martin 	}
    814  1.9.2.3    martin 
    815  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
    816  1.9.2.3    martin 	tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
    817  1.9.2.3    martin 	tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
    818  1.9.2.3    martin 	tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
    819  1.9.2.3    martin 	{ \
    820  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    821  1.9.2.3    martin 		    __RET_ADDR); \
    822  1.9.2.3    martin 		return atomic_cas_##name(ptr, exp, new); \
    823  1.9.2.3    martin 	} \
    824  1.9.2.3    martin 	tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
    825  1.9.2.3    martin 	tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
    826  1.9.2.3    martin 	tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
    827  1.9.2.3    martin 	{ \
    828  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    829  1.9.2.3    martin 		    __RET_ADDR); \
    830  1.9.2.3    martin 		return atomic_cas_##name##_ni(ptr, exp, new); \
    831  1.9.2.3    martin 	}
    832  1.9.2.3    martin 
    833  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
    834  1.9.2.3    martin 	tret atomic_swap_##name(volatile targ1 *, targ2); \
    835  1.9.2.3    martin 	tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \
    836  1.9.2.3    martin 	tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
    837  1.9.2.3    martin 	{ \
    838  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    839  1.9.2.3    martin 		    __RET_ADDR); \
    840  1.9.2.3    martin 		return atomic_swap_##name(ptr, val); \
    841  1.9.2.3    martin 	}
    842  1.9.2.3    martin 
    843  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
    844  1.9.2.3    martin 	void atomic_dec_##name(volatile targ1 *); \
    845  1.9.2.3    martin 	void kasan_atomic_dec_##name(volatile targ1 *); \
    846  1.9.2.3    martin 	void kasan_atomic_dec_##name(volatile targ1 *ptr) \
    847  1.9.2.3    martin 	{ \
    848  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    849  1.9.2.3    martin 		    __RET_ADDR); \
    850  1.9.2.3    martin 		atomic_dec_##name(ptr); \
    851  1.9.2.3    martin 	} \
    852  1.9.2.3    martin 	tret atomic_dec_##name##_nv(volatile targ1 *); \
    853  1.9.2.3    martin 	tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \
    854  1.9.2.3    martin 	tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
    855  1.9.2.3    martin 	{ \
    856  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    857  1.9.2.3    martin 		    __RET_ADDR); \
    858  1.9.2.3    martin 		return atomic_dec_##name##_nv(ptr); \
    859  1.9.2.3    martin 	}
    860  1.9.2.3    martin 
    861  1.9.2.3    martin #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \
    862  1.9.2.3    martin 	void atomic_inc_##name(volatile targ1 *); \
    863  1.9.2.3    martin 	void kasan_atomic_inc_##name(volatile targ1 *); \
    864  1.9.2.3    martin 	void kasan_atomic_inc_##name(volatile targ1 *ptr) \
    865  1.9.2.3    martin 	{ \
    866  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    867  1.9.2.3    martin 		    __RET_ADDR); \
    868  1.9.2.3    martin 		atomic_inc_##name(ptr); \
    869  1.9.2.3    martin 	} \
    870  1.9.2.3    martin 	tret atomic_inc_##name##_nv(volatile targ1 *); \
    871  1.9.2.3    martin 	tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \
    872  1.9.2.3    martin 	tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
    873  1.9.2.3    martin 	{ \
    874  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    875  1.9.2.3    martin 		    __RET_ADDR); \
    876  1.9.2.3    martin 		return atomic_inc_##name##_nv(ptr); \
    877  1.9.2.3    martin 	}
    878  1.9.2.3    martin 
    879  1.9.2.3    martin ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
    880  1.9.2.3    martin ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
    881  1.9.2.3    martin ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
    882  1.9.2.3    martin ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
    883  1.9.2.3    martin ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
    884  1.9.2.3    martin 
    885  1.9.2.3    martin ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
    886  1.9.2.3    martin ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
    887  1.9.2.3    martin ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
    888  1.9.2.3    martin ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
    889  1.9.2.3    martin 
    890  1.9.2.3    martin ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
    891  1.9.2.3    martin ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
    892  1.9.2.3    martin ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
    893  1.9.2.3    martin ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
    894  1.9.2.3    martin 
    895  1.9.2.3    martin ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
    896  1.9.2.3    martin ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
    897  1.9.2.3    martin ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
    898  1.9.2.3    martin ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
    899  1.9.2.3    martin ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
    900  1.9.2.3    martin 
    901  1.9.2.3    martin ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
    902  1.9.2.3    martin ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
    903  1.9.2.3    martin ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
    904  1.9.2.3    martin ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
    905  1.9.2.3    martin ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
    906  1.9.2.3    martin 
    907  1.9.2.3    martin ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
    908  1.9.2.3    martin ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
    909  1.9.2.3    martin ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
    910  1.9.2.3    martin ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
    911  1.9.2.3    martin ASAN_ATOMIC_FUNC_DEC(ptr, void *, void);
    912  1.9.2.3    martin 
    913  1.9.2.3    martin ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
    914  1.9.2.3    martin ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
    915  1.9.2.3    martin ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
    916  1.9.2.3    martin ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
    917  1.9.2.3    martin ASAN_ATOMIC_FUNC_INC(ptr, void *, void);
    918  1.9.2.3    martin 
    919  1.9.2.3    martin /* -------------------------------------------------------------------------- */
    920  1.9.2.3    martin 
    921  1.9.2.3    martin #ifdef __HAVE_KASAN_INSTR_BUS
    922  1.9.2.3    martin 
    923  1.9.2.3    martin #include <sys/bus.h>
    924  1.9.2.3    martin 
    925  1.9.2.3    martin #undef bus_space_read_multi_1
    926  1.9.2.3    martin #undef bus_space_read_multi_2
    927  1.9.2.3    martin #undef bus_space_read_multi_4
    928  1.9.2.3    martin #undef bus_space_read_multi_8
    929  1.9.2.3    martin #undef bus_space_read_multi_stream_1
    930  1.9.2.3    martin #undef bus_space_read_multi_stream_2
    931  1.9.2.3    martin #undef bus_space_read_multi_stream_4
    932  1.9.2.3    martin #undef bus_space_read_multi_stream_8
    933  1.9.2.3    martin #undef bus_space_read_region_1
    934  1.9.2.3    martin #undef bus_space_read_region_2
    935  1.9.2.3    martin #undef bus_space_read_region_4
    936  1.9.2.3    martin #undef bus_space_read_region_8
    937  1.9.2.3    martin #undef bus_space_read_region_stream_1
    938  1.9.2.3    martin #undef bus_space_read_region_stream_2
    939  1.9.2.3    martin #undef bus_space_read_region_stream_4
    940  1.9.2.3    martin #undef bus_space_read_region_stream_8
    941  1.9.2.3    martin #undef bus_space_write_multi_1
    942  1.9.2.3    martin #undef bus_space_write_multi_2
    943  1.9.2.3    martin #undef bus_space_write_multi_4
    944  1.9.2.3    martin #undef bus_space_write_multi_8
    945  1.9.2.3    martin #undef bus_space_write_multi_stream_1
    946  1.9.2.3    martin #undef bus_space_write_multi_stream_2
    947  1.9.2.3    martin #undef bus_space_write_multi_stream_4
    948  1.9.2.3    martin #undef bus_space_write_multi_stream_8
    949  1.9.2.3    martin #undef bus_space_write_region_1
    950  1.9.2.3    martin #undef bus_space_write_region_2
    951  1.9.2.3    martin #undef bus_space_write_region_4
    952  1.9.2.3    martin #undef bus_space_write_region_8
    953  1.9.2.3    martin #undef bus_space_write_region_stream_1
    954  1.9.2.3    martin #undef bus_space_write_region_stream_2
    955  1.9.2.3    martin #undef bus_space_write_region_stream_4
    956  1.9.2.3    martin #undef bus_space_write_region_stream_8
    957  1.9.2.3    martin 
    958  1.9.2.3    martin #define ASAN_BUS_READ_FUNC(bytes, bits) \
    959  1.9.2.3    martin 	void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t,	\
    960  1.9.2.3    martin 	    bus_size_t, uint##bits##_t *, bus_size_t);				\
    961  1.9.2.3    martin 	void kasan_bus_space_read_multi_##bytes(bus_space_tag_t,		\
    962  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    963  1.9.2.3    martin 	void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag,		\
    964  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    965  1.9.2.3    martin 	    bus_size_t count)							\
    966  1.9.2.3    martin 	{									\
    967  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
    968  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    969  1.9.2.3    martin 		bus_space_read_multi_##bytes(tag, hnd, size, buf, count);	\
    970  1.9.2.3    martin 	}									\
    971  1.9.2.3    martin 	void bus_space_read_multi_stream_##bytes(bus_space_tag_t,		\
    972  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    973  1.9.2.3    martin 	void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t,		\
    974  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    975  1.9.2.3    martin 	void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag,	\
    976  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    977  1.9.2.3    martin 	    bus_size_t count)							\
    978  1.9.2.3    martin 	{									\
    979  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
    980  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    981  1.9.2.3    martin 		bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
    982  1.9.2.3    martin 	}									\
    983  1.9.2.3    martin 	void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t,	\
    984  1.9.2.3    martin 	    bus_size_t, uint##bits##_t *, bus_size_t);				\
    985  1.9.2.3    martin 	void kasan_bus_space_read_region_##bytes(bus_space_tag_t,		\
    986  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    987  1.9.2.3    martin 	void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag,		\
    988  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    989  1.9.2.3    martin 	    bus_size_t count)							\
    990  1.9.2.3    martin 	{									\
    991  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
    992  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    993  1.9.2.3    martin 		bus_space_read_region_##bytes(tag, hnd, size, buf, count);	\
    994  1.9.2.3    martin 	}									\
    995  1.9.2.3    martin 	void bus_space_read_region_stream_##bytes(bus_space_tag_t,		\
    996  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    997  1.9.2.3    martin 	void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t,	\
    998  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    999  1.9.2.3    martin 	void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag,	\
   1000  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
   1001  1.9.2.3    martin 	    bus_size_t count)							\
   1002  1.9.2.3    martin 	{									\
   1003  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
   1004  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
   1005  1.9.2.3    martin 		bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
   1006  1.9.2.3    martin 	}
   1007  1.9.2.3    martin 
   1008  1.9.2.3    martin #define ASAN_BUS_WRITE_FUNC(bytes, bits) \
   1009  1.9.2.3    martin 	void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t,	\
   1010  1.9.2.3    martin 	    bus_size_t, const uint##bits##_t *, bus_size_t);			\
   1011  1.9.2.3    martin 	void kasan_bus_space_write_multi_##bytes(bus_space_tag_t,		\
   1012  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1013  1.9.2.3    martin 	void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag,		\
   1014  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
   1015  1.9.2.3    martin 	    bus_size_t count)							\
   1016  1.9.2.3    martin 	{									\
   1017  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
   1018  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
   1019  1.9.2.3    martin 		bus_space_write_multi_##bytes(tag, hnd, size, buf, count);	\
   1020  1.9.2.3    martin 	}									\
   1021  1.9.2.3    martin 	void bus_space_write_multi_stream_##bytes(bus_space_tag_t,		\
   1022  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1023  1.9.2.3    martin 	void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t,	\
   1024  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1025  1.9.2.3    martin 	void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag,	\
   1026  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
   1027  1.9.2.3    martin 	    bus_size_t count)							\
   1028  1.9.2.3    martin 	{									\
   1029  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
   1030  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
   1031  1.9.2.3    martin 		bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
   1032  1.9.2.3    martin 	}									\
   1033  1.9.2.3    martin 	void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
   1034  1.9.2.3    martin 	    bus_size_t, const uint##bits##_t *, bus_size_t);			\
   1035  1.9.2.3    martin 	void kasan_bus_space_write_region_##bytes(bus_space_tag_t,		\
   1036  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1037  1.9.2.3    martin 	void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag,		\
   1038  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
   1039  1.9.2.3    martin 	    bus_size_t count)							\
   1040  1.9.2.3    martin 	{									\
   1041  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
   1042  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
   1043  1.9.2.3    martin 		bus_space_write_region_##bytes(tag, hnd, size, buf, count);	\
   1044  1.9.2.3    martin 	}									\
   1045  1.9.2.3    martin 	void bus_space_write_region_stream_##bytes(bus_space_tag_t,		\
   1046  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1047  1.9.2.3    martin 	void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t,	\
   1048  1.9.2.3    martin 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1049  1.9.2.3    martin 	void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag,	\
   1050  1.9.2.3    martin 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
   1051  1.9.2.3    martin 	    bus_size_t count)							\
   1052  1.9.2.3    martin 	{									\
   1053  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)buf,				\
   1054  1.9.2.3    martin 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
   1055  1.9.2.3    martin 		bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
   1056  1.9.2.3    martin 	}
   1057  1.9.2.3    martin 
   1058  1.9.2.3    martin ASAN_BUS_READ_FUNC(1, 8)
   1059  1.9.2.3    martin ASAN_BUS_READ_FUNC(2, 16)
   1060  1.9.2.3    martin ASAN_BUS_READ_FUNC(4, 32)
   1061  1.9.2.3    martin ASAN_BUS_READ_FUNC(8, 64)
   1062  1.9.2.3    martin 
   1063  1.9.2.3    martin ASAN_BUS_WRITE_FUNC(1, 8)
   1064  1.9.2.3    martin ASAN_BUS_WRITE_FUNC(2, 16)
   1065  1.9.2.3    martin ASAN_BUS_WRITE_FUNC(4, 32)
   1066  1.9.2.3    martin ASAN_BUS_WRITE_FUNC(8, 64)
   1067  1.9.2.3    martin 
   1068  1.9.2.3    martin #endif /* __HAVE_KASAN_INSTR_BUS */
   1069  1.9.2.3    martin 
   1070  1.9.2.3    martin /* -------------------------------------------------------------------------- */
   1071  1.9.2.3    martin 
   1072  1.9.2.3    martin #include <sys/mbuf.h>
   1073  1.9.2.3    martin 
   1074  1.9.2.3    martin static void
   1075  1.9.2.3    martin kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
   1076  1.9.2.3    martin     bool write, uintptr_t pc)
   1077  1.9.2.3    martin {
   1078  1.9.2.3    martin 	kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc);
   1079  1.9.2.3    martin }
   1080  1.9.2.3    martin 
   1081  1.9.2.3    martin static void
   1082  1.9.2.3    martin kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
   1083  1.9.2.3    martin     bool write, uintptr_t pc)
   1084  1.9.2.3    martin {
   1085  1.9.2.3    martin 	bus_addr_t minlen;
   1086  1.9.2.3    martin 
   1087  1.9.2.3    martin 	for (; m != NULL && len != 0; m = m->m_next) {
   1088  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc);
   1089  1.9.2.3    martin 
   1090  1.9.2.3    martin 		if (offset >= m->m_len) {
   1091  1.9.2.3    martin 			offset -= m->m_len;
   1092  1.9.2.3    martin 			continue;
   1093  1.9.2.3    martin 		}
   1094  1.9.2.3    martin 
   1095  1.9.2.3    martin 		minlen = MIN(len, m->m_len - offset);
   1096  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset),
   1097  1.9.2.3    martin 		    minlen, write, pc);
   1098  1.9.2.3    martin 
   1099  1.9.2.3    martin 		offset = 0;
   1100  1.9.2.3    martin 		len -= minlen;
   1101  1.9.2.3    martin 	}
   1102  1.9.2.3    martin }
   1103  1.9.2.3    martin 
   1104  1.9.2.3    martin static void
   1105  1.9.2.3    martin kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
   1106  1.9.2.3    martin     bool write, uintptr_t pc)
   1107  1.9.2.3    martin {
   1108  1.9.2.3    martin 	bus_size_t minlen, resid;
   1109  1.9.2.3    martin 	struct iovec *iov;
   1110  1.9.2.3    martin 	int i;
   1111  1.9.2.3    martin 
   1112  1.9.2.3    martin 	kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc);
   1113  1.9.2.3    martin 
   1114  1.9.2.3    martin 	if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
   1115  1.9.2.3    martin 		return;
   1116  1.9.2.3    martin 
   1117  1.9.2.3    martin 	resid = uio->uio_resid;
   1118  1.9.2.3    martin 	iov = uio->uio_iov;
   1119  1.9.2.3    martin 
   1120  1.9.2.3    martin 	for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
   1121  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]),
   1122  1.9.2.3    martin 		    false, pc);
   1123  1.9.2.3    martin 		minlen = MIN(resid, iov[i].iov_len);
   1124  1.9.2.3    martin 		kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen,
   1125  1.9.2.3    martin 		    write, pc);
   1126  1.9.2.3    martin 		resid -= minlen;
   1127  1.9.2.3    martin 	}
   1128  1.9.2.3    martin }
   1129  1.9.2.3    martin 
   1130  1.9.2.3    martin void
   1131  1.9.2.3    martin kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
   1132  1.9.2.3    martin {
   1133  1.9.2.3    martin 	bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0;
   1134  1.9.2.3    martin 
   1135  1.9.2.3    martin 	switch (map->dm_buftype) {
   1136  1.9.2.3    martin 	case KASAN_DMA_LINEAR:
   1137  1.9.2.3    martin 		kasan_dma_sync_linear(map->dm_buf, offset, len, write,
   1138  1.9.2.3    martin 		    __RET_ADDR);
   1139  1.9.2.3    martin 		break;
   1140  1.9.2.3    martin 	case KASAN_DMA_MBUF:
   1141  1.9.2.3    martin 		kasan_dma_sync_mbuf(map->dm_buf, offset, len, write,
   1142  1.9.2.3    martin 		    __RET_ADDR);
   1143  1.9.2.3    martin 		break;
   1144  1.9.2.3    martin 	case KASAN_DMA_UIO:
   1145  1.9.2.3    martin 		kasan_dma_sync_uio(map->dm_buf, offset, len, write,
   1146  1.9.2.3    martin 		    __RET_ADDR);
   1147  1.9.2.3    martin 		break;
   1148  1.9.2.3    martin 	case KASAN_DMA_RAW:
   1149  1.9.2.3    martin 		break;
   1150  1.9.2.3    martin 	default:
   1151  1.9.2.3    martin 		panic("%s: impossible", __func__);
   1152  1.9.2.3    martin 	}
   1153  1.9.2.3    martin }
   1154  1.9.2.3    martin 
   1155  1.9.2.3    martin void
   1156  1.9.2.3    martin kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
   1157  1.9.2.3    martin {
   1158  1.9.2.3    martin 	map->dm_buf = buf;
   1159  1.9.2.3    martin 	map->dm_buflen = buflen;
   1160  1.9.2.3    martin 	map->dm_buftype = type;
   1161  1.9.2.3    martin }
   1162  1.9.2.3    martin 
   1163  1.9.2.3    martin /* -------------------------------------------------------------------------- */
   1164  1.9.2.3    martin 
   1165  1.9.2.2  christos void __asan_register_globals(struct __asan_global *, size_t);
   1166  1.9.2.2  christos void __asan_unregister_globals(struct __asan_global *, size_t);
   1167  1.9.2.2  christos 
   1168  1.9.2.2  christos void
   1169  1.9.2.2  christos __asan_register_globals(struct __asan_global *globals, size_t n)
   1170  1.9.2.2  christos {
   1171  1.9.2.2  christos 	size_t i;
   1172  1.9.2.2  christos 
   1173  1.9.2.2  christos 	for (i = 0; i < n; i++) {
   1174  1.9.2.2  christos 		kasan_mark(globals[i].beg, globals[i].size,
   1175  1.9.2.2  christos 		    globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
   1176  1.9.2.2  christos 	}
   1177  1.9.2.2  christos }
   1178  1.9.2.2  christos 
   1179  1.9.2.2  christos void
   1180  1.9.2.2  christos __asan_unregister_globals(struct __asan_global *globals, size_t n)
   1181  1.9.2.2  christos {
   1182  1.9.2.2  christos 	/* never called */
   1183  1.9.2.2  christos }
   1184  1.9.2.2  christos 
   1185  1.9.2.2  christos #define ASAN_LOAD_STORE(size)					\
   1186  1.9.2.2  christos 	void __asan_load##size(unsigned long);			\
   1187  1.9.2.2  christos 	void __asan_load##size(unsigned long addr)		\
   1188  1.9.2.2  christos 	{							\
   1189  1.9.2.2  christos 		kasan_shadow_check(addr, size, false, __RET_ADDR);\
   1190  1.9.2.2  christos 	} 							\
   1191  1.9.2.2  christos 	void __asan_load##size##_noabort(unsigned long);	\
   1192  1.9.2.2  christos 	void __asan_load##size##_noabort(unsigned long addr)	\
   1193  1.9.2.2  christos 	{							\
   1194  1.9.2.2  christos 		kasan_shadow_check(addr, size, false, __RET_ADDR);\
   1195  1.9.2.2  christos 	}							\
   1196  1.9.2.2  christos 	void __asan_store##size(unsigned long);			\
   1197  1.9.2.2  christos 	void __asan_store##size(unsigned long addr)		\
   1198  1.9.2.2  christos 	{							\
   1199  1.9.2.2  christos 		kasan_shadow_check(addr, size, true, __RET_ADDR);\
   1200  1.9.2.2  christos 	}							\
   1201  1.9.2.2  christos 	void __asan_store##size##_noabort(unsigned long);	\
   1202  1.9.2.2  christos 	void __asan_store##size##_noabort(unsigned long addr)	\
   1203  1.9.2.2  christos 	{							\
   1204  1.9.2.2  christos 		kasan_shadow_check(addr, size, true, __RET_ADDR);\
   1205  1.9.2.2  christos 	}
   1206  1.9.2.2  christos 
   1207  1.9.2.2  christos ASAN_LOAD_STORE(1);
   1208  1.9.2.2  christos ASAN_LOAD_STORE(2);
   1209  1.9.2.2  christos ASAN_LOAD_STORE(4);
   1210  1.9.2.2  christos ASAN_LOAD_STORE(8);
   1211  1.9.2.2  christos ASAN_LOAD_STORE(16);
   1212  1.9.2.2  christos 
   1213  1.9.2.2  christos void __asan_loadN(unsigned long, size_t);
   1214  1.9.2.2  christos void __asan_loadN_noabort(unsigned long, size_t);
   1215  1.9.2.2  christos void __asan_storeN(unsigned long, size_t);
   1216  1.9.2.2  christos void __asan_storeN_noabort(unsigned long, size_t);
   1217  1.9.2.2  christos void __asan_handle_no_return(void);
   1218  1.9.2.2  christos 
   1219  1.9.2.2  christos void
   1220  1.9.2.2  christos __asan_loadN(unsigned long addr, size_t size)
   1221  1.9.2.2  christos {
   1222  1.9.2.2  christos 	kasan_shadow_check(addr, size, false, __RET_ADDR);
   1223  1.9.2.2  christos }
   1224  1.9.2.2  christos 
   1225  1.9.2.2  christos void
   1226  1.9.2.2  christos __asan_loadN_noabort(unsigned long addr, size_t size)
   1227  1.9.2.2  christos {
   1228  1.9.2.2  christos 	kasan_shadow_check(addr, size, false, __RET_ADDR);
   1229  1.9.2.2  christos }
   1230  1.9.2.2  christos 
   1231  1.9.2.2  christos void
   1232  1.9.2.2  christos __asan_storeN(unsigned long addr, size_t size)
   1233  1.9.2.2  christos {
   1234  1.9.2.2  christos 	kasan_shadow_check(addr, size, true, __RET_ADDR);
   1235  1.9.2.2  christos }
   1236  1.9.2.2  christos 
   1237  1.9.2.2  christos void
   1238  1.9.2.2  christos __asan_storeN_noabort(unsigned long addr, size_t size)
   1239  1.9.2.2  christos {
   1240  1.9.2.2  christos 	kasan_shadow_check(addr, size, true, __RET_ADDR);
   1241  1.9.2.2  christos }
   1242  1.9.2.2  christos 
   1243  1.9.2.2  christos void
   1244  1.9.2.2  christos __asan_handle_no_return(void)
   1245  1.9.2.2  christos {
   1246  1.9.2.2  christos 	/* nothing */
   1247  1.9.2.2  christos }
   1248  1.9.2.2  christos 
   1249  1.9.2.2  christos #define ASAN_SET_SHADOW(byte) \
   1250  1.9.2.2  christos 	void __asan_set_shadow_##byte(void *, size_t);			\
   1251  1.9.2.2  christos 	void __asan_set_shadow_##byte(void *addr, size_t size)		\
   1252  1.9.2.2  christos 	{								\
   1253  1.9.2.2  christos 		__builtin_memset((void *)addr, 0x##byte, size);		\
   1254  1.9.2.2  christos 	}
   1255  1.9.2.2  christos 
   1256  1.9.2.2  christos ASAN_SET_SHADOW(00);
   1257  1.9.2.2  christos ASAN_SET_SHADOW(f1);
   1258  1.9.2.2  christos ASAN_SET_SHADOW(f2);
   1259  1.9.2.2  christos ASAN_SET_SHADOW(f3);
   1260  1.9.2.2  christos ASAN_SET_SHADOW(f5);
   1261  1.9.2.2  christos ASAN_SET_SHADOW(f8);
   1262  1.9.2.2  christos 
   1263  1.9.2.2  christos void __asan_poison_stack_memory(const void *, size_t);
   1264  1.9.2.2  christos void __asan_unpoison_stack_memory(const void *, size_t);
   1265  1.9.2.2  christos 
   1266  1.9.2.3    martin void
   1267  1.9.2.3    martin __asan_poison_stack_memory(const void *addr, size_t size)
   1268  1.9.2.2  christos {
   1269  1.9.2.2  christos 	size = roundup(size, KASAN_SHADOW_SCALE_SIZE);
   1270  1.9.2.2  christos 	kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
   1271  1.9.2.2  christos }
   1272  1.9.2.2  christos 
   1273  1.9.2.3    martin void
   1274  1.9.2.3    martin __asan_unpoison_stack_memory(const void *addr, size_t size)
   1275  1.9.2.2  christos {
   1276  1.9.2.2  christos 	kasan_shadow_Nbyte_markvalid(addr, size);
   1277  1.9.2.2  christos }
   1278  1.9.2.3    martin 
   1279  1.9.2.3    martin void __asan_alloca_poison(const void *, size_t);
   1280  1.9.2.3    martin void __asan_allocas_unpoison(const void *, const void *);
   1281  1.9.2.3    martin 
   1282  1.9.2.3    martin void __asan_alloca_poison(const void *addr, size_t size)
   1283  1.9.2.3    martin {
   1284  1.9.2.3    martin 	const void *l, *r;
   1285  1.9.2.3    martin 
   1286  1.9.2.3    martin 	KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0);
   1287  1.9.2.3    martin 
   1288  1.9.2.3    martin 	l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
   1289  1.9.2.3    martin 	r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
   1290  1.9.2.3    martin 
   1291  1.9.2.3    martin 	kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
   1292  1.9.2.3    martin 	kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
   1293  1.9.2.3    martin 	    KASAN_STACK_MID);
   1294  1.9.2.3    martin 	kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
   1295  1.9.2.3    martin }
   1296  1.9.2.3    martin 
   1297  1.9.2.3    martin void __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
   1298  1.9.2.3    martin {
   1299  1.9.2.3    martin 	size_t size;
   1300  1.9.2.3    martin 
   1301  1.9.2.3    martin 	if (__predict_false(!stkbegin))
   1302  1.9.2.3    martin 		return;
   1303  1.9.2.3    martin 	if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
   1304  1.9.2.3    martin 		return;
   1305  1.9.2.3    martin 	size = (uintptr_t)stkend - (uintptr_t)stkbegin;
   1306  1.9.2.3    martin 
   1307  1.9.2.3    martin 	kasan_shadow_Nbyte_fill(stkbegin, size, 0);
   1308  1.9.2.3    martin }
   1309