Home | History | Annotate | Line # | Download | only in kern
subr_asan.c revision 1.16.2.1
      1  1.16.2.1    ad /*	$NetBSD: subr_asan.c,v 1.16.2.1 2020/01/25 22:38:51 ad Exp $	*/
      2       1.1  maxv 
      3       1.1  maxv /*
      4  1.16.2.1    ad  * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
      5       1.1  maxv  * All rights reserved.
      6       1.1  maxv  *
      7       1.1  maxv  * This code is derived from software contributed to The NetBSD Foundation
      8       1.1  maxv  * by Maxime Villard.
      9       1.1  maxv  *
     10       1.1  maxv  * Redistribution and use in source and binary forms, with or without
     11       1.1  maxv  * modification, are permitted provided that the following conditions
     12       1.1  maxv  * are met:
     13       1.1  maxv  * 1. Redistributions of source code must retain the above copyright
     14       1.1  maxv  *    notice, this list of conditions and the following disclaimer.
     15       1.1  maxv  * 2. Redistributions in binary form must reproduce the above copyright
     16       1.1  maxv  *    notice, this list of conditions and the following disclaimer in the
     17       1.1  maxv  *    documentation and/or other materials provided with the distribution.
     18       1.1  maxv  *
     19       1.1  maxv  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20       1.1  maxv  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21       1.1  maxv  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22       1.1  maxv  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23       1.1  maxv  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24       1.1  maxv  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25       1.1  maxv  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26       1.1  maxv  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27       1.1  maxv  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28       1.1  maxv  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29       1.1  maxv  * POSSIBILITY OF SUCH DAMAGE.
     30       1.1  maxv  */
     31       1.1  maxv 
     32       1.1  maxv #include <sys/cdefs.h>
     33  1.16.2.1    ad __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.16.2.1 2020/01/25 22:38:51 ad Exp $");
     34       1.1  maxv 
     35       1.1  maxv #include <sys/param.h>
     36       1.1  maxv #include <sys/device.h>
     37       1.1  maxv #include <sys/kernel.h>
     38       1.1  maxv #include <sys/param.h>
     39       1.1  maxv #include <sys/conf.h>
     40       1.1  maxv #include <sys/systm.h>
     41       1.1  maxv #include <sys/types.h>
     42       1.1  maxv #include <sys/asan.h>
     43       1.1  maxv 
     44       1.1  maxv #include <uvm/uvm.h>
     45       1.1  maxv 
     46      1.10  maxv #ifdef KASAN_PANIC
     47      1.10  maxv #define REPORT panic
     48      1.10  maxv #else
     49      1.10  maxv #define REPORT printf
     50      1.10  maxv #endif
     51      1.10  maxv 
     52       1.1  maxv /* ASAN constants. Part of the compiler ABI. */
     53       1.1  maxv #define KASAN_SHADOW_SCALE_SHIFT	3
     54       1.1  maxv #define KASAN_SHADOW_SCALE_SIZE		(1UL << KASAN_SHADOW_SCALE_SHIFT)
     55       1.1  maxv #define KASAN_SHADOW_MASK		(KASAN_SHADOW_SCALE_SIZE - 1)
     56       1.1  maxv 
     57       1.1  maxv /* The MD code. */
     58       1.1  maxv #include <machine/asan.h>
     59       1.1  maxv 
     60       1.1  maxv /* ASAN ABI version. */
     61       1.1  maxv #if defined(__clang__) && (__clang_major__ - 0 >= 6)
     62       1.1  maxv #define ASAN_ABI_VERSION	8
     63       1.1  maxv #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
     64       1.1  maxv #define ASAN_ABI_VERSION	8
     65       1.1  maxv #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
     66       1.1  maxv #define ASAN_ABI_VERSION	6
     67       1.1  maxv #else
     68       1.1  maxv #error "Unsupported compiler version"
     69       1.1  maxv #endif
     70       1.1  maxv 
     71       1.1  maxv #define __RET_ADDR	(unsigned long)__builtin_return_address(0)
     72       1.1  maxv 
     73       1.1  maxv /* Global variable descriptor. Part of the compiler ABI.  */
     74       1.1  maxv struct __asan_global_source_location {
     75       1.1  maxv 	const char *filename;
     76       1.1  maxv 	int line_no;
     77       1.1  maxv 	int column_no;
     78       1.1  maxv };
     79       1.1  maxv struct __asan_global {
     80       1.1  maxv 	const void *beg;		/* address of the global variable */
     81       1.1  maxv 	size_t size;			/* size of the global variable */
     82       1.1  maxv 	size_t size_with_redzone;	/* size with the redzone */
     83       1.1  maxv 	const void *name;		/* name of the variable */
     84       1.1  maxv 	const void *module_name;	/* name of the module where the var is declared */
     85       1.1  maxv 	unsigned long has_dynamic_init;	/* the var has dyn initializer (c++) */
     86       1.1  maxv 	struct __asan_global_source_location *location;
     87       1.1  maxv #if ASAN_ABI_VERSION >= 7
     88       1.1  maxv 	uintptr_t odr_indicator;	/* the address of the ODR indicator symbol */
     89       1.1  maxv #endif
     90       1.1  maxv };
     91       1.1  maxv 
     92       1.1  maxv static bool kasan_enabled __read_mostly = false;
     93       1.1  maxv 
     94       1.1  maxv /* -------------------------------------------------------------------------- */
     95       1.1  maxv 
     96       1.1  maxv void
     97       1.1  maxv kasan_shadow_map(void *addr, size_t size)
     98       1.1  maxv {
     99       1.1  maxv 	size_t sz, npages, i;
    100       1.1  maxv 	vaddr_t sva, eva;
    101       1.1  maxv 
    102       1.1  maxv 	KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
    103       1.1  maxv 
    104       1.1  maxv 	sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE;
    105       1.1  maxv 
    106       1.1  maxv 	sva = (vaddr_t)kasan_md_addr_to_shad(addr);
    107       1.1  maxv 	eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz;
    108       1.1  maxv 
    109       1.1  maxv 	sva = rounddown(sva, PAGE_SIZE);
    110       1.1  maxv 	eva = roundup(eva, PAGE_SIZE);
    111       1.1  maxv 
    112       1.1  maxv 	npages = (eva - sva) / PAGE_SIZE;
    113       1.1  maxv 
    114       1.1  maxv 	KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END);
    115       1.1  maxv 
    116       1.1  maxv 	for (i = 0; i < npages; i++) {
    117       1.1  maxv 		kasan_md_shadow_map_page(sva + i * PAGE_SIZE);
    118       1.1  maxv 	}
    119       1.1  maxv }
    120       1.1  maxv 
    121       1.1  maxv static void
    122       1.1  maxv kasan_ctors(void)
    123       1.1  maxv {
    124       1.1  maxv 	extern uint64_t __CTOR_LIST__, __CTOR_END__;
    125       1.1  maxv 	size_t nentries, i;
    126       1.1  maxv 	uint64_t *ptr;
    127       1.1  maxv 
    128       1.1  maxv 	nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) /
    129       1.1  maxv 	    sizeof(uintptr_t);
    130       1.1  maxv 
    131       1.1  maxv 	ptr = &__CTOR_LIST__;
    132       1.1  maxv 	for (i = 0; i < nentries; i++) {
    133       1.1  maxv 		void (*func)(void);
    134       1.1  maxv 
    135       1.1  maxv 		func = (void *)(*ptr);
    136       1.1  maxv 		(*func)();
    137       1.1  maxv 
    138       1.1  maxv 		ptr++;
    139       1.1  maxv 	}
    140       1.1  maxv }
    141       1.1  maxv 
    142       1.1  maxv void
    143       1.1  maxv kasan_early_init(void *stack)
    144       1.1  maxv {
    145       1.1  maxv 	kasan_md_early_init(stack);
    146       1.1  maxv }
    147       1.1  maxv 
    148       1.1  maxv void
    149       1.1  maxv kasan_init(void)
    150       1.1  maxv {
    151       1.1  maxv 	/* MD initialization. */
    152       1.1  maxv 	kasan_md_init();
    153       1.1  maxv 
    154       1.1  maxv 	/* Now officially enabled. */
    155       1.1  maxv 	kasan_enabled = true;
    156       1.1  maxv 
    157       1.1  maxv 	/* Call the ASAN constructors. */
    158       1.1  maxv 	kasan_ctors();
    159       1.1  maxv }
    160       1.1  maxv 
    161       1.5  maxv static inline const char *
    162       1.5  maxv kasan_code_name(uint8_t code)
    163       1.5  maxv {
    164       1.5  maxv 	switch (code) {
    165       1.6  maxv 	case KASAN_GENERIC_REDZONE:
    166       1.6  maxv 		return "GenericRedZone";
    167       1.6  maxv 	case KASAN_MALLOC_REDZONE:
    168       1.6  maxv 		return "MallocRedZone";
    169       1.6  maxv 	case KASAN_KMEM_REDZONE:
    170       1.6  maxv 		return "KmemRedZone";
    171       1.6  maxv 	case KASAN_POOL_REDZONE:
    172       1.6  maxv 		return "PoolRedZone";
    173       1.6  maxv 	case KASAN_POOL_FREED:
    174       1.6  maxv 		return "PoolUseAfterFree";
    175       1.5  maxv 	case 1 ... 7:
    176       1.5  maxv 		return "RedZonePartial";
    177       1.5  maxv 	case KASAN_STACK_LEFT:
    178       1.5  maxv 		return "StackLeft";
    179       1.5  maxv 	case KASAN_STACK_RIGHT:
    180       1.5  maxv 		return "StackRight";
    181       1.5  maxv 	case KASAN_STACK_PARTIAL:
    182       1.5  maxv 		return "StackPartial";
    183       1.5  maxv 	case KASAN_USE_AFTER_SCOPE:
    184       1.5  maxv 		return "UseAfterScope";
    185       1.5  maxv 	default:
    186       1.5  maxv 		return "Unknown";
    187       1.5  maxv 	}
    188       1.5  maxv }
    189       1.5  maxv 
    190       1.1  maxv static void
    191       1.5  maxv kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
    192       1.5  maxv     uint8_t code)
    193       1.1  maxv {
    194      1.10  maxv 	REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s,"
    195       1.5  maxv 	    " %s]\n",
    196       1.1  maxv 	    (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""),
    197       1.5  maxv 	    (write ? "write" : "read"), kasan_code_name(code));
    198       1.1  maxv 	kasan_md_unwind();
    199       1.1  maxv }
    200       1.1  maxv 
    201       1.1  maxv static __always_inline void
    202       1.1  maxv kasan_shadow_1byte_markvalid(unsigned long addr)
    203       1.1  maxv {
    204       1.1  maxv 	int8_t *byte = kasan_md_addr_to_shad((void *)addr);
    205       1.1  maxv 	int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
    206       1.1  maxv 
    207       1.1  maxv 	*byte = last;
    208       1.1  maxv }
    209       1.1  maxv 
    210       1.1  maxv static __always_inline void
    211       1.4  maxv kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
    212       1.4  maxv {
    213       1.4  maxv 	size_t i;
    214       1.4  maxv 
    215       1.4  maxv 	for (i = 0; i < size; i++) {
    216       1.4  maxv 		kasan_shadow_1byte_markvalid((unsigned long)addr+i);
    217       1.4  maxv 	}
    218       1.4  maxv }
    219       1.4  maxv 
    220       1.4  maxv static __always_inline void
    221       1.6  maxv kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
    222       1.1  maxv {
    223       1.1  maxv 	void *shad;
    224       1.1  maxv 
    225       1.1  maxv 	if (__predict_false(size == 0))
    226       1.1  maxv 		return;
    227       1.1  maxv 	if (__predict_false(kasan_md_unsupported((vaddr_t)addr)))
    228       1.1  maxv 		return;
    229       1.1  maxv 
    230       1.1  maxv 	KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
    231       1.1  maxv 	KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
    232       1.1  maxv 
    233       1.1  maxv 	shad = (void *)kasan_md_addr_to_shad(addr);
    234       1.1  maxv 	size = size >> KASAN_SHADOW_SCALE_SHIFT;
    235       1.1  maxv 
    236       1.6  maxv 	__builtin_memset(shad, code, size);
    237       1.1  maxv }
    238       1.1  maxv 
    239       1.1  maxv void
    240       1.1  maxv kasan_add_redzone(size_t *size)
    241       1.1  maxv {
    242       1.1  maxv 	*size = roundup(*size, KASAN_SHADOW_SCALE_SIZE);
    243       1.1  maxv 	*size += KASAN_SHADOW_SCALE_SIZE;
    244       1.1  maxv }
    245       1.1  maxv 
    246       1.1  maxv void
    247       1.1  maxv kasan_softint(struct lwp *l)
    248       1.1  maxv {
    249       1.1  maxv 	const void *stk = (const void *)uvm_lwp_getuarea(l);
    250       1.1  maxv 
    251       1.1  maxv 	kasan_shadow_Nbyte_fill(stk, USPACE, 0);
    252       1.1  maxv }
    253       1.1  maxv 
    254       1.2  maxv /*
    255       1.2  maxv  * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
    256       1.2  maxv  * and the rest as invalid. There are generally two use cases:
    257       1.2  maxv  *
    258       1.6  maxv  *  o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
    259       1.6  maxv  *    the redzone at the end of the buffer as invalid.
    260       1.2  maxv  *
    261       1.6  maxv  *  o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
    262       1.2  maxv  */
    263       1.1  maxv void
    264       1.6  maxv kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code)
    265       1.1  maxv {
    266       1.9  maxv 	size_t i, n, redz;
    267       1.9  maxv 	int8_t *shad;
    268       1.9  maxv 
    269       1.9  maxv 	KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
    270       1.9  maxv 	redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE);
    271       1.9  maxv 	KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0);
    272       1.9  maxv 	shad = kasan_md_addr_to_shad(addr);
    273       1.9  maxv 
    274       1.9  maxv 	/* Chunks of 8 bytes, valid. */
    275       1.9  maxv 	n = size / KASAN_SHADOW_SCALE_SIZE;
    276       1.9  maxv 	for (i = 0; i < n; i++) {
    277       1.9  maxv 		*shad++ = 0;
    278       1.9  maxv 	}
    279       1.9  maxv 
    280       1.9  maxv 	/* Possibly one chunk, mid. */
    281       1.9  maxv 	if ((size & KASAN_SHADOW_MASK) != 0) {
    282       1.9  maxv 		*shad++ = (size & KASAN_SHADOW_MASK);
    283       1.9  maxv 	}
    284       1.9  maxv 
    285       1.9  maxv 	/* Chunks of 8 bytes, invalid. */
    286       1.9  maxv 	n = redz / KASAN_SHADOW_SCALE_SIZE;
    287       1.9  maxv 	for (i = 0; i < n; i++) {
    288       1.9  maxv 		*shad++ = code;
    289       1.9  maxv 	}
    290       1.1  maxv }
    291       1.1  maxv 
    292       1.1  maxv /* -------------------------------------------------------------------------- */
    293       1.1  maxv 
    294       1.1  maxv #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) 		\
    295       1.1  maxv 	(addr >> KASAN_SHADOW_SCALE_SHIFT) !=			\
    296       1.1  maxv 	    ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
    297       1.1  maxv 
    298       1.1  maxv static __always_inline bool
    299       1.5  maxv kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
    300       1.1  maxv {
    301       1.1  maxv 	int8_t *byte = kasan_md_addr_to_shad((void *)addr);
    302       1.1  maxv 	int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
    303       1.1  maxv 
    304       1.5  maxv 	if (__predict_true(*byte == 0 || last <= *byte)) {
    305       1.5  maxv 		return true;
    306       1.5  maxv 	}
    307       1.5  maxv 	*code = *byte;
    308       1.5  maxv 	return false;
    309       1.1  maxv }
    310       1.1  maxv 
    311       1.1  maxv static __always_inline bool
    312       1.5  maxv kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
    313       1.1  maxv {
    314       1.1  maxv 	int8_t *byte, last;
    315       1.1  maxv 
    316       1.1  maxv 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
    317       1.5  maxv 		return (kasan_shadow_1byte_isvalid(addr, code) &&
    318       1.5  maxv 		    kasan_shadow_1byte_isvalid(addr+1, code));
    319       1.1  maxv 	}
    320       1.1  maxv 
    321       1.1  maxv 	byte = kasan_md_addr_to_shad((void *)addr);
    322       1.1  maxv 	last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
    323       1.1  maxv 
    324       1.5  maxv 	if (__predict_true(*byte == 0 || last <= *byte)) {
    325       1.5  maxv 		return true;
    326       1.5  maxv 	}
    327       1.5  maxv 	*code = *byte;
    328       1.5  maxv 	return false;
    329       1.1  maxv }
    330       1.1  maxv 
    331       1.1  maxv static __always_inline bool
    332       1.5  maxv kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
    333       1.1  maxv {
    334       1.1  maxv 	int8_t *byte, last;
    335       1.1  maxv 
    336       1.1  maxv 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
    337       1.5  maxv 		return (kasan_shadow_2byte_isvalid(addr, code) &&
    338       1.5  maxv 		    kasan_shadow_2byte_isvalid(addr+2, code));
    339       1.1  maxv 	}
    340       1.1  maxv 
    341       1.1  maxv 	byte = kasan_md_addr_to_shad((void *)addr);
    342       1.1  maxv 	last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
    343       1.1  maxv 
    344       1.5  maxv 	if (__predict_true(*byte == 0 || last <= *byte)) {
    345       1.5  maxv 		return true;
    346       1.5  maxv 	}
    347       1.5  maxv 	*code = *byte;
    348       1.5  maxv 	return false;
    349       1.1  maxv }
    350       1.1  maxv 
    351       1.1  maxv static __always_inline bool
    352       1.5  maxv kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
    353       1.1  maxv {
    354       1.1  maxv 	int8_t *byte, last;
    355       1.1  maxv 
    356       1.1  maxv 	if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
    357       1.5  maxv 		return (kasan_shadow_4byte_isvalid(addr, code) &&
    358       1.5  maxv 		    kasan_shadow_4byte_isvalid(addr+4, code));
    359       1.1  maxv 	}
    360       1.1  maxv 
    361       1.1  maxv 	byte = kasan_md_addr_to_shad((void *)addr);
    362       1.1  maxv 	last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
    363       1.1  maxv 
    364       1.5  maxv 	if (__predict_true(*byte == 0 || last <= *byte)) {
    365       1.5  maxv 		return true;
    366       1.5  maxv 	}
    367       1.5  maxv 	*code = *byte;
    368       1.5  maxv 	return false;
    369       1.1  maxv }
    370       1.1  maxv 
    371       1.1  maxv static __always_inline bool
    372       1.5  maxv kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
    373       1.1  maxv {
    374       1.1  maxv 	size_t i;
    375       1.1  maxv 
    376       1.1  maxv 	for (i = 0; i < size; i++) {
    377       1.5  maxv 		if (!kasan_shadow_1byte_isvalid(addr+i, code))
    378       1.1  maxv 			return false;
    379       1.1  maxv 	}
    380       1.1  maxv 
    381       1.1  maxv 	return true;
    382       1.1  maxv }
    383       1.1  maxv 
    384       1.1  maxv static __always_inline void
    385       1.1  maxv kasan_shadow_check(unsigned long addr, size_t size, bool write,
    386       1.1  maxv     unsigned long retaddr)
    387       1.1  maxv {
    388       1.5  maxv 	uint8_t code;
    389       1.1  maxv 	bool valid;
    390       1.1  maxv 
    391       1.1  maxv 	if (__predict_false(!kasan_enabled))
    392       1.1  maxv 		return;
    393       1.1  maxv 	if (__predict_false(size == 0))
    394       1.1  maxv 		return;
    395       1.1  maxv 	if (__predict_false(kasan_md_unsupported(addr)))
    396       1.1  maxv 		return;
    397       1.1  maxv 
    398       1.1  maxv 	if (__builtin_constant_p(size)) {
    399       1.1  maxv 		switch (size) {
    400       1.1  maxv 		case 1:
    401       1.5  maxv 			valid = kasan_shadow_1byte_isvalid(addr, &code);
    402       1.1  maxv 			break;
    403       1.1  maxv 		case 2:
    404       1.5  maxv 			valid = kasan_shadow_2byte_isvalid(addr, &code);
    405       1.1  maxv 			break;
    406       1.1  maxv 		case 4:
    407       1.5  maxv 			valid = kasan_shadow_4byte_isvalid(addr, &code);
    408       1.1  maxv 			break;
    409       1.1  maxv 		case 8:
    410       1.5  maxv 			valid = kasan_shadow_8byte_isvalid(addr, &code);
    411       1.1  maxv 			break;
    412       1.1  maxv 		default:
    413       1.5  maxv 			valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
    414       1.1  maxv 			break;
    415       1.1  maxv 		}
    416       1.1  maxv 	} else {
    417       1.5  maxv 		valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
    418       1.1  maxv 	}
    419       1.1  maxv 
    420       1.1  maxv 	if (__predict_false(!valid)) {
    421       1.5  maxv 		kasan_report(addr, size, write, retaddr, code);
    422       1.1  maxv 	}
    423       1.1  maxv }
    424       1.1  maxv 
    425       1.1  maxv /* -------------------------------------------------------------------------- */
    426       1.1  maxv 
    427       1.1  maxv void *
    428       1.1  maxv kasan_memcpy(void *dst, const void *src, size_t len)
    429       1.1  maxv {
    430       1.1  maxv 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
    431       1.1  maxv 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
    432       1.1  maxv 	return __builtin_memcpy(dst, src, len);
    433       1.1  maxv }
    434       1.1  maxv 
    435       1.1  maxv int
    436       1.1  maxv kasan_memcmp(const void *b1, const void *b2, size_t len)
    437       1.1  maxv {
    438       1.1  maxv 	kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
    439       1.1  maxv 	kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
    440       1.1  maxv 	return __builtin_memcmp(b1, b2, len);
    441       1.1  maxv }
    442       1.1  maxv 
    443       1.1  maxv void *
    444       1.1  maxv kasan_memset(void *b, int c, size_t len)
    445       1.1  maxv {
    446       1.1  maxv 	kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
    447       1.1  maxv 	return __builtin_memset(b, c, len);
    448       1.1  maxv }
    449       1.1  maxv 
    450      1.12  maxv void *
    451      1.12  maxv kasan_memmove(void *dst, const void *src, size_t len)
    452      1.12  maxv {
    453      1.12  maxv 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
    454      1.12  maxv 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
    455      1.12  maxv 	return __builtin_memmove(dst, src, len);
    456      1.12  maxv }
    457      1.12  maxv 
    458       1.1  maxv char *
    459       1.1  maxv kasan_strcpy(char *dst, const char *src)
    460       1.1  maxv {
    461       1.1  maxv 	char *save = dst;
    462       1.1  maxv 
    463       1.1  maxv 	while (1) {
    464       1.1  maxv 		kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
    465       1.1  maxv 		kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
    466       1.1  maxv 		*dst = *src;
    467       1.1  maxv 		if (*src == '\0')
    468       1.1  maxv 			break;
    469       1.1  maxv 		src++, dst++;
    470       1.1  maxv 	}
    471       1.1  maxv 
    472       1.1  maxv 	return save;
    473       1.1  maxv }
    474       1.1  maxv 
    475       1.1  maxv int
    476       1.1  maxv kasan_strcmp(const char *s1, const char *s2)
    477       1.1  maxv {
    478       1.1  maxv 	while (1) {
    479       1.1  maxv 		kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
    480       1.1  maxv 		kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
    481       1.1  maxv 		if (*s1 != *s2)
    482       1.1  maxv 			break;
    483       1.1  maxv 		if (*s1 == '\0')
    484       1.1  maxv 			return 0;
    485       1.1  maxv 		s1++, s2++;
    486       1.1  maxv 	}
    487       1.1  maxv 
    488       1.1  maxv 	return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
    489       1.1  maxv }
    490       1.1  maxv 
    491       1.1  maxv size_t
    492       1.1  maxv kasan_strlen(const char *str)
    493       1.1  maxv {
    494       1.1  maxv 	const char *s;
    495       1.1  maxv 
    496       1.1  maxv 	s = str;
    497       1.1  maxv 	while (1) {
    498       1.1  maxv 		kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
    499       1.1  maxv 		if (*s == '\0')
    500       1.1  maxv 			break;
    501       1.1  maxv 		s++;
    502       1.1  maxv 	}
    503       1.1  maxv 
    504       1.1  maxv 	return (s - str);
    505       1.1  maxv }
    506       1.1  maxv 
    507       1.8  maxv #undef kcopy
    508       1.8  maxv #undef copystr
    509       1.7  maxv #undef copyinstr
    510       1.7  maxv #undef copyoutstr
    511       1.7  maxv #undef copyin
    512       1.7  maxv 
    513       1.8  maxv int	kasan_kcopy(const void *, void *, size_t);
    514       1.8  maxv int	kasan_copystr(const void *, void *, size_t, size_t *);
    515       1.7  maxv int	kasan_copyinstr(const void *, void *, size_t, size_t *);
    516       1.7  maxv int	kasan_copyoutstr(const void *, void *, size_t, size_t *);
    517       1.7  maxv int	kasan_copyin(const void *, void *, size_t);
    518       1.8  maxv int	kcopy(const void *, void *, size_t);
    519       1.8  maxv int	copystr(const void *, void *, size_t, size_t *);
    520       1.7  maxv int	copyinstr(const void *, void *, size_t, size_t *);
    521       1.7  maxv int	copyoutstr(const void *, void *, size_t, size_t *);
    522       1.7  maxv int	copyin(const void *, void *, size_t);
    523       1.7  maxv 
    524       1.7  maxv int
    525       1.8  maxv kasan_kcopy(const void *src, void *dst, size_t len)
    526       1.8  maxv {
    527       1.8  maxv 	kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
    528       1.8  maxv 	kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
    529       1.8  maxv 	return kcopy(src, dst, len);
    530       1.8  maxv }
    531       1.8  maxv 
    532       1.8  maxv int
    533       1.8  maxv kasan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
    534       1.8  maxv {
    535       1.8  maxv 	kasan_shadow_check((unsigned long)kdaddr, len, true, __RET_ADDR);
    536       1.8  maxv 	return copystr(kfaddr, kdaddr, len, done);
    537       1.8  maxv }
    538       1.8  maxv 
    539       1.8  maxv int
    540       1.7  maxv kasan_copyin(const void *uaddr, void *kaddr, size_t len)
    541       1.7  maxv {
    542       1.7  maxv 	kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
    543       1.7  maxv 	return copyin(uaddr, kaddr, len);
    544       1.7  maxv }
    545       1.7  maxv 
    546       1.7  maxv int
    547       1.7  maxv kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
    548       1.7  maxv {
    549       1.7  maxv 	kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
    550       1.7  maxv 	return copyinstr(uaddr, kaddr, len, done);
    551       1.7  maxv }
    552       1.7  maxv 
    553       1.7  maxv int
    554       1.7  maxv kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
    555       1.7  maxv {
    556       1.7  maxv 	kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
    557       1.7  maxv 	return copyoutstr(kaddr, uaddr, len, done);
    558       1.7  maxv }
    559       1.7  maxv 
    560       1.1  maxv /* -------------------------------------------------------------------------- */
    561       1.1  maxv 
    562      1.16  maxv #undef _ucas_32
    563      1.16  maxv #undef _ucas_32_mp
    564      1.16  maxv #undef _ucas_64
    565      1.16  maxv #undef _ucas_64_mp
    566      1.16  maxv #undef _ufetch_8
    567      1.16  maxv #undef _ufetch_16
    568      1.16  maxv #undef _ufetch_32
    569      1.16  maxv #undef _ufetch_64
    570      1.16  maxv 
    571      1.16  maxv int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    572      1.16  maxv int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    573      1.16  maxv int
    574      1.16  maxv kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
    575      1.16  maxv     uint32_t *ret)
    576      1.16  maxv {
    577      1.16  maxv 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    578      1.16  maxv 	    __RET_ADDR);
    579      1.16  maxv 	return _ucas_32(uaddr, old, new, ret);
    580      1.16  maxv }
    581      1.16  maxv 
    582      1.16  maxv #ifdef __HAVE_UCAS_MP
    583      1.16  maxv int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    584      1.16  maxv int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
    585      1.16  maxv int
    586      1.16  maxv kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
    587      1.16  maxv     uint32_t *ret)
    588      1.16  maxv {
    589      1.16  maxv 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    590      1.16  maxv 	    __RET_ADDR);
    591      1.16  maxv 	return _ucas_32_mp(uaddr, old, new, ret);
    592      1.16  maxv }
    593      1.16  maxv #endif
    594      1.16  maxv 
    595      1.16  maxv #ifdef _LP64
    596      1.16  maxv int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    597      1.16  maxv int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    598      1.16  maxv int
    599      1.16  maxv kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
    600      1.16  maxv     uint64_t *ret)
    601      1.16  maxv {
    602      1.16  maxv 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    603      1.16  maxv 	    __RET_ADDR);
    604      1.16  maxv 	return _ucas_64(uaddr, old, new, ret);
    605      1.16  maxv }
    606      1.16  maxv 
    607      1.16  maxv #ifdef __HAVE_UCAS_MP
    608      1.16  maxv int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    609      1.16  maxv int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
    610      1.16  maxv int
    611      1.16  maxv kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
    612      1.16  maxv     uint64_t *ret)
    613      1.16  maxv {
    614      1.16  maxv 	kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
    615      1.16  maxv 	    __RET_ADDR);
    616      1.16  maxv 	return _ucas_64_mp(uaddr, old, new, ret);
    617      1.16  maxv }
    618      1.16  maxv #endif
    619      1.16  maxv #endif
    620      1.16  maxv 
    621      1.16  maxv int _ufetch_8(const uint8_t *, uint8_t *);
    622      1.16  maxv int kasan__ufetch_8(const uint8_t *, uint8_t *);
    623      1.16  maxv int
    624      1.16  maxv kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
    625      1.16  maxv {
    626      1.16  maxv 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    627      1.16  maxv 	    __RET_ADDR);
    628      1.16  maxv 	return _ufetch_8(uaddr, valp);
    629      1.16  maxv }
    630      1.16  maxv 
    631      1.16  maxv int _ufetch_16(const uint16_t *, uint16_t *);
    632      1.16  maxv int kasan__ufetch_16(const uint16_t *, uint16_t *);
    633      1.16  maxv int
    634      1.16  maxv kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
    635      1.16  maxv {
    636      1.16  maxv 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    637      1.16  maxv 	    __RET_ADDR);
    638      1.16  maxv 	return _ufetch_16(uaddr, valp);
    639      1.16  maxv }
    640      1.16  maxv 
    641      1.16  maxv int _ufetch_32(const uint32_t *, uint32_t *);
    642      1.16  maxv int kasan__ufetch_32(const uint32_t *, uint32_t *);
    643      1.16  maxv int
    644      1.16  maxv kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
    645      1.16  maxv {
    646      1.16  maxv 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    647      1.16  maxv 	    __RET_ADDR);
    648      1.16  maxv 	return _ufetch_32(uaddr, valp);
    649      1.16  maxv }
    650      1.16  maxv 
    651      1.16  maxv #ifdef _LP64
    652      1.16  maxv int _ufetch_64(const uint64_t *, uint64_t *);
    653      1.16  maxv int kasan__ufetch_64(const uint64_t *, uint64_t *);
    654      1.16  maxv int
    655      1.16  maxv kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
    656      1.16  maxv {
    657      1.16  maxv 	kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
    658      1.16  maxv 	    __RET_ADDR);
    659      1.16  maxv 	return _ufetch_64(uaddr, valp);
    660      1.16  maxv }
    661      1.16  maxv #endif
    662      1.16  maxv 
    663      1.16  maxv /* -------------------------------------------------------------------------- */
    664      1.16  maxv 
    665      1.11  maxv #undef atomic_add_32
    666      1.11  maxv #undef atomic_add_int
    667      1.11  maxv #undef atomic_add_long
    668      1.11  maxv #undef atomic_add_ptr
    669      1.11  maxv #undef atomic_add_64
    670      1.11  maxv #undef atomic_add_32_nv
    671      1.11  maxv #undef atomic_add_int_nv
    672      1.11  maxv #undef atomic_add_long_nv
    673      1.11  maxv #undef atomic_add_ptr_nv
    674      1.11  maxv #undef atomic_add_64_nv
    675      1.11  maxv #undef atomic_and_32
    676      1.11  maxv #undef atomic_and_uint
    677      1.11  maxv #undef atomic_and_ulong
    678      1.11  maxv #undef atomic_and_64
    679      1.11  maxv #undef atomic_and_32_nv
    680      1.11  maxv #undef atomic_and_uint_nv
    681      1.11  maxv #undef atomic_and_ulong_nv
    682      1.11  maxv #undef atomic_and_64_nv
    683      1.11  maxv #undef atomic_or_32
    684      1.11  maxv #undef atomic_or_uint
    685      1.11  maxv #undef atomic_or_ulong
    686      1.11  maxv #undef atomic_or_64
    687      1.11  maxv #undef atomic_or_32_nv
    688      1.11  maxv #undef atomic_or_uint_nv
    689      1.11  maxv #undef atomic_or_ulong_nv
    690      1.11  maxv #undef atomic_or_64_nv
    691      1.11  maxv #undef atomic_cas_32
    692      1.11  maxv #undef atomic_cas_uint
    693      1.11  maxv #undef atomic_cas_ulong
    694      1.11  maxv #undef atomic_cas_ptr
    695      1.11  maxv #undef atomic_cas_64
    696      1.11  maxv #undef atomic_cas_32_ni
    697      1.11  maxv #undef atomic_cas_uint_ni
    698      1.11  maxv #undef atomic_cas_ulong_ni
    699      1.11  maxv #undef atomic_cas_ptr_ni
    700      1.11  maxv #undef atomic_cas_64_ni
    701      1.11  maxv #undef atomic_swap_32
    702      1.11  maxv #undef atomic_swap_uint
    703      1.11  maxv #undef atomic_swap_ulong
    704      1.11  maxv #undef atomic_swap_ptr
    705      1.11  maxv #undef atomic_swap_64
    706      1.11  maxv #undef atomic_dec_32
    707      1.11  maxv #undef atomic_dec_uint
    708      1.11  maxv #undef atomic_dec_ulong
    709      1.11  maxv #undef atomic_dec_ptr
    710      1.11  maxv #undef atomic_dec_64
    711      1.11  maxv #undef atomic_dec_32_nv
    712      1.11  maxv #undef atomic_dec_uint_nv
    713      1.11  maxv #undef atomic_dec_ulong_nv
    714      1.11  maxv #undef atomic_dec_ptr_nv
    715      1.11  maxv #undef atomic_dec_64_nv
    716      1.11  maxv #undef atomic_inc_32
    717      1.11  maxv #undef atomic_inc_uint
    718      1.11  maxv #undef atomic_inc_ulong
    719      1.11  maxv #undef atomic_inc_ptr
    720      1.11  maxv #undef atomic_inc_64
    721      1.11  maxv #undef atomic_inc_32_nv
    722      1.11  maxv #undef atomic_inc_uint_nv
    723      1.11  maxv #undef atomic_inc_ulong_nv
    724      1.11  maxv #undef atomic_inc_ptr_nv
    725      1.11  maxv #undef atomic_inc_64_nv
    726      1.11  maxv 
    727      1.11  maxv #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
    728      1.11  maxv 	void atomic_add_##name(volatile targ1 *, targ2); \
    729      1.11  maxv 	void kasan_atomic_add_##name(volatile targ1 *, targ2); \
    730      1.11  maxv 	void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
    731      1.11  maxv 	{ \
    732      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    733      1.11  maxv 		    __RET_ADDR); \
    734      1.11  maxv 		atomic_add_##name(ptr, val); \
    735      1.11  maxv 	} \
    736      1.11  maxv 	tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
    737      1.11  maxv 	tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
    738      1.11  maxv 	tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
    739      1.11  maxv 	{ \
    740      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    741      1.11  maxv 		    __RET_ADDR); \
    742      1.11  maxv 		return atomic_add_##name##_nv(ptr, val); \
    743      1.11  maxv 	}
    744      1.11  maxv 
    745      1.11  maxv #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
    746      1.11  maxv 	void atomic_and_##name(volatile targ1 *, targ2); \
    747      1.11  maxv 	void kasan_atomic_and_##name(volatile targ1 *, targ2); \
    748      1.11  maxv 	void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
    749      1.11  maxv 	{ \
    750      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    751      1.11  maxv 		    __RET_ADDR); \
    752      1.11  maxv 		atomic_and_##name(ptr, val); \
    753      1.11  maxv 	} \
    754      1.11  maxv 	tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
    755      1.11  maxv 	tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
    756      1.11  maxv 	tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
    757      1.11  maxv 	{ \
    758      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    759      1.11  maxv 		    __RET_ADDR); \
    760      1.11  maxv 		return atomic_and_##name##_nv(ptr, val); \
    761      1.11  maxv 	}
    762      1.11  maxv 
    763      1.11  maxv #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
    764      1.11  maxv 	void atomic_or_##name(volatile targ1 *, targ2); \
    765      1.11  maxv 	void kasan_atomic_or_##name(volatile targ1 *, targ2); \
    766      1.11  maxv 	void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
    767      1.11  maxv 	{ \
    768      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    769      1.11  maxv 		    __RET_ADDR); \
    770      1.11  maxv 		atomic_or_##name(ptr, val); \
    771      1.11  maxv 	} \
    772      1.11  maxv 	tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
    773      1.11  maxv 	tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
    774      1.11  maxv 	tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
    775      1.11  maxv 	{ \
    776      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    777      1.11  maxv 		    __RET_ADDR); \
    778      1.11  maxv 		return atomic_or_##name##_nv(ptr, val); \
    779      1.11  maxv 	}
    780      1.11  maxv 
    781      1.11  maxv #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
    782      1.11  maxv 	tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
    783      1.11  maxv 	tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
    784      1.11  maxv 	tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
    785      1.11  maxv 	{ \
    786      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    787      1.11  maxv 		    __RET_ADDR); \
    788      1.11  maxv 		return atomic_cas_##name(ptr, exp, new); \
    789      1.11  maxv 	} \
    790      1.11  maxv 	tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
    791      1.11  maxv 	tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
    792      1.11  maxv 	tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
    793      1.11  maxv 	{ \
    794      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    795      1.11  maxv 		    __RET_ADDR); \
    796      1.11  maxv 		return atomic_cas_##name##_ni(ptr, exp, new); \
    797      1.11  maxv 	}
    798      1.11  maxv 
    799      1.11  maxv #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
    800      1.11  maxv 	tret atomic_swap_##name(volatile targ1 *, targ2); \
    801      1.11  maxv 	tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \
    802      1.11  maxv 	tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
    803      1.11  maxv 	{ \
    804      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    805      1.11  maxv 		    __RET_ADDR); \
    806      1.11  maxv 		return atomic_swap_##name(ptr, val); \
    807      1.11  maxv 	}
    808      1.11  maxv 
    809      1.11  maxv #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
    810      1.11  maxv 	void atomic_dec_##name(volatile targ1 *); \
    811      1.11  maxv 	void kasan_atomic_dec_##name(volatile targ1 *); \
    812      1.11  maxv 	void kasan_atomic_dec_##name(volatile targ1 *ptr) \
    813      1.11  maxv 	{ \
    814      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    815      1.11  maxv 		    __RET_ADDR); \
    816      1.11  maxv 		atomic_dec_##name(ptr); \
    817      1.11  maxv 	} \
    818      1.11  maxv 	tret atomic_dec_##name##_nv(volatile targ1 *); \
    819      1.11  maxv 	tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \
    820      1.11  maxv 	tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
    821      1.11  maxv 	{ \
    822      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    823      1.11  maxv 		    __RET_ADDR); \
    824      1.11  maxv 		return atomic_dec_##name##_nv(ptr); \
    825      1.11  maxv 	}
    826      1.11  maxv 
    827      1.11  maxv #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \
    828      1.11  maxv 	void atomic_inc_##name(volatile targ1 *); \
    829      1.11  maxv 	void kasan_atomic_inc_##name(volatile targ1 *); \
    830      1.11  maxv 	void kasan_atomic_inc_##name(volatile targ1 *ptr) \
    831      1.11  maxv 	{ \
    832      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    833      1.11  maxv 		    __RET_ADDR); \
    834      1.11  maxv 		atomic_inc_##name(ptr); \
    835      1.11  maxv 	} \
    836      1.11  maxv 	tret atomic_inc_##name##_nv(volatile targ1 *); \
    837      1.11  maxv 	tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \
    838      1.11  maxv 	tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
    839      1.11  maxv 	{ \
    840      1.11  maxv 		kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
    841      1.11  maxv 		    __RET_ADDR); \
    842      1.11  maxv 		return atomic_inc_##name##_nv(ptr); \
    843      1.11  maxv 	}
    844      1.11  maxv 
    845      1.11  maxv ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
    846      1.11  maxv ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
    847      1.11  maxv ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
    848      1.11  maxv ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
    849      1.11  maxv ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
    850      1.11  maxv 
    851      1.11  maxv ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
    852      1.11  maxv ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
    853      1.11  maxv ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
    854      1.11  maxv ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
    855      1.11  maxv 
    856      1.11  maxv ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
    857      1.11  maxv ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
    858      1.11  maxv ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
    859      1.11  maxv ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
    860      1.11  maxv 
    861      1.11  maxv ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
    862      1.11  maxv ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
    863      1.11  maxv ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
    864      1.11  maxv ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
    865      1.11  maxv ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
    866      1.11  maxv 
    867      1.11  maxv ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
    868      1.11  maxv ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
    869      1.11  maxv ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
    870      1.11  maxv ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
    871      1.11  maxv ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
    872      1.11  maxv 
    873      1.11  maxv ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
    874      1.11  maxv ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
    875      1.11  maxv ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
    876      1.11  maxv ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
    877      1.11  maxv ASAN_ATOMIC_FUNC_DEC(ptr, void *, void);
    878      1.11  maxv 
    879      1.11  maxv ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
    880      1.11  maxv ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
    881      1.11  maxv ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
    882      1.11  maxv ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
    883      1.11  maxv ASAN_ATOMIC_FUNC_INC(ptr, void *, void);
    884      1.11  maxv 
    885      1.11  maxv /* -------------------------------------------------------------------------- */
    886      1.11  maxv 
    887      1.14  maxv #ifdef __HAVE_KASAN_INSTR_BUS
    888      1.14  maxv 
    889      1.13  maxv #include <sys/bus.h>
    890      1.13  maxv 
    891      1.13  maxv #undef bus_space_read_multi_1
    892      1.13  maxv #undef bus_space_read_multi_2
    893      1.13  maxv #undef bus_space_read_multi_4
    894      1.13  maxv #undef bus_space_read_multi_8
    895      1.13  maxv #undef bus_space_read_multi_stream_1
    896      1.13  maxv #undef bus_space_read_multi_stream_2
    897      1.13  maxv #undef bus_space_read_multi_stream_4
    898      1.13  maxv #undef bus_space_read_multi_stream_8
    899      1.13  maxv #undef bus_space_read_region_1
    900      1.13  maxv #undef bus_space_read_region_2
    901      1.13  maxv #undef bus_space_read_region_4
    902      1.13  maxv #undef bus_space_read_region_8
    903      1.13  maxv #undef bus_space_read_region_stream_1
    904      1.13  maxv #undef bus_space_read_region_stream_2
    905      1.13  maxv #undef bus_space_read_region_stream_4
    906      1.13  maxv #undef bus_space_read_region_stream_8
    907      1.13  maxv #undef bus_space_write_multi_1
    908      1.13  maxv #undef bus_space_write_multi_2
    909      1.13  maxv #undef bus_space_write_multi_4
    910      1.13  maxv #undef bus_space_write_multi_8
    911      1.13  maxv #undef bus_space_write_multi_stream_1
    912      1.13  maxv #undef bus_space_write_multi_stream_2
    913      1.13  maxv #undef bus_space_write_multi_stream_4
    914      1.13  maxv #undef bus_space_write_multi_stream_8
    915      1.13  maxv #undef bus_space_write_region_1
    916      1.13  maxv #undef bus_space_write_region_2
    917      1.13  maxv #undef bus_space_write_region_4
    918      1.13  maxv #undef bus_space_write_region_8
    919      1.13  maxv #undef bus_space_write_region_stream_1
    920      1.13  maxv #undef bus_space_write_region_stream_2
    921      1.13  maxv #undef bus_space_write_region_stream_4
    922      1.13  maxv #undef bus_space_write_region_stream_8
    923      1.13  maxv 
    924      1.13  maxv #define ASAN_BUS_READ_FUNC(bytes, bits) \
    925      1.13  maxv 	void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t,	\
    926      1.13  maxv 	    bus_size_t, uint##bits##_t *, bus_size_t);				\
    927      1.13  maxv 	void kasan_bus_space_read_multi_##bytes(bus_space_tag_t,		\
    928      1.13  maxv 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    929      1.13  maxv 	void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag,		\
    930      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    931      1.13  maxv 	    bus_size_t count)							\
    932      1.13  maxv 	{									\
    933      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
    934      1.13  maxv 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    935      1.13  maxv 		bus_space_read_multi_##bytes(tag, hnd, size, buf, count);	\
    936      1.13  maxv 	}									\
    937      1.13  maxv 	void bus_space_read_multi_stream_##bytes(bus_space_tag_t,		\
    938      1.13  maxv 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    939      1.13  maxv 	void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t,		\
    940      1.13  maxv 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    941      1.13  maxv 	void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag,	\
    942      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    943      1.13  maxv 	    bus_size_t count)							\
    944      1.13  maxv 	{									\
    945      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
    946      1.13  maxv 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    947      1.13  maxv 		bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
    948      1.13  maxv 	}									\
    949      1.13  maxv 	void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t,	\
    950      1.13  maxv 	    bus_size_t, uint##bits##_t *, bus_size_t);				\
    951      1.13  maxv 	void kasan_bus_space_read_region_##bytes(bus_space_tag_t,		\
    952      1.13  maxv 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    953      1.13  maxv 	void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag,		\
    954      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    955      1.13  maxv 	    bus_size_t count)							\
    956      1.13  maxv 	{									\
    957      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
    958      1.13  maxv 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    959      1.13  maxv 		bus_space_read_region_##bytes(tag, hnd, size, buf, count);	\
    960      1.13  maxv 	}									\
    961      1.13  maxv 	void bus_space_read_region_stream_##bytes(bus_space_tag_t,		\
    962      1.13  maxv 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    963      1.13  maxv 	void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t,	\
    964      1.13  maxv 	    bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t);	\
    965      1.13  maxv 	void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag,	\
    966      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf,	\
    967      1.13  maxv 	    bus_size_t count)							\
    968      1.13  maxv 	{									\
    969      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
    970      1.13  maxv 		    sizeof(uint##bits##_t) * count, false, __RET_ADDR);		\
    971      1.13  maxv 		bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
    972      1.13  maxv 	}
    973      1.13  maxv 
    974      1.13  maxv #define ASAN_BUS_WRITE_FUNC(bytes, bits) \
    975      1.13  maxv 	void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t,	\
    976      1.13  maxv 	    bus_size_t, const uint##bits##_t *, bus_size_t);			\
    977      1.13  maxv 	void kasan_bus_space_write_multi_##bytes(bus_space_tag_t,		\
    978      1.13  maxv 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
    979      1.13  maxv 	void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag,		\
    980      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
    981      1.13  maxv 	    bus_size_t count)							\
    982      1.13  maxv 	{									\
    983      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
    984      1.13  maxv 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
    985      1.13  maxv 		bus_space_write_multi_##bytes(tag, hnd, size, buf, count);	\
    986      1.13  maxv 	}									\
    987      1.13  maxv 	void bus_space_write_multi_stream_##bytes(bus_space_tag_t,		\
    988      1.13  maxv 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
    989      1.13  maxv 	void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t,	\
    990      1.13  maxv 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
    991      1.13  maxv 	void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag,	\
    992      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
    993      1.13  maxv 	    bus_size_t count)							\
    994      1.13  maxv 	{									\
    995      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
    996      1.13  maxv 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
    997      1.13  maxv 		bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
    998      1.13  maxv 	}									\
    999      1.13  maxv 	void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
   1000      1.13  maxv 	    bus_size_t, const uint##bits##_t *, bus_size_t);			\
   1001      1.13  maxv 	void kasan_bus_space_write_region_##bytes(bus_space_tag_t,		\
   1002      1.13  maxv 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1003      1.13  maxv 	void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag,		\
   1004      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
   1005      1.13  maxv 	    bus_size_t count)							\
   1006      1.13  maxv 	{									\
   1007      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
   1008      1.13  maxv 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
   1009      1.13  maxv 		bus_space_write_region_##bytes(tag, hnd, size, buf, count);	\
   1010      1.13  maxv 	}									\
   1011      1.13  maxv 	void bus_space_write_region_stream_##bytes(bus_space_tag_t,		\
   1012      1.13  maxv 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1013      1.13  maxv 	void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t,	\
   1014      1.13  maxv 	    bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
   1015      1.13  maxv 	void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag,	\
   1016      1.13  maxv 	    bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf,	\
   1017      1.13  maxv 	    bus_size_t count)							\
   1018      1.13  maxv 	{									\
   1019      1.13  maxv 		kasan_shadow_check((uintptr_t)buf,				\
   1020      1.13  maxv 		    sizeof(uint##bits##_t) * count, true, __RET_ADDR);		\
   1021      1.13  maxv 		bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
   1022      1.13  maxv 	}
   1023      1.13  maxv 
   1024      1.13  maxv ASAN_BUS_READ_FUNC(1, 8)
   1025      1.13  maxv ASAN_BUS_READ_FUNC(2, 16)
   1026      1.13  maxv ASAN_BUS_READ_FUNC(4, 32)
   1027      1.13  maxv ASAN_BUS_READ_FUNC(8, 64)
   1028      1.13  maxv 
   1029      1.13  maxv ASAN_BUS_WRITE_FUNC(1, 8)
   1030      1.13  maxv ASAN_BUS_WRITE_FUNC(2, 16)
   1031      1.13  maxv ASAN_BUS_WRITE_FUNC(4, 32)
   1032      1.13  maxv ASAN_BUS_WRITE_FUNC(8, 64)
   1033      1.13  maxv 
   1034      1.14  maxv #endif /* __HAVE_KASAN_INSTR_BUS */
   1035      1.14  maxv 
   1036      1.13  maxv /* -------------------------------------------------------------------------- */
   1037      1.13  maxv 
   1038      1.15  maxv #ifdef __HAVE_KASAN_INSTR_DMA
   1039      1.15  maxv 
   1040      1.15  maxv #include <sys/mbuf.h>
   1041      1.15  maxv 
   1042      1.15  maxv static void
   1043      1.15  maxv kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
   1044      1.15  maxv     bool write, uintptr_t pc)
   1045      1.15  maxv {
   1046      1.15  maxv 	kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc);
   1047      1.15  maxv }
   1048      1.15  maxv 
   1049      1.15  maxv static void
   1050      1.15  maxv kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
   1051      1.15  maxv     bool write, uintptr_t pc)
   1052      1.15  maxv {
   1053      1.15  maxv 	bus_addr_t minlen;
   1054      1.15  maxv 
   1055      1.15  maxv 	for (; m != NULL && len != 0; m = m->m_next) {
   1056      1.15  maxv 		kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc);
   1057      1.15  maxv 
   1058      1.15  maxv 		if (offset >= m->m_len) {
   1059      1.15  maxv 			offset -= m->m_len;
   1060      1.15  maxv 			continue;
   1061      1.15  maxv 		}
   1062      1.15  maxv 
   1063      1.15  maxv 		minlen = MIN(len, m->m_len - offset);
   1064      1.15  maxv 		kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset),
   1065      1.15  maxv 		    minlen, write, pc);
   1066      1.15  maxv 
   1067      1.15  maxv 		offset = 0;
   1068      1.15  maxv 		len -= minlen;
   1069      1.15  maxv 	}
   1070      1.15  maxv }
   1071      1.15  maxv 
   1072      1.15  maxv static void
   1073      1.15  maxv kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
   1074      1.15  maxv     bool write, uintptr_t pc)
   1075      1.15  maxv {
   1076      1.15  maxv 	bus_size_t minlen, resid;
   1077      1.15  maxv 	struct iovec *iov;
   1078      1.15  maxv 	int i;
   1079      1.15  maxv 
   1080      1.15  maxv 	kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc);
   1081      1.15  maxv 
   1082  1.16.2.1    ad 	if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
   1083  1.16.2.1    ad 		return;
   1084  1.16.2.1    ad 
   1085      1.15  maxv 	resid = uio->uio_resid;
   1086      1.15  maxv 	iov = uio->uio_iov;
   1087      1.15  maxv 
   1088      1.15  maxv 	for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
   1089      1.15  maxv 		kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]),
   1090      1.15  maxv 		    false, pc);
   1091      1.15  maxv 		minlen = MIN(resid, iov[i].iov_len);
   1092      1.15  maxv 		kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen,
   1093      1.15  maxv 		    write, pc);
   1094      1.15  maxv 		resid -= minlen;
   1095      1.15  maxv 	}
   1096      1.15  maxv }
   1097      1.15  maxv 
   1098      1.15  maxv void
   1099      1.15  maxv kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
   1100      1.15  maxv {
   1101      1.15  maxv 	bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0;
   1102      1.15  maxv 
   1103      1.15  maxv 	switch (map->dm_buftype) {
   1104      1.15  maxv 	case KASAN_DMA_LINEAR:
   1105      1.15  maxv 		kasan_dma_sync_linear(map->dm_buf, offset, len, write,
   1106      1.15  maxv 		    __RET_ADDR);
   1107      1.15  maxv 		break;
   1108      1.15  maxv 	case KASAN_DMA_MBUF:
   1109      1.15  maxv 		kasan_dma_sync_mbuf(map->dm_buf, offset, len, write,
   1110      1.15  maxv 		    __RET_ADDR);
   1111      1.15  maxv 		break;
   1112      1.15  maxv 	case KASAN_DMA_UIO:
   1113      1.15  maxv 		kasan_dma_sync_uio(map->dm_buf, offset, len, write,
   1114      1.15  maxv 		    __RET_ADDR);
   1115      1.15  maxv 		break;
   1116      1.15  maxv 	case KASAN_DMA_RAW:
   1117      1.15  maxv 		break;
   1118      1.15  maxv 	default:
   1119      1.15  maxv 		panic("%s: impossible", __func__);
   1120      1.15  maxv 	}
   1121      1.15  maxv }
   1122      1.15  maxv 
   1123      1.15  maxv void
   1124      1.15  maxv kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
   1125      1.15  maxv {
   1126      1.15  maxv 	map->dm_buf = buf;
   1127      1.15  maxv 	map->dm_buflen = buflen;
   1128      1.15  maxv 	map->dm_buftype = type;
   1129      1.15  maxv }
   1130      1.15  maxv 
   1131      1.15  maxv #endif /* __HAVE_KASAN_INSTR_DMA */
   1132      1.15  maxv 
   1133      1.15  maxv /* -------------------------------------------------------------------------- */
   1134      1.15  maxv 
   1135       1.1  maxv void __asan_register_globals(struct __asan_global *, size_t);
   1136       1.1  maxv void __asan_unregister_globals(struct __asan_global *, size_t);
   1137       1.1  maxv 
   1138       1.1  maxv void
   1139       1.1  maxv __asan_register_globals(struct __asan_global *globals, size_t n)
   1140       1.1  maxv {
   1141       1.1  maxv 	size_t i;
   1142       1.1  maxv 
   1143       1.1  maxv 	for (i = 0; i < n; i++) {
   1144       1.2  maxv 		kasan_mark(globals[i].beg, globals[i].size,
   1145       1.6  maxv 		    globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
   1146       1.1  maxv 	}
   1147       1.1  maxv }
   1148       1.1  maxv 
   1149       1.1  maxv void
   1150       1.1  maxv __asan_unregister_globals(struct __asan_global *globals, size_t n)
   1151       1.1  maxv {
   1152       1.1  maxv 	/* never called */
   1153       1.1  maxv }
   1154       1.1  maxv 
   1155       1.1  maxv #define ASAN_LOAD_STORE(size)					\
   1156       1.1  maxv 	void __asan_load##size(unsigned long);			\
   1157       1.1  maxv 	void __asan_load##size(unsigned long addr)		\
   1158       1.1  maxv 	{							\
   1159       1.1  maxv 		kasan_shadow_check(addr, size, false, __RET_ADDR);\
   1160       1.1  maxv 	} 							\
   1161       1.1  maxv 	void __asan_load##size##_noabort(unsigned long);	\
   1162       1.1  maxv 	void __asan_load##size##_noabort(unsigned long addr)	\
   1163       1.1  maxv 	{							\
   1164       1.1  maxv 		kasan_shadow_check(addr, size, false, __RET_ADDR);\
   1165       1.1  maxv 	}							\
   1166       1.1  maxv 	void __asan_store##size(unsigned long);			\
   1167       1.1  maxv 	void __asan_store##size(unsigned long addr)		\
   1168       1.1  maxv 	{							\
   1169       1.1  maxv 		kasan_shadow_check(addr, size, true, __RET_ADDR);\
   1170       1.1  maxv 	}							\
   1171       1.1  maxv 	void __asan_store##size##_noabort(unsigned long);	\
   1172       1.1  maxv 	void __asan_store##size##_noabort(unsigned long addr)	\
   1173       1.1  maxv 	{							\
   1174       1.1  maxv 		kasan_shadow_check(addr, size, true, __RET_ADDR);\
   1175       1.1  maxv 	}
   1176       1.1  maxv 
   1177       1.1  maxv ASAN_LOAD_STORE(1);
   1178       1.1  maxv ASAN_LOAD_STORE(2);
   1179       1.1  maxv ASAN_LOAD_STORE(4);
   1180       1.1  maxv ASAN_LOAD_STORE(8);
   1181       1.1  maxv ASAN_LOAD_STORE(16);
   1182       1.1  maxv 
   1183       1.1  maxv void __asan_loadN(unsigned long, size_t);
   1184       1.1  maxv void __asan_loadN_noabort(unsigned long, size_t);
   1185       1.1  maxv void __asan_storeN(unsigned long, size_t);
   1186       1.1  maxv void __asan_storeN_noabort(unsigned long, size_t);
   1187       1.1  maxv void __asan_handle_no_return(void);
   1188       1.1  maxv 
   1189       1.1  maxv void
   1190       1.1  maxv __asan_loadN(unsigned long addr, size_t size)
   1191       1.1  maxv {
   1192       1.1  maxv 	kasan_shadow_check(addr, size, false, __RET_ADDR);
   1193       1.1  maxv }
   1194       1.1  maxv 
   1195       1.1  maxv void
   1196       1.1  maxv __asan_loadN_noabort(unsigned long addr, size_t size)
   1197       1.1  maxv {
   1198       1.1  maxv 	kasan_shadow_check(addr, size, false, __RET_ADDR);
   1199       1.1  maxv }
   1200       1.1  maxv 
   1201       1.1  maxv void
   1202       1.1  maxv __asan_storeN(unsigned long addr, size_t size)
   1203       1.1  maxv {
   1204       1.1  maxv 	kasan_shadow_check(addr, size, true, __RET_ADDR);
   1205       1.1  maxv }
   1206       1.1  maxv 
   1207       1.1  maxv void
   1208       1.1  maxv __asan_storeN_noabort(unsigned long addr, size_t size)
   1209       1.1  maxv {
   1210       1.1  maxv 	kasan_shadow_check(addr, size, true, __RET_ADDR);
   1211       1.1  maxv }
   1212       1.1  maxv 
   1213       1.1  maxv void
   1214       1.1  maxv __asan_handle_no_return(void)
   1215       1.1  maxv {
   1216       1.1  maxv 	/* nothing */
   1217       1.1  maxv }
   1218       1.1  maxv 
   1219       1.1  maxv #define ASAN_SET_SHADOW(byte) \
   1220       1.1  maxv 	void __asan_set_shadow_##byte(void *, size_t);			\
   1221       1.1  maxv 	void __asan_set_shadow_##byte(void *addr, size_t size)		\
   1222       1.1  maxv 	{								\
   1223       1.1  maxv 		__builtin_memset((void *)addr, 0x##byte, size);		\
   1224       1.1  maxv 	}
   1225       1.1  maxv 
   1226       1.1  maxv ASAN_SET_SHADOW(00);
   1227       1.1  maxv ASAN_SET_SHADOW(f1);
   1228       1.1  maxv ASAN_SET_SHADOW(f2);
   1229       1.1  maxv ASAN_SET_SHADOW(f3);
   1230       1.1  maxv ASAN_SET_SHADOW(f5);
   1231       1.1  maxv ASAN_SET_SHADOW(f8);
   1232       1.4  maxv 
   1233       1.4  maxv void __asan_poison_stack_memory(const void *, size_t);
   1234       1.4  maxv void __asan_unpoison_stack_memory(const void *, size_t);
   1235       1.4  maxv 
   1236  1.16.2.1    ad void
   1237  1.16.2.1    ad __asan_poison_stack_memory(const void *addr, size_t size)
   1238       1.4  maxv {
   1239       1.4  maxv 	size = roundup(size, KASAN_SHADOW_SCALE_SIZE);
   1240       1.4  maxv 	kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
   1241       1.4  maxv }
   1242       1.4  maxv 
   1243  1.16.2.1    ad void
   1244  1.16.2.1    ad __asan_unpoison_stack_memory(const void *addr, size_t size)
   1245       1.4  maxv {
   1246       1.4  maxv 	kasan_shadow_Nbyte_markvalid(addr, size);
   1247       1.4  maxv }
   1248