linux_atomic64.c revision 1.3.2.2
1 1.3.2.2 pgoyette /* $NetBSD: linux_atomic64.c,v 1.3.2.2 2018/09/06 06:56:37 pgoyette Exp $ */ 2 1.3.2.2 pgoyette 3 1.3.2.2 pgoyette /*- 4 1.3.2.2 pgoyette * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 1.3.2.2 pgoyette * All rights reserved. 6 1.3.2.2 pgoyette * 7 1.3.2.2 pgoyette * This code is derived from software contributed to The NetBSD Foundation 8 1.3.2.2 pgoyette * by Taylor R. Campbell. 9 1.3.2.2 pgoyette * 10 1.3.2.2 pgoyette * Redistribution and use in source and binary forms, with or without 11 1.3.2.2 pgoyette * modification, are permitted provided that the following conditions 12 1.3.2.2 pgoyette * are met: 13 1.3.2.2 pgoyette * 1. Redistributions of source code must retain the above copyright 14 1.3.2.2 pgoyette * notice, this list of conditions and the following disclaimer. 15 1.3.2.2 pgoyette * 2. Redistributions in binary form must reproduce the above copyright 16 1.3.2.2 pgoyette * notice, this list of conditions and the following disclaimer in the 17 1.3.2.2 pgoyette * documentation and/or other materials provided with the distribution. 18 1.3.2.2 pgoyette * 19 1.3.2.2 pgoyette * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.3.2.2 pgoyette * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.3.2.2 pgoyette * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.3.2.2 pgoyette * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.3.2.2 pgoyette * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.3.2.2 pgoyette * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.3.2.2 pgoyette * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.3.2.2 pgoyette * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.3.2.2 pgoyette * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.3.2.2 pgoyette * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.3.2.2 pgoyette * POSSIBILITY OF SUCH DAMAGE. 30 1.3.2.2 pgoyette */ 31 1.3.2.2 pgoyette 32 1.3.2.2 pgoyette #include <sys/cdefs.h> 33 1.3.2.2 pgoyette __KERNEL_RCSID(0, "$NetBSD: linux_atomic64.c,v 1.3.2.2 2018/09/06 06:56:37 pgoyette Exp $"); 34 1.3.2.2 pgoyette 35 1.3.2.2 pgoyette #include <sys/param.h> 36 1.3.2.2 pgoyette #include <sys/bitops.h> 37 1.3.2.2 pgoyette #include <sys/lock.h> 38 1.3.2.2 pgoyette 39 1.3.2.2 pgoyette #include <linux/atomic.h> 40 1.3.2.2 pgoyette 41 1.3.2.2 pgoyette #ifdef __HAVE_ATOMIC64_OPS 42 1.3.2.2 pgoyette 43 1.3.2.2 pgoyette int 44 1.3.2.2 pgoyette linux_atomic64_init(void) 45 1.3.2.2 pgoyette { 46 1.3.2.2 pgoyette return 0; 47 1.3.2.2 pgoyette } 48 1.3.2.2 pgoyette 49 1.3.2.2 pgoyette void 50 1.3.2.2 pgoyette linux_atomic64_fini(void) 51 1.3.2.2 pgoyette { 52 1.3.2.2 pgoyette } 53 1.3.2.2 pgoyette 54 1.3.2.2 pgoyette #else 55 1.3.2.2 pgoyette 56 1.3.2.2 pgoyette static struct { 57 1.3.2.2 pgoyette kmutex_t lock; 58 1.3.2.2 pgoyette uint32_t gen; /* for unlocked read */ 59 1.3.2.2 pgoyette char pad[CACHE_LINE_SIZE - 60 1.3.2.2 pgoyette sizeof(kmutex_t) - sizeof(uint32_t)]; 61 1.3.2.2 pgoyette } atomic64_tab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned; 62 1.3.2.2 pgoyette CTASSERT(sizeof(atomic64_tab) == PAGE_SIZE); 63 1.3.2.2 pgoyette CTASSERT(sizeof(atomic64_tab[0]) == CACHE_LINE_SIZE); 64 1.3.2.2 pgoyette 65 1.3.2.2 pgoyette int 66 1.3.2.2 pgoyette linux_atomic64_init(void) 67 1.3.2.2 pgoyette { 68 1.3.2.2 pgoyette size_t i; 69 1.3.2.2 pgoyette 70 1.3.2.2 pgoyette for (i = 0; i < __arraycount(atomic64_tab); i++) { 71 1.3.2.2 pgoyette mutex_init(&atomic64_tab[i].lock, MUTEX_DEFAULT, IPL_HIGH); 72 1.3.2.2 pgoyette atomic64_tab[i].gen = 0; 73 1.3.2.2 pgoyette } 74 1.3.2.2 pgoyette 75 1.3.2.2 pgoyette return 0; 76 1.3.2.2 pgoyette } 77 1.3.2.2 pgoyette 78 1.3.2.2 pgoyette void 79 1.3.2.2 pgoyette linux_atomic64_fini(void) 80 1.3.2.2 pgoyette { 81 1.3.2.2 pgoyette size_t i; 82 1.3.2.2 pgoyette 83 1.3.2.2 pgoyette for (i = 0; i < __arraycount(atomic64_tab); i++) { 84 1.3.2.2 pgoyette KASSERT((atomic64_tab[i].gen & 1) == 0); 85 1.3.2.2 pgoyette mutex_destroy(&atomic64_tab[i].lock); 86 1.3.2.2 pgoyette } 87 1.3.2.2 pgoyette } 88 1.3.2.2 pgoyette 89 1.3.2.2 pgoyette static inline size_t 90 1.3.2.2 pgoyette atomic64_hash(const struct atomic64 *a) 91 1.3.2.2 pgoyette { 92 1.3.2.2 pgoyette 93 1.3.2.2 pgoyette return ((uintptr_t)a >> ilog2(CACHE_LINE_SIZE)) % 94 1.3.2.2 pgoyette __arraycount(atomic64_tab); 95 1.3.2.2 pgoyette } 96 1.3.2.2 pgoyette 97 1.3.2.2 pgoyette static void 98 1.3.2.2 pgoyette atomic64_lock(struct atomic64 *a) 99 1.3.2.2 pgoyette { 100 1.3.2.2 pgoyette size_t i = atomic64_hash(a); 101 1.3.2.2 pgoyette 102 1.3.2.2 pgoyette mutex_spin_enter(&atomic64_tab[i].lock); 103 1.3.2.2 pgoyette KASSERT((atomic64_tab[i].gen & 1) == 0); 104 1.3.2.2 pgoyette atomic64_tab[i].gen |= 1; 105 1.3.2.2 pgoyette membar_producer(); 106 1.3.2.2 pgoyette } 107 1.3.2.2 pgoyette 108 1.3.2.2 pgoyette static void 109 1.3.2.2 pgoyette atomic64_unlock(struct atomic64 *a) 110 1.3.2.2 pgoyette { 111 1.3.2.2 pgoyette size_t i = atomic64_hash(a); 112 1.3.2.2 pgoyette 113 1.3.2.2 pgoyette KASSERT(mutex_owned(&atomic64_tab[i].lock)); 114 1.3.2.2 pgoyette KASSERT((atomic64_tab[i].gen & 1) == 1); 115 1.3.2.2 pgoyette 116 1.3.2.2 pgoyette membar_producer(); 117 1.3.2.2 pgoyette atomic64_tab[i].gen |= 1; /* paranoia */ 118 1.3.2.2 pgoyette atomic64_tab[i].gen++; 119 1.3.2.2 pgoyette mutex_spin_exit(&atomic64_tab[i].lock); 120 1.3.2.2 pgoyette } 121 1.3.2.2 pgoyette 122 1.3.2.2 pgoyette uint64_t 123 1.3.2.2 pgoyette atomic64_read(const struct atomic64 *a) 124 1.3.2.2 pgoyette { 125 1.3.2.2 pgoyette size_t i = atomic64_hash(a); 126 1.3.2.2 pgoyette uint32_t gen; 127 1.3.2.2 pgoyette uint64_t value; 128 1.3.2.2 pgoyette 129 1.3.2.2 pgoyette do { 130 1.3.2.2 pgoyette while (__predict_false((gen = atomic64_tab[i].gen) & 1)) 131 1.3.2.2 pgoyette SPINLOCK_BACKOFF_HOOK; 132 1.3.2.2 pgoyette membar_consumer(); 133 1.3.2.2 pgoyette value = a->a_v; 134 1.3.2.2 pgoyette membar_consumer(); 135 1.3.2.2 pgoyette } while (__predict_false(atomic64_tab[i].gen != gen)); 136 1.3.2.2 pgoyette 137 1.3.2.2 pgoyette return value; 138 1.3.2.2 pgoyette } 139 1.3.2.2 pgoyette 140 1.3.2.2 pgoyette void 141 1.3.2.2 pgoyette atomic64_set(struct atomic64 *a, uint64_t value) 142 1.3.2.2 pgoyette { 143 1.3.2.2 pgoyette 144 1.3.2.2 pgoyette atomic64_lock(a); 145 1.3.2.2 pgoyette a->a_v = value; 146 1.3.2.2 pgoyette atomic64_unlock(a); 147 1.3.2.2 pgoyette } 148 1.3.2.2 pgoyette 149 1.3.2.2 pgoyette void 150 1.3.2.2 pgoyette atomic64_add(int64_t delta, struct atomic64 *a) 151 1.3.2.2 pgoyette { 152 1.3.2.2 pgoyette 153 1.3.2.2 pgoyette atomic64_lock(a); 154 1.3.2.2 pgoyette a->a_v += delta; 155 1.3.2.2 pgoyette atomic64_unlock(a); 156 1.3.2.2 pgoyette } 157 1.3.2.2 pgoyette 158 1.3.2.2 pgoyette void 159 1.3.2.2 pgoyette atomic64_sub(int64_t delta, struct atomic64 *a) 160 1.3.2.2 pgoyette { 161 1.3.2.2 pgoyette 162 1.3.2.2 pgoyette atomic64_lock(a); 163 1.3.2.2 pgoyette a->a_v -= delta; 164 1.3.2.2 pgoyette atomic64_unlock(a); 165 1.3.2.2 pgoyette } 166 1.3.2.2 pgoyette 167 1.3.2.2 pgoyette int64_t 168 1.3.2.2 pgoyette atomic64_add_return(int64_t delta, struct atomic64 *a) 169 1.3.2.2 pgoyette { 170 1.3.2.2 pgoyette int64_t v; 171 1.3.2.2 pgoyette 172 1.3.2.2 pgoyette atomic64_lock(a); 173 1.3.2.2 pgoyette v = (int64_t)(a->a_v += delta); 174 1.3.2.2 pgoyette atomic64_unlock(a); 175 1.3.2.2 pgoyette 176 1.3.2.2 pgoyette return v; 177 1.3.2.2 pgoyette } 178 1.3.2.2 pgoyette 179 1.3.2.2 pgoyette uint64_t 180 1.3.2.2 pgoyette atomic64_xchg(struct atomic64 *a, uint64_t new) 181 1.3.2.2 pgoyette { 182 1.3.2.2 pgoyette uint64_t old; 183 1.3.2.2 pgoyette 184 1.3.2.2 pgoyette atomic64_lock(a); 185 1.3.2.2 pgoyette old = a->a_v; 186 1.3.2.2 pgoyette a->a_v = new; 187 1.3.2.2 pgoyette atomic64_unlock(a); 188 1.3.2.2 pgoyette 189 1.3.2.2 pgoyette return old; 190 1.3.2.2 pgoyette } 191 1.3.2.2 pgoyette 192 1.3.2.2 pgoyette uint64_t 193 1.3.2.2 pgoyette atomic64_cmpxchg(struct atomic64 *a, uint64_t expect, uint64_t new) 194 1.3.2.2 pgoyette { 195 1.3.2.2 pgoyette uint64_t old; 196 1.3.2.2 pgoyette 197 1.3.2.2 pgoyette atomic64_lock(a); 198 1.3.2.2 pgoyette old = a->a_v; 199 1.3.2.2 pgoyette if (old == expect) 200 1.3.2.2 pgoyette a->a_v = new; 201 1.3.2.2 pgoyette atomic64_unlock(a); 202 1.3.2.2 pgoyette 203 1.3.2.2 pgoyette return old; 204 1.3.2.2 pgoyette } 205 1.3.2.2 pgoyette 206 1.3.2.2 pgoyette #endif 207
Indexes created Fri Sep 26 08:10:20 GMT 2025