1 1.1 riastrad /* $NetBSD: linux_atomic64.c,v 1.3 2018/08/27 15:11:17 riastradh Exp $ */ 2 1.1 riastrad 3 1.1 riastrad /*- 4 1.1 riastrad * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 1.1 riastrad * All rights reserved. 6 1.1 riastrad * 7 1.1 riastrad * This code is derived from software contributed to The NetBSD Foundation 8 1.1 riastrad * by Taylor R. Campbell. 9 1.1 riastrad * 10 1.1 riastrad * Redistribution and use in source and binary forms, with or without 11 1.1 riastrad * modification, are permitted provided that the following conditions 12 1.1 riastrad * are met: 13 1.1 riastrad * 1. Redistributions of source code must retain the above copyright 14 1.1 riastrad * notice, this list of conditions and the following disclaimer. 15 1.1 riastrad * 2. Redistributions in binary form must reproduce the above copyright 16 1.1 riastrad * notice, this list of conditions and the following disclaimer in the 17 1.1 riastrad * documentation and/or other materials provided with the distribution. 18 1.1 riastrad * 19 1.1 riastrad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.1 riastrad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.1 riastrad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.1 riastrad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.1 riastrad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.1 riastrad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.1 riastrad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.1 riastrad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.1 riastrad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.1 riastrad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.1 riastrad * POSSIBILITY OF SUCH DAMAGE. 30 1.1 riastrad */ 31 1.1 riastrad 32 1.1 riastrad #include <sys/cdefs.h> 33 1.1 riastrad __KERNEL_RCSID(0, "$NetBSD: linux_atomic64.c,v 1.3 2018/08/27 15:11:17 riastradh Exp $"); 34 1.1 riastrad 35 1.1 riastrad #include <sys/param.h> 36 1.1 riastrad #include <sys/bitops.h> 37 1.1 riastrad #include <sys/lock.h> 38 1.1 riastrad 39 1.1 riastrad #include <linux/atomic.h> 40 1.1 riastrad 41 1.1 riastrad #ifdef __HAVE_ATOMIC64_OPS 42 1.1 riastrad 43 1.1 riastrad int 44 1.1 riastrad linux_atomic64_init(void) 45 1.1 riastrad { 46 1.1 riastrad return 0; 47 1.1 riastrad } 48 1.1 riastrad 49 1.1 riastrad void 50 1.1 riastrad linux_atomic64_fini(void) 51 1.1 riastrad { 52 1.1 riastrad } 53 1.1 riastrad 54 1.1 riastrad #else 55 1.1 riastrad 56 1.1 riastrad static struct { 57 1.1 riastrad kmutex_t lock; 58 1.1 riastrad uint32_t gen; /* for unlocked read */ 59 1.1 riastrad char pad[CACHE_LINE_SIZE - 60 1.1 riastrad sizeof(kmutex_t) - sizeof(uint32_t)]; 61 1.1 riastrad } atomic64_tab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned; 62 1.1 riastrad CTASSERT(sizeof(atomic64_tab) == PAGE_SIZE); 63 1.1 riastrad CTASSERT(sizeof(atomic64_tab[0]) == CACHE_LINE_SIZE); 64 1.1 riastrad 65 1.1 riastrad int 66 1.1 riastrad linux_atomic64_init(void) 67 1.1 riastrad { 68 1.1 riastrad size_t i; 69 1.1 riastrad 70 1.1 riastrad for (i = 0; i < __arraycount(atomic64_tab); i++) { 71 1.1 riastrad mutex_init(&atomic64_tab[i].lock, MUTEX_DEFAULT, IPL_HIGH); 72 1.1 riastrad atomic64_tab[i].gen = 0; 73 1.1 riastrad } 74 1.1 riastrad 75 1.1 riastrad return 0; 76 1.1 riastrad } 77 1.1 riastrad 78 1.1 riastrad void 79 1.1 riastrad linux_atomic64_fini(void) 80 1.1 riastrad { 81 1.1 riastrad size_t i; 82 1.1 riastrad 83 1.1 riastrad for (i = 0; i < __arraycount(atomic64_tab); i++) { 84 1.1 riastrad KASSERT((atomic64_tab[i].gen & 1) == 0); 85 1.1 riastrad mutex_destroy(&atomic64_tab[i].lock); 86 1.1 riastrad } 87 1.1 riastrad } 88 1.1 riastrad 89 1.1 riastrad static inline size_t 90 1.1 riastrad atomic64_hash(const struct atomic64 *a) 91 1.1 riastrad { 92 1.1 riastrad 93 1.1 riastrad return ((uintptr_t)a >> ilog2(CACHE_LINE_SIZE)) % 94 1.1 riastrad __arraycount(atomic64_tab); 95 1.1 riastrad } 96 1.1 riastrad 97 1.1 riastrad static void 98 1.1 riastrad atomic64_lock(struct atomic64 *a) 99 1.1 riastrad { 100 1.1 riastrad size_t i = atomic64_hash(a); 101 1.1 riastrad 102 1.1 riastrad mutex_spin_enter(&atomic64_tab[i].lock); 103 1.1 riastrad KASSERT((atomic64_tab[i].gen & 1) == 0); 104 1.1 riastrad atomic64_tab[i].gen |= 1; 105 1.1 riastrad membar_producer(); 106 1.1 riastrad } 107 1.1 riastrad 108 1.1 riastrad static void 109 1.1 riastrad atomic64_unlock(struct atomic64 *a) 110 1.1 riastrad { 111 1.1 riastrad size_t i = atomic64_hash(a); 112 1.1 riastrad 113 1.1 riastrad KASSERT(mutex_owned(&atomic64_tab[i].lock)); 114 1.1 riastrad KASSERT((atomic64_tab[i].gen & 1) == 1); 115 1.1 riastrad 116 1.1 riastrad membar_producer(); 117 1.1 riastrad atomic64_tab[i].gen |= 1; /* paranoia */ 118 1.1 riastrad atomic64_tab[i].gen++; 119 1.1 riastrad mutex_spin_exit(&atomic64_tab[i].lock); 120 1.1 riastrad } 121 1.1 riastrad 122 1.1 riastrad uint64_t 123 1.1 riastrad atomic64_read(const struct atomic64 *a) 124 1.1 riastrad { 125 1.1 riastrad size_t i = atomic64_hash(a); 126 1.1 riastrad uint32_t gen; 127 1.1 riastrad uint64_t value; 128 1.1 riastrad 129 1.1 riastrad do { 130 1.1 riastrad while (__predict_false((gen = atomic64_tab[i].gen) & 1)) 131 1.1 riastrad SPINLOCK_BACKOFF_HOOK; 132 1.1 riastrad membar_consumer(); 133 1.1 riastrad value = a->a_v; 134 1.1 riastrad membar_consumer(); 135 1.1 riastrad } while (__predict_false(atomic64_tab[i].gen != gen)); 136 1.1 riastrad 137 1.1 riastrad return value; 138 1.1 riastrad } 139 1.1 riastrad 140 1.1 riastrad void 141 1.1 riastrad atomic64_set(struct atomic64 *a, uint64_t value) 142 1.1 riastrad { 143 1.1 riastrad 144 1.1 riastrad atomic64_lock(a); 145 1.1 riastrad a->a_v = value; 146 1.1 riastrad atomic64_unlock(a); 147 1.1 riastrad } 148 1.1 riastrad 149 1.1 riastrad void 150 1.2 riastrad atomic64_add(int64_t delta, struct atomic64 *a) 151 1.1 riastrad { 152 1.1 riastrad 153 1.1 riastrad atomic64_lock(a); 154 1.1 riastrad a->a_v += delta; 155 1.1 riastrad atomic64_unlock(a); 156 1.1 riastrad } 157 1.1 riastrad 158 1.1 riastrad void 159 1.2 riastrad atomic64_sub(int64_t delta, struct atomic64 *a) 160 1.1 riastrad { 161 1.1 riastrad 162 1.1 riastrad atomic64_lock(a); 163 1.1 riastrad a->a_v -= delta; 164 1.1 riastrad atomic64_unlock(a); 165 1.1 riastrad } 166 1.1 riastrad 167 1.3 riastrad int64_t 168 1.3 riastrad atomic64_add_return(int64_t delta, struct atomic64 *a) 169 1.3 riastrad { 170 1.3 riastrad int64_t v; 171 1.3 riastrad 172 1.3 riastrad atomic64_lock(a); 173 1.3 riastrad v = (int64_t)(a->a_v += delta); 174 1.3 riastrad atomic64_unlock(a); 175 1.3 riastrad 176 1.3 riastrad return v; 177 1.3 riastrad } 178 1.3 riastrad 179 1.1 riastrad uint64_t 180 1.1 riastrad atomic64_xchg(struct atomic64 *a, uint64_t new) 181 1.1 riastrad { 182 1.1 riastrad uint64_t old; 183 1.1 riastrad 184 1.1 riastrad atomic64_lock(a); 185 1.1 riastrad old = a->a_v; 186 1.1 riastrad a->a_v = new; 187 1.1 riastrad atomic64_unlock(a); 188 1.1 riastrad 189 1.1 riastrad return old; 190 1.1 riastrad } 191 1.1 riastrad 192 1.1 riastrad uint64_t 193 1.1 riastrad atomic64_cmpxchg(struct atomic64 *a, uint64_t expect, uint64_t new) 194 1.1 riastrad { 195 1.1 riastrad uint64_t old; 196 1.1 riastrad 197 1.1 riastrad atomic64_lock(a); 198 1.1 riastrad old = a->a_v; 199 1.1 riastrad if (old == expect) 200 1.1 riastrad a->a_v = new; 201 1.1 riastrad atomic64_unlock(a); 202 1.1 riastrad 203 1.1 riastrad return old; 204 1.1 riastrad } 205 1.1 riastrad 206 1.1 riastrad #endif 207