1 /* $NetBSD: __aarch64_lse.S,v 1.7 2022/08/06 21:31:33 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2021 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nick Hudson. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 #include "atomic_op_asm.h" 35 36 #if SZ == 1 37 #define OPSFX b 38 #define R0 w0 39 #define R1 w1 40 #define R4 w4 41 #endif 42 43 #if SZ == 2 44 #define OPSFX h 45 #define R0 w0 46 #define R1 w1 47 #define R4 w4 48 #endif 49 50 #if SZ == 4 51 #define OPSFX 52 #define R0 w0 53 #define R1 w1 54 #define R4 w4 55 #endif 56 57 #if SZ == 8 58 #define OPSFX 59 #define R0 x0 60 #define R1 x1 61 #define R4 x4 62 #endif 63 64 #if defined(AR_relax) 65 #define ACQ 66 #define REL 67 #define DMB 68 #endif 69 70 #if defined(AR_acq) 71 #define ACQ a 72 #define REL 73 #define DMB 74 #endif 75 76 #if defined(AR_rel) 77 #define ACQ 78 #define REL l 79 #define DMB 80 #endif 81 82 #if defined(AR_acq_rel) 83 #define ACQ a 84 #define REL l 85 #define DMB 86 #endif 87 88 #if defined(AR_sync) 89 #define ACQ 90 #define REL 91 #define DMB dmb ish 92 #endif 93 94 #if defined(OP_clr) 95 #define INSNOP bic 96 #endif 97 98 #if defined(OP_set) 99 #define INSNOP orr 100 #endif 101 102 #if defined(OP_add) 103 #define INSNOP add 104 #endif 105 106 #if defined(OP_eor) 107 #define INSNOP eor 108 #endif 109 110 #define _CONCAT3(A, B, C) __CONCAT3(A,B,C) 111 #define _CONCAT4(A, B, C, D) __CONCAT4(A,B,C,D) 112 #define _CONCAT5(A, B, C, D, E) __CONCAT5(A,B,C,D,E) 113 114 #define FUNC2 _CONCAT3(__aarch64_,OP,AR) 115 #define FUNC3 _CONCAT4(__aarch64_,OP,SZ,AR) 116 117 #define CASP_FUNC FUNC2 118 #define CAS_FUNC FUNC3 119 #define SWP_FUNC FUNC3 120 #define INSN_FUNC FUNC3 121 122 #define LDXR _CONCAT4(ld, ACQ, xr, OPSFX) 123 #define STXR _CONCAT4(st, REL, xr, OPSFX) 124 #define LDXP _CONCAT3(ld, ACQ, xp) 125 #define STXP _CONCAT3(st, REL, xp) 126 127 #ifdef _HAVE_LSE 128 #define SWP _CONCAT4(swp, ACQ, REL, OPSFX) 129 #define CAS _CONCAT4(cas, ACQ, REL, OPSFX) 130 #define CASP _CONCAT3(casp, ACQ, REL) 131 #define INSN _CONCAT5(ld, OP, ACQ, REL, OPSFX) 132 133 .hidden __aarch64_have_lse_atomics 134 .arch armv8-a+lse 135 136 #define DO_LSE_INSN_IF_SUPPORTED(label) \ 137 adrp x4, __aarch64_have_lse_atomics ;\ 138 ldrb w4, [x4, #:lo12:__aarch64_have_lse_atomics] ;\ 139 cbnz w4, label 140 141 #endif 142 143 #if defined(OP_swp) 144 ENTRY_NP(SWP_FUNC) 145 #ifdef _HAVE_LSE 146 DO_LSE_INSN_IF_SUPPORTED(99f) 147 DMB 148 SWP R0, R0, [x1] 149 DMB 150 ret 151 99: 152 #endif 153 mov x4, x0 /* need x0 for return value */ 154 DMB /* potential barrier */ 155 1: LDXR R0, [x1] /* load old value */ 156 STXR w3, R4, [x1] /* store new value */ 157 cbnz w3, 2f /* succeed?? no, try again */ 158 DMB /* potential barrier */ 159 ret /* return old value */ 160 2: b 1b 161 END(SWP_FUNC) 162 #endif 163 164 #if defined(OP_cas) 165 ENTRY_NP(CAS_FUNC) 166 #ifdef _HAVE_LSE 167 DO_LSE_INSN_IF_SUPPORTED(99f) 168 DMB 169 CAS R0, R1, [x2] 170 DMB 171 ret 172 99: 173 #endif 174 mov x4, x0 /* need x0 for return value */ 175 DMB /* potential barrier */ 176 1: LDXR R0, [x2] /* load old value */ 177 cmp R0, R4 /* compare */ 178 b.ne 2f /* not equal? return */ 179 STXR w3, R1, [x2] /* store new value */ 180 cbnz w3, 3f /* succeed? nope, try again. */ 181 DMB /* potential barrier */ 182 2: ret /* return. */ 183 3: b 1b 184 END(CAS_FUNC) 185 #endif 186 187 #if defined(OP_casp) 188 ENTRY_NP(CASP_FUNC) 189 #ifdef _HAVE_LSE 190 DO_LSE_INSN_IF_SUPPORTED(99f) 191 DMB 192 CASP x0, x1, x2, x3, [x4] 193 DMB 194 ret 195 99: 196 #endif 197 mov x5, x0 /* need x0 for return value */ 198 mov x6, x1 /* need x1 for return value */ 199 DMB /* potential barrier */ 200 1: LDXP x0, x1, [x4] /* load old value */ 201 cmp x0, x5 /* compare */ 202 b.ne 2f /* not equal? return */ 203 cmp x1, x6 204 b.ne 2f /* not equal? return */ 205 STXP w7, x2, x3, [x4] /* store new value */ 206 cbnz w7, 3f /* succeed? nope, try again. */ 207 DMB /* potential barrier */ 208 2: ret /* return. */ 209 3: b 1b 210 END(CASP_FUNC) 211 #endif 212 213 #if defined(OP_set) || defined(OP_clr) || defined(OP_add) || defined(OP_eor) 214 ENTRY_NP(INSN_FUNC) 215 #ifdef _HAVE_LSE 216 DO_LSE_INSN_IF_SUPPORTED(99f) 217 DMB 218 INSN R0, R0, [x1] 219 DMB 220 ret 221 99: 222 #endif 223 mov x4, x0 /* need x0 for return value */ 224 DMB /* potential barrier */ 225 1: LDXR R0, [x1] /* load old value */ 226 INSNOP R4, R0, R4 227 STXR w3, R4, [x1] /* store new value */ 228 cbnz w3, 2f /* succeed?? no, try again */ 229 DMB /* potential barrier */ 230 ret /* return old value */ 231 2: b 1b 232 END(INSN_FUNC) 233 #endif 234
Indexes created Mon Sep 22 05:09:51 GMT 2025