1 /* $NetBSD: aes_neon_impl.c,v 1.5 2020/10/10 08:24:10 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(1, "$NetBSD: aes_neon_impl.c,v 1.5 2020/10/10 08:24:10 jmcneill Exp $"); 31 32 #include <sys/types.h> 33 #include <sys/proc.h> 34 35 #include <crypto/aes/aes.h> 36 #include <crypto/aes/aes_impl.h> 37 #include <crypto/aes/arch/arm/aes_neon.h> 38 39 #ifdef __aarch64__ 40 #include <aarch64/armreg.h> 41 #endif 42 43 #ifdef _KERNEL 44 #ifndef __aarch64__ 45 #include <arm/locore.h> 46 #endif 47 #include <arm/fpu.h> 48 #else 49 #include <sys/sysctl.h> 50 #include <stddef.h> 51 #define fpu_kern_enter() ((void)0) 52 #define fpu_kern_leave() ((void)0) 53 #endif 54 55 static void 56 aes_neon_setenckey_impl(struct aesenc *enc, const uint8_t *key, 57 uint32_t nrounds) 58 { 59 60 fpu_kern_enter(); 61 aes_neon_setenckey(enc, key, nrounds); 62 fpu_kern_leave(); 63 } 64 65 static void 66 aes_neon_setdeckey_impl(struct aesdec *dec, const uint8_t *key, 67 uint32_t nrounds) 68 { 69 70 fpu_kern_enter(); 71 aes_neon_setdeckey(dec, key, nrounds); 72 fpu_kern_leave(); 73 } 74 75 static void 76 aes_neon_enc_impl(const struct aesenc *enc, const uint8_t in[static 16], 77 uint8_t out[static 16], uint32_t nrounds) 78 { 79 80 fpu_kern_enter(); 81 aes_neon_enc(enc, in, out, nrounds); 82 fpu_kern_leave(); 83 } 84 85 static void 86 aes_neon_dec_impl(const struct aesdec *dec, const uint8_t in[static 16], 87 uint8_t out[static 16], uint32_t nrounds) 88 { 89 90 fpu_kern_enter(); 91 aes_neon_dec(dec, in, out, nrounds); 92 fpu_kern_leave(); 93 } 94 95 static void 96 aes_neon_cbc_enc_impl(const struct aesenc *enc, const uint8_t in[static 16], 97 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 98 uint32_t nrounds) 99 { 100 101 if (nbytes == 0) 102 return; 103 fpu_kern_enter(); 104 aes_neon_cbc_enc(enc, in, out, nbytes, iv, nrounds); 105 fpu_kern_leave(); 106 } 107 108 static void 109 aes_neon_cbc_dec_impl(const struct aesdec *dec, const uint8_t in[static 16], 110 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 111 uint32_t nrounds) 112 { 113 114 if (nbytes == 0) 115 return; 116 fpu_kern_enter(); 117 aes_neon_cbc_dec(dec, in, out, nbytes, iv, nrounds); 118 fpu_kern_leave(); 119 } 120 121 static void 122 aes_neon_xts_enc_impl(const struct aesenc *enc, const uint8_t in[static 16], 123 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 124 uint32_t nrounds) 125 { 126 127 if (nbytes == 0) 128 return; 129 fpu_kern_enter(); 130 aes_neon_xts_enc(enc, in, out, nbytes, iv, nrounds); 131 fpu_kern_leave(); 132 } 133 134 static void 135 aes_neon_xts_dec_impl(const struct aesdec *dec, const uint8_t in[static 16], 136 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 137 uint32_t nrounds) 138 { 139 140 if (nbytes == 0) 141 return; 142 fpu_kern_enter(); 143 aes_neon_xts_dec(dec, in, out, nbytes, iv, nrounds); 144 fpu_kern_leave(); 145 } 146 147 static void 148 aes_neon_cbcmac_update1_impl(const struct aesenc *enc, 149 const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16], 150 uint32_t nrounds) 151 { 152 153 fpu_kern_enter(); 154 aes_neon_cbcmac_update1(enc, in, nbytes, auth, nrounds); 155 fpu_kern_leave(); 156 } 157 158 static void 159 aes_neon_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16], 160 uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32], 161 uint32_t nrounds) 162 { 163 164 fpu_kern_enter(); 165 aes_neon_ccm_enc1(enc, in, out, nbytes, authctr, nrounds); 166 fpu_kern_leave(); 167 } 168 169 static void 170 aes_neon_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16], 171 uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32], 172 uint32_t nrounds) 173 { 174 175 fpu_kern_enter(); 176 aes_neon_ccm_dec1(enc, in, out, nbytes, authctr, nrounds); 177 fpu_kern_leave(); 178 } 179 180 static int 181 aes_neon_probe(void) 182 { 183 #ifdef __aarch64__ 184 struct aarch64_sysctl_cpu_id *id; 185 #endif 186 int result = 0; 187 188 /* Verify that the CPU supports NEON. */ 189 #ifdef __aarch64__ 190 #ifdef _KERNEL 191 id = &curcpu()->ci_id; 192 #else 193 struct aarch64_sysctl_cpu_id ids; 194 size_t idlen; 195 id = &ids; 196 idlen = sizeof ids; 197 if (sysctlbyname("machdep.cpu0.cpu_id", id, &idlen, NULL, 0)) 198 return -1; 199 if (idlen != sizeof ids) 200 return -1; 201 #endif 202 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 203 case ID_AA64PFR0_EL1_ADV_SIMD_NONE: 204 return -1; 205 default: 206 break; 207 } 208 #else 209 #ifdef _KERNEL 210 if (!cpu_neon_present) 211 return -1; 212 #else 213 int neon; 214 size_t neonlen = sizeof neon; 215 if (0 && sysctlbyname("machdep.neon_present", &neon, &neonlen, NULL, 0)) 216 return -1; 217 if (0 && !neon) 218 return -1; 219 #endif 220 #endif 221 222 fpu_kern_enter(); 223 result = aes_neon_selftest(); 224 fpu_kern_leave(); 225 226 return result; 227 } 228 229 struct aes_impl aes_neon_impl = { 230 .ai_name = "ARM NEON vpaes", 231 .ai_probe = aes_neon_probe, 232 .ai_setenckey = aes_neon_setenckey_impl, 233 .ai_setdeckey = aes_neon_setdeckey_impl, 234 .ai_enc = aes_neon_enc_impl, 235 .ai_dec = aes_neon_dec_impl, 236 .ai_cbc_enc = aes_neon_cbc_enc_impl, 237 .ai_cbc_dec = aes_neon_cbc_dec_impl, 238 .ai_xts_enc = aes_neon_xts_enc_impl, 239 .ai_xts_dec = aes_neon_xts_dec_impl, 240 .ai_cbcmac_update1 = aes_neon_cbcmac_update1_impl, 241 .ai_ccm_enc1 = aes_neon_ccm_enc1_impl, 242 .ai_ccm_dec1 = aes_neon_ccm_dec1_impl, 243 }; 244
Indexes created Sat Sep 20 22:09:52 GMT 2025