aes_sse2.c revision 1.1
1 /* 2 * Copyright (c) 2016 Thomas Pornin <pornin (at) bolet.org> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining 5 * a copy of this software and associated documentation files (the 6 * "Software"), to deal in the Software without restriction, including 7 * without limitation the rights to use, copy, modify, merge, publish, 8 * distribute, sublicense, and/or sell copies of the Software, and to 9 * permit persons to whom the Software is furnished to do so, subject to 10 * the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be 13 * included in all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 19 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 20 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 * SOFTWARE. 23 */ 24 25 #include <sys/cdefs.h> 26 __KERNEL_RCSID(1, "$NetBSD: aes_sse2.c,v 1.1 2020/06/29 23:47:54 riastradh Exp $"); 27 28 #include <sys/types.h> 29 30 #include <lib/libkern/libkern.h> 31 32 #include "aes_sse2_impl.h" 33 34 static void 35 br_range_dec32le(uint32_t *p32, size_t nwords, const void *v) 36 { 37 const uint8_t *p8 = v; 38 39 while (nwords --> 0) { 40 uint32_t x0 = *p8++; 41 uint32_t x1 = *p8++; 42 uint32_t x2 = *p8++; 43 uint32_t x3 = *p8++; 44 45 *p32++ = x0 | (x1 << 8) | (x2 << 16) | (x3 << 24); 46 } 47 } 48 49 void 50 aes_sse2_bitslice_Sbox(__m128i q[static 4]) 51 { 52 __m128i x0, x1, x2, x3, x4, x5, x6, x7; 53 __m128i y1, y2, y3, y4, y5, y6, y7, y8, y9; 54 __m128i y10, y11, y12, y13, y14, y15, y16, y17, y18, y19; 55 __m128i y20, y21; 56 __m128i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9; 57 __m128i z10, z11, z12, z13, z14, z15, z16, z17; 58 __m128i t0, t1, t2, t3, t4, t5, t6, t7, t8, t9; 59 __m128i t10, t11, t12, t13, t14, t15, t16, t17, t18, t19; 60 __m128i t20, t21, t22, t23, t24, t25, t26, t27, t28, t29; 61 __m128i t30, t31, t32, t33, t34, t35, t36, t37, t38, t39; 62 __m128i t40, t41, t42, t43, t44, t45, t46, t47, t48, t49; 63 __m128i t50, t51, t52, t53, t54, t55, t56, t57, t58, t59; 64 __m128i t60, t61, t62, t63, t64, t65, t66, t67; 65 __m128i s0, s1, s2, s3, s4, s5, s6, s7; 66 67 x0 = _mm_shuffle_epi32(q[3], 0x0e); 68 x1 = _mm_shuffle_epi32(q[2], 0x0e); 69 x2 = _mm_shuffle_epi32(q[1], 0x0e); 70 x3 = _mm_shuffle_epi32(q[0], 0x0e); 71 x4 = q[3]; 72 x5 = q[2]; 73 x6 = q[1]; 74 x7 = q[0]; 75 76 /* 77 * Top linear transformation. 78 */ 79 y14 = x3 ^ x5; 80 y13 = x0 ^ x6; 81 y9 = x0 ^ x3; 82 y8 = x0 ^ x5; 83 t0 = x1 ^ x2; 84 y1 = t0 ^ x7; 85 y4 = y1 ^ x3; 86 y12 = y13 ^ y14; 87 y2 = y1 ^ x0; 88 y5 = y1 ^ x6; 89 y3 = y5 ^ y8; 90 t1 = x4 ^ y12; 91 y15 = t1 ^ x5; 92 y20 = t1 ^ x1; 93 y6 = y15 ^ x7; 94 y10 = y15 ^ t0; 95 y11 = y20 ^ y9; 96 y7 = x7 ^ y11; 97 y17 = y10 ^ y11; 98 y19 = y10 ^ y8; 99 y16 = t0 ^ y11; 100 y21 = y13 ^ y16; 101 y18 = x0 ^ y16; 102 103 /* 104 * Non-linear section. 105 */ 106 t2 = y12 & y15; 107 t3 = y3 & y6; 108 t4 = t3 ^ t2; 109 t5 = y4 & x7; 110 t6 = t5 ^ t2; 111 t7 = y13 & y16; 112 t8 = y5 & y1; 113 t9 = t8 ^ t7; 114 t10 = y2 & y7; 115 t11 = t10 ^ t7; 116 t12 = y9 & y11; 117 t13 = y14 & y17; 118 t14 = t13 ^ t12; 119 t15 = y8 & y10; 120 t16 = t15 ^ t12; 121 t17 = t4 ^ t14; 122 t18 = t6 ^ t16; 123 t19 = t9 ^ t14; 124 t20 = t11 ^ t16; 125 t21 = t17 ^ y20; 126 t22 = t18 ^ y19; 127 t23 = t19 ^ y21; 128 t24 = t20 ^ y18; 129 130 t25 = t21 ^ t22; 131 t26 = t21 & t23; 132 t27 = t24 ^ t26; 133 t28 = t25 & t27; 134 t29 = t28 ^ t22; 135 t30 = t23 ^ t24; 136 t31 = t22 ^ t26; 137 t32 = t31 & t30; 138 t33 = t32 ^ t24; 139 t34 = t23 ^ t33; 140 t35 = t27 ^ t33; 141 t36 = t24 & t35; 142 t37 = t36 ^ t34; 143 t38 = t27 ^ t36; 144 t39 = t29 & t38; 145 t40 = t25 ^ t39; 146 147 t41 = t40 ^ t37; 148 t42 = t29 ^ t33; 149 t43 = t29 ^ t40; 150 t44 = t33 ^ t37; 151 t45 = t42 ^ t41; 152 z0 = t44 & y15; 153 z1 = t37 & y6; 154 z2 = t33 & x7; 155 z3 = t43 & y16; 156 z4 = t40 & y1; 157 z5 = t29 & y7; 158 z6 = t42 & y11; 159 z7 = t45 & y17; 160 z8 = t41 & y10; 161 z9 = t44 & y12; 162 z10 = t37 & y3; 163 z11 = t33 & y4; 164 z12 = t43 & y13; 165 z13 = t40 & y5; 166 z14 = t29 & y2; 167 z15 = t42 & y9; 168 z16 = t45 & y14; 169 z17 = t41 & y8; 170 171 /* 172 * Bottom linear transformation. 173 */ 174 t46 = z15 ^ z16; 175 t47 = z10 ^ z11; 176 t48 = z5 ^ z13; 177 t49 = z9 ^ z10; 178 t50 = z2 ^ z12; 179 t51 = z2 ^ z5; 180 t52 = z7 ^ z8; 181 t53 = z0 ^ z3; 182 t54 = z6 ^ z7; 183 t55 = z16 ^ z17; 184 t56 = z12 ^ t48; 185 t57 = t50 ^ t53; 186 t58 = z4 ^ t46; 187 t59 = z3 ^ t54; 188 t60 = t46 ^ t57; 189 t61 = z14 ^ t57; 190 t62 = t52 ^ t58; 191 t63 = t49 ^ t58; 192 t64 = z4 ^ t59; 193 t65 = t61 ^ t62; 194 t66 = z1 ^ t63; 195 s0 = t59 ^ t63; 196 s6 = t56 ^ ~t62; 197 s7 = t48 ^ ~t60; 198 t67 = t64 ^ t65; 199 s3 = t53 ^ t66; 200 s4 = t51 ^ t66; 201 s5 = t47 ^ t65; 202 s1 = t64 ^ ~s3; 203 s2 = t55 ^ ~t67; 204 205 q[3] = _mm_unpacklo_epi64(s4, s0); 206 q[2] = _mm_unpacklo_epi64(s5, s1); 207 q[1] = _mm_unpacklo_epi64(s6, s2); 208 q[0] = _mm_unpacklo_epi64(s7, s3); 209 } 210 211 void 212 aes_sse2_ortho(__m128i q[static 4]) 213 { 214 #define SWAPN(cl, ch, s, x, y) do { \ 215 __m128i a, b; \ 216 a = (x); \ 217 b = (y); \ 218 (x) = (a & _mm_set1_epi64x(cl)) | \ 219 _mm_slli_epi64(b & _mm_set1_epi64x(cl), (s)); \ 220 (y) = _mm_srli_epi64(a & _mm_set1_epi64x(ch), (s)) | \ 221 (b & _mm_set1_epi64x(ch)); \ 222 } while (0) 223 224 #define SWAP2(x, y) SWAPN(0x5555555555555555, 0xAAAAAAAAAAAAAAAA, 1, x, y) 225 #define SWAP4(x, y) SWAPN(0x3333333333333333, 0xCCCCCCCCCCCCCCCC, 2, x, y) 226 #define SWAP8(x, y) SWAPN(0x0F0F0F0F0F0F0F0F, 0xF0F0F0F0F0F0F0F0, 4, x, y) 227 228 SWAP2(q[0], q[1]); 229 SWAP2(q[2], q[3]); 230 231 SWAP4(q[0], q[2]); 232 SWAP4(q[1], q[3]); 233 234 __m128i q0 = q[0]; 235 __m128i q1 = q[1]; 236 __m128i q2 = q[2]; 237 __m128i q3 = q[3]; 238 __m128i q4 = _mm_shuffle_epi32(q[0], 0x0e); 239 __m128i q5 = _mm_shuffle_epi32(q[1], 0x0e); 240 __m128i q6 = _mm_shuffle_epi32(q[2], 0x0e); 241 __m128i q7 = _mm_shuffle_epi32(q[3], 0x0e); 242 SWAP8(q0, q4); 243 SWAP8(q1, q5); 244 SWAP8(q2, q6); 245 SWAP8(q3, q7); 246 q[0] = _mm_unpacklo_epi64(q0, q4); 247 q[1] = _mm_unpacklo_epi64(q1, q5); 248 q[2] = _mm_unpacklo_epi64(q2, q6); 249 q[3] = _mm_unpacklo_epi64(q3, q7); 250 } 251 252 __m128i 253 aes_sse2_interleave_in(__m128i w) 254 { 255 __m128i lo, hi; 256 257 lo = _mm_shuffle_epi32(w, 0x10); 258 hi = _mm_shuffle_epi32(w, 0x32); 259 lo &= _mm_set1_epi64x(0x00000000FFFFFFFF); 260 hi &= _mm_set1_epi64x(0x00000000FFFFFFFF); 261 lo |= _mm_slli_epi64(lo, 16); 262 hi |= _mm_slli_epi64(hi, 16); 263 lo &= _mm_set1_epi32(0x0000FFFF); 264 hi &= _mm_set1_epi32(0x0000FFFF); 265 lo |= _mm_slli_epi64(lo, 8); 266 hi |= _mm_slli_epi64(hi, 8); 267 lo &= _mm_set1_epi16(0x00FF); 268 hi &= _mm_set1_epi16(0x00FF); 269 return lo | _mm_slli_epi64(hi, 8); 270 } 271 272 __m128i 273 aes_sse2_interleave_out(__m128i q) 274 { 275 __m128i lo, hi; 276 277 lo = q; 278 hi = _mm_srli_si128(q, 1); 279 lo &= _mm_set1_epi16(0x00FF); 280 hi &= _mm_set1_epi16(0x00FF); 281 lo |= _mm_srli_epi64(lo, 8); 282 hi |= _mm_srli_epi64(hi, 8); 283 lo &= _mm_set1_epi32(0x0000FFFF); 284 hi &= _mm_set1_epi32(0x0000FFFF); 285 lo |= _mm_srli_epi64(lo, 16); 286 hi |= _mm_srli_epi64(hi, 16); 287 return (__m128i)_mm_shuffle_ps((__m128)lo, (__m128)hi, 0x88); 288 } 289 290 static const unsigned char Rcon[] = { 291 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36 292 }; 293 294 static uint32_t 295 sub_word(uint32_t x) 296 { 297 __m128i q[4]; 298 uint32_t y; 299 300 memset(q, 0, sizeof(q)); 301 q[0] = _mm_loadu_si32(&x); 302 aes_sse2_ortho(q); 303 aes_sse2_bitslice_Sbox(q); 304 aes_sse2_ortho(q); 305 _mm_storeu_si32(&y, q[0]); 306 return y; 307 } 308 309 unsigned 310 aes_sse2_keysched(uint64_t *comp_skey, const void *key, size_t key_len) 311 { 312 unsigned num_rounds; 313 int i, j, k, nk, nkf; 314 uint32_t tmp; 315 uint32_t skey[60]; 316 317 switch (key_len) { 318 case 16: 319 num_rounds = 10; 320 break; 321 case 24: 322 num_rounds = 12; 323 break; 324 case 32: 325 num_rounds = 14; 326 break; 327 default: 328 /* abort(); */ 329 return 0; 330 } 331 nk = (int)(key_len >> 2); 332 nkf = (int)((num_rounds + 1) << 2); 333 br_range_dec32le(skey, (key_len >> 2), key); 334 tmp = skey[(key_len >> 2) - 1]; 335 for (i = nk, j = 0, k = 0; i < nkf; i ++) { 336 if (j == 0) { 337 tmp = (tmp << 24) | (tmp >> 8); 338 tmp = sub_word(tmp) ^ Rcon[k]; 339 } else if (nk > 6 && j == 4) { 340 tmp = sub_word(tmp); 341 } 342 tmp ^= skey[i - nk]; 343 skey[i] = tmp; 344 if (++ j == nk) { 345 j = 0; 346 k ++; 347 } 348 } 349 350 for (i = 0, j = 0; i < nkf; i += 4, j += 2) { 351 __m128i q[4], q0, q1, q2, q3, q4, q5, q6, q7; 352 __m128i w; 353 354 w = _mm_loadu_epi8(skey + i); 355 q[0] = q[1] = q[2] = q[3] = aes_sse2_interleave_in(w); 356 aes_sse2_ortho(q); 357 q0 = q[0] & _mm_set1_epi64x(0x1111111111111111); 358 q1 = q[1] & _mm_set1_epi64x(0x2222222222222222); 359 q2 = q[2] & _mm_set1_epi64x(0x4444444444444444); 360 q3 = q[3] & _mm_set1_epi64x(0x8888888888888888); 361 q4 = _mm_shuffle_epi32(q0, 0x0e); 362 q5 = _mm_shuffle_epi32(q1, 0x0e); 363 q6 = _mm_shuffle_epi32(q2, 0x0e); 364 q7 = _mm_shuffle_epi32(q3, 0x0e); 365 _mm_storeu_si64(&comp_skey[j + 0], q0 | q1 | q2 | q3); 366 _mm_storeu_si64(&comp_skey[j + 1], q4 | q5 | q6 | q7); 367 } 368 return num_rounds; 369 } 370 371 void 372 aes_sse2_skey_expand(uint64_t *skey, 373 unsigned num_rounds, const uint64_t *comp_skey) 374 { 375 unsigned u, v, n; 376 377 n = (num_rounds + 1) << 1; 378 for (u = 0, v = 0; u < n; u ++, v += 4) { 379 __m128i x0, x1, x2, x3; 380 381 x0 = x1 = x2 = x3 = _mm_loadu_si64(&comp_skey[u]); 382 x0 &= 0x1111111111111111; 383 x1 &= 0x2222222222222222; 384 x2 &= 0x4444444444444444; 385 x3 &= 0x8888888888888888; 386 x1 = _mm_srli_epi64(x1, 1); 387 x2 = _mm_srli_epi64(x2, 2); 388 x3 = _mm_srli_epi64(x3, 3); 389 x0 = _mm_sub_epi64(_mm_slli_epi64(x0, 4), x0); 390 x1 = _mm_sub_epi64(_mm_slli_epi64(x1, 4), x1); 391 x2 = _mm_sub_epi64(_mm_slli_epi64(x2, 4), x2); 392 x3 = _mm_sub_epi64(_mm_slli_epi64(x3, 4), x3); 393 _mm_storeu_si64(&skey[v + 0], x0); 394 _mm_storeu_si64(&skey[v + 1], x1); 395 _mm_storeu_si64(&skey[v + 2], x2); 396 _mm_storeu_si64(&skey[v + 3], x3); 397 } 398 } 399
Indexes created Wed Oct 01 12:09:54 GMT 2025