1 /* $NetBSD: aes_sse2_4x32_subr.c,v 1.1 2025/11/23 22:48:27 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2025 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(1, "$NetBSD: aes_sse2_4x32_subr.c,v 1.1 2025/11/23 22:48:27 riastradh Exp $"); 31 32 #include <crypto/aes/aes.h> 33 34 #ifdef _KERNEL 35 #include <sys/systm.h> 36 #include <lib/libkern/libkern.h> 37 #else 38 #include <err.h> 39 #include <assert.h> 40 #include <inttypes.h> 41 #include <stdio.h> 42 #include <string.h> 43 #define KASSERT assert 44 #define panic(fmt, args...) err(1, fmt, ##args) 45 #endif 46 47 #include "aes_sse2_4x32_impl.h" 48 #include "aes_sse2_4x32_subr.h" 49 50 #ifndef _MM_TRANSPOSE4_EPI32 51 #define _MM_TRANSPOSE4_EPI32(r0, r1, r2, r3) do \ 52 { \ 53 __m128i _mm_tmp0, _mm_tmp1, _mm_tmp2, _mm_tmp3; \ 54 \ 55 _mm_tmp0 = _mm_unpacklo_epi32(r0, r1); \ 56 _mm_tmp2 = _mm_unpacklo_epi32(r2, r3); \ 57 _mm_tmp1 = _mm_unpackhi_epi32(r0, r1); \ 58 _mm_tmp3 = _mm_unpackhi_epi32(r2, r3); \ 59 (r0) = (__m128i)_mm_movelh_ps((__m128)_mm_tmp0, (__m128)_mm_tmp2); \ 60 (r1) = (__m128i)_mm_movehl_ps((__m128)_mm_tmp2, (__m128)_mm_tmp0); \ 61 (r2) = (__m128i)_mm_movelh_ps((__m128)_mm_tmp1, (__m128)_mm_tmp3); \ 62 (r3) = (__m128i)_mm_movehl_ps((__m128)_mm_tmp3, (__m128)_mm_tmp1); \ 63 } while (0) 64 #endif 65 66 void 67 aes_sse2_4x32_setkey(uint32_t rk[static 60], const void *key, uint32_t nrounds) 68 { 69 size_t key_len; 70 71 switch (nrounds) { 72 case 10: 73 key_len = 16; 74 break; 75 case 12: 76 key_len = 24; 77 break; 78 case 14: 79 key_len = 32; 80 break; 81 default: 82 panic("invalid AES nrounds: %u", nrounds); 83 } 84 85 aes_sse2_4x32_keysched(rk, key, key_len); 86 } 87 88 void 89 aes_sse2_4x32_enc(const struct aesenc *enc, const uint8_t in[static 16], 90 uint8_t out[static 16], uint32_t nrounds) 91 { 92 uint32_t sk_exp[120]; 93 __m128i q[8]; 94 95 /* Expand round keys for bitslicing. */ 96 aes_sse2_4x32_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk); 97 98 /* Load input block interleaved with garbage blocks. */ 99 q[0] = _mm_loadu_epi8(in); 100 q[2] = q[4] = q[6] = _mm_setzero_si128(); 101 q[1] = q[3] = q[5] = q[7] = _mm_setzero_si128(); 102 103 /* Transform to bitslice, decrypt, transform from bitslice. */ 104 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 105 aes_sse2_4x32_ortho(q); 106 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 107 aes_sse2_4x32_ortho(q); 108 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 109 110 /* Store output block. */ 111 _mm_storeu_epi8(out, q[0]); 112 113 /* Paranoia: Zero temporary buffers. */ 114 explicit_memset(sk_exp, 0, sizeof sk_exp); 115 explicit_memset(q, 0, sizeof q); 116 } 117 118 void 119 aes_sse2_4x32_dec(const struct aesdec *dec, const uint8_t in[static 16], 120 uint8_t out[static 16], uint32_t nrounds) 121 { 122 uint32_t sk_exp[120]; 123 __m128i q[8]; 124 125 /* Expand round keys for bitslicing. */ 126 aes_sse2_4x32_skey_expand(sk_exp, nrounds, dec->aesd_aes.aes_rk); 127 128 /* Load input block interleaved with garbage blocks. */ 129 q[0] = _mm_loadu_epi8(in); 130 q[2] = q[4] = q[6] = _mm_setzero_si128(); 131 q[1] = q[3] = q[5] = q[7] = _mm_setzero_si128(); 132 133 /* Transform to bitslice, decrypt, transform from bitslice. */ 134 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 135 aes_sse2_4x32_ortho(q); 136 aes_sse2_4x32_bitslice_decrypt(nrounds, sk_exp, q); 137 aes_sse2_4x32_ortho(q); 138 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 139 140 /* Store output block. */ 141 _mm_storeu_epi8(out, q[0]); 142 143 /* Paranoia: Zero temporary buffers. */ 144 explicit_memset(sk_exp, 0, sizeof sk_exp); 145 explicit_memset(q, 0, sizeof q); 146 } 147 148 void 149 aes_sse2_4x32_cbc_enc(const struct aesenc *enc, const uint8_t in[static 16], 150 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 151 uint32_t nrounds) 152 { 153 uint32_t sk_exp[120]; 154 __m128i q[8]; 155 __m128i cv; 156 157 KASSERT(nbytes); 158 KASSERT(nbytes % 16 == 0); 159 160 /* Expand round keys for bitslicing. */ 161 aes_sse2_4x32_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk); 162 163 /* Load the IV. */ 164 cv = _mm_loadu_epi8(iv); 165 166 /* 167 * Zero the registers we won't be using, since CBC encryption 168 * is inherently sequential so we can only do one block at a 169 * time. 170 */ 171 q[2] = q[4] = q[6] = _mm_setzero_si128(); 172 q[1] = q[3] = q[5] = q[7] = _mm_setzero_si128(); 173 174 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 175 /* Load input block and apply CV. */ 176 q[0] = cv ^ _mm_loadu_epi8(in); 177 178 /* Transform to bitslice, encrypt, transform from bitslice. */ 179 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 180 aes_sse2_4x32_ortho(q); 181 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 182 aes_sse2_4x32_ortho(q); 183 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 184 185 /* Remember ciphertext as CV and store output block. */ 186 cv = q[0]; 187 _mm_storeu_epi8(out, cv); 188 } 189 190 /* Store updated IV. */ 191 _mm_storeu_epi8(iv, cv); 192 193 /* Paranoia: Zero temporary buffers. */ 194 explicit_memset(sk_exp, 0, sizeof sk_exp); 195 explicit_memset(q, 0, sizeof q); 196 } 197 198 void 199 aes_sse2_4x32_cbc_dec(const struct aesdec *dec, const uint8_t in[static 16], 200 uint8_t out[static 16], size_t nbytes, uint8_t ivp[static 16], 201 uint32_t nrounds) 202 { 203 uint32_t sk_exp[120]; 204 __m128i q[8]; 205 __m128i cv, iv, w; 206 207 KASSERT(nbytes); 208 KASSERT(nbytes % 16 == 0); 209 210 /* Expand round keys for bitslicing. */ 211 aes_sse2_4x32_skey_expand(sk_exp, nrounds, dec->aesd_aes.aes_rk); 212 213 /* Load the IV. */ 214 iv = _mm_loadu_epi8(ivp); 215 216 /* Load the last cipher block. */ 217 cv = _mm_loadu_epi8(in + nbytes - 16); 218 219 /* Store the updated IV. */ 220 _mm_storeu_epi8(ivp, cv); 221 222 /* Process the last blocks if not an even multiple of eight. */ 223 if (nbytes % (8*16)) { 224 unsigned i, n = (nbytes/16) % 8; 225 226 KASSERT(n > 0); 227 KASSERT(n < 8); 228 229 for (i = 8; i --> n;) 230 q[i] = _mm_setzero_si128(); 231 q[i] = cv; 232 while (i --> 0) 233 q[i] = _mm_loadu_epi8(in + nbytes - 16*n + 16*i); 234 235 /* Decrypt up to seven blocks. */ 236 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 237 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 238 aes_sse2_4x32_ortho(q); 239 aes_sse2_4x32_bitslice_decrypt(nrounds, sk_exp, q); 240 aes_sse2_4x32_ortho(q); 241 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 242 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 243 244 do { 245 n--; 246 w = q[n]; 247 if ((nbytes -= 16) == 0) 248 goto out; 249 cv = _mm_loadu_epi8(in + nbytes - 16); 250 _mm_storeu_epi8(out + nbytes, w ^ cv); 251 } while (n); 252 } 253 254 for (;;) { 255 KASSERT(nbytes >= 128); 256 nbytes -= 128; 257 258 /* 259 * 1. Set up upper cipher block from cv. 260 * 2. Load lower cipher blocks from input. 261 */ 262 q[7] = cv; /* _mm_loadu_epi8(in + nbytes + 16*7) */ 263 q[6] = _mm_loadu_epi8(in + nbytes + 16*6); 264 q[5] = _mm_loadu_epi8(in + nbytes + 16*5); 265 q[4] = _mm_loadu_epi8(in + nbytes + 16*4); 266 q[3] = _mm_loadu_epi8(in + nbytes + 16*3); 267 q[2] = _mm_loadu_epi8(in + nbytes + 16*2); 268 q[1] = _mm_loadu_epi8(in + nbytes + 16*1); 269 q[0] = _mm_loadu_epi8(in + nbytes + 16*0); 270 271 /* Decrypt eight blocks at a time. */ 272 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 273 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 274 aes_sse2_4x32_ortho(q); 275 aes_sse2_4x32_bitslice_decrypt(nrounds, sk_exp, q); 276 aes_sse2_4x32_ortho(q); 277 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 278 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 279 280 /* Store the seven upper output blocks. */ 281 cv = _mm_loadu_epi8(in + nbytes + 16*6); 282 _mm_storeu_epi8(out + nbytes + 16*7, cv ^ q[7]); 283 cv = _mm_loadu_epi8(in + nbytes + 16*5); 284 _mm_storeu_epi8(out + nbytes + 16*6, cv ^ q[6]); 285 cv = _mm_loadu_epi8(in + nbytes + 16*4); 286 _mm_storeu_epi8(out + nbytes + 16*5, cv ^ q[5]); 287 cv = _mm_loadu_epi8(in + nbytes + 16*3); 288 _mm_storeu_epi8(out + nbytes + 16*4, cv ^ q[4]); 289 cv = _mm_loadu_epi8(in + nbytes + 16*2); 290 _mm_storeu_epi8(out + nbytes + 16*3, cv ^ q[3]); 291 cv = _mm_loadu_epi8(in + nbytes + 16*1); 292 _mm_storeu_epi8(out + nbytes + 16*2, cv ^ q[2]); 293 cv = _mm_loadu_epi8(in + nbytes + 16*0); 294 _mm_storeu_epi8(out + nbytes + 16*1, cv ^ q[1]); 295 296 /* 297 * Get the first output block, but don't load the CV 298 * yet -- it might be the previous ciphertext block, or 299 * it might be the IV. 300 */ 301 w = q[0]; 302 303 /* Stop if we've reached the first output block. */ 304 if (nbytes == 0) 305 goto out; 306 307 /* 308 * Load the preceding cipher block, and apply it as the 309 * chaining value to this one. 310 */ 311 cv = _mm_loadu_epi8(in + nbytes - 16); 312 _mm_storeu_epi8(out + nbytes, w ^ cv); 313 } 314 315 out: /* Store the first output block. */ 316 _mm_storeu_epi8(out, w ^ iv); 317 318 /* Paranoia: Zero temporary buffers. */ 319 explicit_memset(sk_exp, 0, sizeof sk_exp); 320 explicit_memset(q, 0, sizeof q); 321 } 322 323 static inline __m128i 324 aes_sse2_4x32_xts_update(__m128i t) 325 { 326 const __m128i one = _mm_set_epi64x(1, 1); 327 __m128i s, m, c; 328 329 s = _mm_srli_epi64(t, 63); /* 1 if high bit set else 0 */ 330 m = _mm_sub_epi64(s, one); /* 0 if high bit set else -1 */ 331 m = _mm_shuffle_epi32(m, 0x4e); /* swap halves */ 332 c = _mm_set_epi64x(1, 0x87); /* carry */ 333 334 return _mm_slli_epi64(t, 1) ^ (c & ~m); 335 } 336 337 static int 338 aes_sse2_4x32_xts_update_selftest(void) 339 { 340 static const struct { 341 uint32_t in[4], out[4]; 342 } cases[] = { 343 [0] = { {1}, {2} }, 344 [1] = { {0x80000000U,0,0,0}, {0,1,0,0} }, 345 [2] = { {0,0x80000000U,0,0}, {0,0,1,0} }, 346 [3] = { {0,0,0x80000000U,0}, {0,0,0,1} }, 347 [4] = { {0,0,0,0x80000000U}, {0x87,0,0,0} }, 348 [5] = { {0,0x80000000U,0,0x80000000U}, {0x87,0,1,0} }, 349 }; 350 unsigned i; 351 uint32_t t[4]; 352 int result = 0; 353 354 for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) { 355 t[0] = cases[i].in[0]; 356 t[1] = cases[i].in[1]; 357 t[2] = cases[i].in[2]; 358 t[3] = cases[i].in[3]; 359 _mm_storeu_epi8(t, aes_sse2_4x32_xts_update(_mm_loadu_epi8(t))); 360 if (t[0] != cases[i].out[0] || 361 t[1] != cases[i].out[1] || 362 t[2] != cases[i].out[2] || 363 t[3] != cases[i].out[3]) { 364 printf("%s %u:" 365 " %"PRIx32" %"PRIx32" %"PRIx32" %"PRIx32"\n", 366 __func__, i, t[0], t[1], t[2], t[3]); 367 result = -1; 368 } 369 } 370 371 return result; 372 } 373 374 void 375 aes_sse2_4x32_xts_enc(const struct aesenc *enc, const uint8_t in[static 16], 376 uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16], 377 uint32_t nrounds) 378 { 379 uint32_t sk_exp[120]; 380 __m128i q[8]; 381 __m128i t[9]; 382 unsigned i; 383 384 KASSERT(nbytes); 385 KASSERT(nbytes % 16 == 0); 386 387 /* Expand round keys for bitslicing. */ 388 aes_sse2_4x32_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk); 389 390 /* Load tweak. */ 391 t[0] = _mm_loadu_epi8(tweak); 392 393 /* Handle the first block separately if odd number. */ 394 if (nbytes % (8*16)) { 395 /* Load up the tweaked inputs. */ 396 for (i = 0; i < (nbytes/16) % 8; i++) { 397 q[i] = _mm_loadu_epi8(in + 16*i) ^ t[i]; 398 t[i + 1] = aes_sse2_4x32_xts_update(t[i]); 399 } 400 for (; i < 8; i++) 401 q[i] = _mm_setzero_si128(); 402 403 /* Encrypt up to seven blocks. */ 404 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 405 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 406 aes_sse2_4x32_ortho(q); 407 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 408 aes_sse2_4x32_ortho(q); 409 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 410 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 411 412 /* Store the tweaked outputs. */ 413 for (i = 0; i < (nbytes/16) % 8; i++) 414 _mm_storeu_epi8(out + 16*i, q[i] ^ t[i]); 415 416 /* Advance to the next block. */ 417 t[0] = t[i]; 418 in += nbytes % (8*16); 419 out += nbytes % (8*16); 420 nbytes -= nbytes % (8*16); 421 if (nbytes == 0) 422 goto out; 423 } 424 425 do { 426 KASSERT(nbytes % 128 == 0); 427 KASSERT(nbytes >= 128); 428 429 /* Load up the tweaked inputs. */ 430 for (i = 0; i < 8; i++) { 431 q[i] = _mm_loadu_epi8(in + 16*i) ^ t[i]; 432 t[i + 1] = aes_sse2_4x32_xts_update(t[i]); 433 } 434 435 /* Encrypt eight blocks. */ 436 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 437 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 438 aes_sse2_4x32_ortho(q); 439 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 440 aes_sse2_4x32_ortho(q); 441 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 442 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 443 444 /* Store the tweaked outputs. */ 445 for (i = 0; i < 8; i++) 446 _mm_storeu_epi8(out + 16*i, q[i] ^ t[i]); 447 448 /* Advance to the next block. */ 449 t[0] = t[8]; 450 in += 128; 451 out += 128; 452 nbytes -= 128; 453 } while (nbytes); 454 455 out: /* Store the updated tweak. */ 456 _mm_storeu_epi8(tweak, t[0]); 457 458 /* Paranoia: Zero temporary buffers. */ 459 explicit_memset(sk_exp, 0, sizeof sk_exp); 460 explicit_memset(q, 0, sizeof q); 461 explicit_memset(t, 0, sizeof t); 462 } 463 464 void 465 aes_sse2_4x32_xts_dec(const struct aesdec *dec, const uint8_t in[static 16], 466 uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16], 467 uint32_t nrounds) 468 { 469 uint32_t sk_exp[120]; 470 __m128i q[8]; 471 __m128i t[9]; 472 unsigned i; 473 474 KASSERT(nbytes); 475 KASSERT(nbytes % 16 == 0); 476 477 /* Expand round keys for bitslicing. */ 478 aes_sse2_4x32_skey_expand(sk_exp, nrounds, dec->aesd_aes.aes_rk); 479 480 /* Load tweak. */ 481 t[0] = _mm_loadu_epi8(tweak); 482 483 /* Handle the first block separately if odd number. */ 484 if (nbytes % (8*16)) { 485 /* Load up the tweaked inputs. */ 486 for (i = 0; i < (nbytes/16) % 8; i++) { 487 q[i] = _mm_loadu_epi8(in + 16*i) ^ t[i]; 488 t[i + 1] = aes_sse2_4x32_xts_update(t[i]); 489 } 490 for (; i < 8; i++) 491 q[i] = _mm_setzero_si128(); 492 493 /* Decrypt up to seven blocks. */ 494 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 495 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 496 aes_sse2_4x32_ortho(q); 497 aes_sse2_4x32_bitslice_decrypt(nrounds, sk_exp, q); 498 aes_sse2_4x32_ortho(q); 499 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 500 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 501 502 /* Store the tweaked outputs. */ 503 for (i = 0; i < (nbytes/16) % 8; i++) 504 _mm_storeu_epi8(out + 16*i, q[i] ^ t[i]); 505 506 /* Advance to the next block. */ 507 t[0] = t[i]; 508 in += nbytes % (8*16); 509 out += nbytes % (8*16); 510 nbytes -= nbytes % (8*16); 511 if (nbytes == 0) 512 goto out; 513 } 514 515 do { 516 KASSERT(nbytes % 128 == 0); 517 KASSERT(nbytes >= 128); 518 519 /* Load up the tweaked inputs. */ 520 for (i = 0; i < 8; i++) { 521 q[i] = _mm_loadu_epi8(in + 16*i) ^ t[i]; 522 t[i + 1] = aes_sse2_4x32_xts_update(t[i]); 523 } 524 525 /* Decrypt eight blocks. */ 526 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 527 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 528 aes_sse2_4x32_ortho(q); 529 aes_sse2_4x32_bitslice_decrypt(nrounds, sk_exp, q); 530 aes_sse2_4x32_ortho(q); 531 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 532 _MM_TRANSPOSE4_EPI32(q[1], q[3], q[5], q[7]); 533 534 /* Store the tweaked outputs. */ 535 for (i = 0; i < 8; i++) 536 _mm_storeu_epi8(out + 16*i, q[i] ^ t[i]); 537 538 /* Advance to the next block. */ 539 t[0] = t[8]; 540 in += 128; 541 out += 128; 542 nbytes -= 128; 543 } while (nbytes); 544 545 out: /* Store the updated tweak. */ 546 _mm_storeu_epi8(tweak, t[0]); 547 548 /* Paranoia: Zero temporary buffers. */ 549 explicit_memset(sk_exp, 0, sizeof sk_exp); 550 explicit_memset(q, 0, sizeof q); 551 explicit_memset(t, 0, sizeof t); 552 } 553 554 void 555 aes_sse2_4x32_cbcmac_update1(const struct aesenc *enc, 556 const uint8_t in[static 16], size_t nbytes, 557 uint8_t auth[static 16], uint32_t nrounds) 558 { 559 uint32_t sk_exp[120]; 560 __m128i q[8]; 561 562 KASSERT(nbytes); 563 KASSERT(nbytes % 16 == 0); 564 565 /* Expand round keys for bitslicing. */ 566 aes_sse2_4x32_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk); 567 568 /* Initialize garbage blocks. */ 569 q[1] = q[2] = q[3] = q[4] = q[5] = q[6] = q[7] = _mm_setzero_si128(); 570 571 /* Load initial authenticator. */ 572 q[0] = _mm_loadu_epi8(auth); 573 574 for (; nbytes; nbytes -= 16, in += 16) { 575 /* Combine input block. */ 576 q[0] ^= _mm_loadu_epi8(in); 577 578 /* Transform to bitslice, encrypt, transform from bitslice. */ 579 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 580 aes_sse2_4x32_ortho(q); 581 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 582 aes_sse2_4x32_ortho(q); 583 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 584 } 585 586 /* Store updated authenticator. */ 587 _mm_storeu_epi8(auth, q[0]); 588 589 /* Paranoia: Zero temporary buffers. */ 590 explicit_memset(sk_exp, 0, sizeof sk_exp); 591 explicit_memset(q, 0, sizeof q); 592 } 593 594 void 595 aes_sse2_4x32_ccm_enc1(const struct aesenc *enc, 596 const uint8_t in[static 16], uint8_t out[static 16], size_t nbytes, 597 uint8_t authctr[static 32], uint32_t nrounds) 598 { 599 uint32_t sk_exp[120]; 600 __m128i q[8]; 601 uint32_t c0, c1, c2, c3be; 602 603 KASSERT(nbytes); 604 KASSERT(nbytes % 16 == 0); 605 606 /* Expand round keys for bitslicing. */ 607 aes_sse2_4x32_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk); 608 609 /* Set first block to authenticator. */ 610 q[0] = _mm_loadu_epi8(authctr); 611 612 /* Load initial counter block, big-endian so we can increment it. */ 613 c0 = le32dec(authctr + 16 + 4*0); 614 c1 = le32dec(authctr + 16 + 4*1); 615 c2 = le32dec(authctr + 16 + 4*2); 616 c3be = bswap32(le32dec(authctr + 16 + 4*3)); 617 618 /* Set other blocks to garbage -- can't take advantage. */ 619 q[1] = q[3] = q[4] = q[5] = q[6] = q[7] = _mm_setzero_si128(); 620 621 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 622 /* Update authenticator. */ 623 q[0] ^= _mm_loadu_epi8(in); 624 625 /* Increment 32-bit counter. */ 626 q[2] = _mm_set_epi32(bswap32(++c3be), c2, c1, c0); 627 628 /* Encrypt authenticator and counter. */ 629 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 630 aes_sse2_4x32_ortho(q); 631 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 632 aes_sse2_4x32_ortho(q); 633 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 634 635 /* Encrypt with CTR output. */ 636 _mm_storeu_epi8(out, _mm_loadu_epi8(in) ^ q[2]); 637 } 638 639 /* Update authenticator. */ 640 _mm_storeu_epi8(authctr, q[0]); 641 642 /* Update counter. */ 643 le32enc(authctr + 16 + 4*3, bswap32(c3be)); 644 645 /* Paranoia: Zero temporary buffers. */ 646 explicit_memset(sk_exp, 0, sizeof sk_exp); 647 explicit_memset(q, 0, sizeof q); 648 } 649 650 void 651 aes_sse2_4x32_ccm_dec1(const struct aesenc *enc, 652 const uint8_t in[static 16], uint8_t out[static 16], size_t nbytes, 653 uint8_t authctr[static 32], uint32_t nrounds) 654 { 655 uint32_t sk_exp[120]; 656 __m128i q[8]; 657 uint32_t c0, c1, c2, c3be; 658 __m128i b; 659 660 KASSERT(nbytes); 661 KASSERT(nbytes % 16 == 0); 662 663 /* Expand round keys for bitslicing. */ 664 aes_sse2_4x32_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk); 665 666 /* Load initial counter block, big-endian so we can increment it. */ 667 c0 = le32dec(authctr + 16 + 4*0); 668 c1 = le32dec(authctr + 16 + 4*1); 669 c2 = le32dec(authctr + 16 + 4*2); 670 c3be = bswap32(le32dec(authctr + 16 + 4*3)); 671 672 /* Increment 32-bit counter. */ 673 q[0] = _mm_set_epi32(bswap32(++c3be), c2, c1, c0); 674 675 /* 676 * Set the other blocks to garbage -- we don't have any 677 * plaintext to authenticate yet. 678 */ 679 q[1] = q[3] = q[4] = q[5] = q[6] = q[7] = _mm_setzero_si128(); 680 681 /* Encrypt first CTR. */ 682 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 683 aes_sse2_4x32_ortho(q); 684 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 685 aes_sse2_4x32_ortho(q); 686 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 687 688 /* Load the initial authenticator. */ 689 q[2] = _mm_loadu_epi8(authctr); 690 691 for (;; in += 16, out += 16) { 692 /* Decrypt the block. */ 693 b = _mm_loadu_epi8(in) ^ q[0]; 694 695 /* Update authenticator. */ 696 q[2] ^= b; 697 698 /* Store plaintext. */ 699 _mm_storeu_epi8(out, b); 700 701 /* If this is the last block, stop. */ 702 if ((nbytes -= 16) == 0) 703 break; 704 705 /* Increment 32-bit counter. */ 706 q[0] = _mm_set_epi32(bswap32(++c3be), c2, c1, c0); 707 708 /* Authenticate previous plaintext, encrypt next CTR. */ 709 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 710 aes_sse2_4x32_ortho(q); 711 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 712 aes_sse2_4x32_ortho(q); 713 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 714 } 715 716 /* 717 * Authenticate last plaintext. We're only doing this for the 718 * authenticator, not for the counter, so don't bother to 719 * initialize q[0]. (Even for the sake of sanitizers, they're 720 * already initialized to something by now.) 721 */ 722 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 723 aes_sse2_4x32_ortho(q); 724 aes_sse2_4x32_bitslice_encrypt(nrounds, sk_exp, q); 725 aes_sse2_4x32_ortho(q); 726 _MM_TRANSPOSE4_EPI32(q[0], q[2], q[4], q[6]); 727 728 /* Update authenticator. */ 729 _mm_storeu_epi8(authctr, q[2]); 730 731 /* Update counter. */ 732 le32enc(authctr + 16 + 4*3, bswap32(c3be)); 733 734 /* Paranoia: Zero temporary buffers. */ 735 explicit_memset(sk_exp, 0, sizeof sk_exp); 736 explicit_memset(q, 0, sizeof q); 737 } 738 739 int 740 aes_sse2_4x32_selftest(void) 741 { 742 743 if (aes_sse2_4x32_xts_update_selftest()) 744 return -1; 745 746 /* XXX test aes_sse2_4x32_bitslice_decrypt */ 747 /* XXX test aes_sse2_4x32_bitslice_encrypt */ 748 /* XXX test aes_sse2_4x32_keysched */ 749 /* XXX test aes_sse2_4x32_ortho */ 750 /* XXX test aes_sse2_4x32_skey_expand */ 751 752 return 0; 753 } 754
Indexes created Sun Oct 25 00:26:45 UTC 2026