aes_via.c revision 1.1
1 /* $NetBSD: aes_via.c,v 1.1 2020/06/29 23:39:30 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(1, "$NetBSD: aes_via.c,v 1.1 2020/06/29 23:39:30 riastradh Exp $"); 31 32 #include <sys/types.h> 33 #include <sys/evcnt.h> 34 #include <sys/systm.h> 35 36 #include <crypto/aes/aes.h> 37 #include <crypto/aes/aes_bear.h> 38 39 #include <x86/cpufunc.h> 40 #include <x86/cpuvar.h> 41 #include <x86/fpu.h> 42 #include <x86/specialreg.h> 43 #include <x86/via_padlock.h> 44 45 static void 46 aesvia_reload_keys(void) 47 { 48 49 asm volatile("pushf; popf"); 50 } 51 52 static uint32_t 53 aesvia_keylen_cw0(unsigned nrounds) 54 { 55 56 /* 57 * Determine the control word bits for the key size / number of 58 * rounds. For AES-128, the hardware can do key expansion on 59 * the fly; for AES-192 and AES-256, software must do it. 60 */ 61 switch (nrounds) { 62 case AES_128_NROUNDS: 63 return C3_CRYPT_CWLO_KEY128; 64 case AES_192_NROUNDS: 65 return C3_CRYPT_CWLO_KEY192 | C3_CRYPT_CWLO_KEYGEN_SW; 66 case AES_256_NROUNDS: 67 return C3_CRYPT_CWLO_KEY256 | C3_CRYPT_CWLO_KEYGEN_SW; 68 default: 69 panic("invalid AES nrounds: %u", nrounds); 70 } 71 } 72 73 static void 74 aesvia_setenckey(struct aesenc *enc, const uint8_t *key, uint32_t nrounds) 75 { 76 size_t key_len; 77 78 switch (nrounds) { 79 case AES_128_NROUNDS: 80 enc->aese_aes.aes_rk[0] = le32dec(key + 4*0); 81 enc->aese_aes.aes_rk[1] = le32dec(key + 4*1); 82 enc->aese_aes.aes_rk[2] = le32dec(key + 4*2); 83 enc->aese_aes.aes_rk[3] = le32dec(key + 4*3); 84 return; 85 case AES_192_NROUNDS: 86 key_len = 24; 87 break; 88 case AES_256_NROUNDS: 89 key_len = 32; 90 break; 91 default: 92 panic("invalid AES nrounds: %u", nrounds); 93 } 94 br_aes_ct_keysched_stdenc(enc->aese_aes.aes_rk, key, key_len); 95 } 96 97 static void 98 aesvia_setdeckey(struct aesdec *dec, const uint8_t *key, uint32_t nrounds) 99 { 100 size_t key_len; 101 102 switch (nrounds) { 103 case AES_128_NROUNDS: 104 dec->aesd_aes.aes_rk[0] = le32dec(key + 4*0); 105 dec->aesd_aes.aes_rk[1] = le32dec(key + 4*1); 106 dec->aesd_aes.aes_rk[2] = le32dec(key + 4*2); 107 dec->aesd_aes.aes_rk[3] = le32dec(key + 4*3); 108 return; 109 case AES_192_NROUNDS: 110 key_len = 24; 111 break; 112 case AES_256_NROUNDS: 113 key_len = 32; 114 break; 115 default: 116 panic("invalid AES nrounds: %u", nrounds); 117 } 118 br_aes_ct_keysched_stddec(dec->aesd_aes.aes_rk, key, key_len); 119 } 120 121 static inline void 122 aesvia_enc1(const struct aesenc *enc, const uint8_t in[static 16], 123 uint8_t out[static 16], uint32_t cw0) 124 { 125 const uint32_t cw[4] __aligned(16) = { 126 [0] = (cw0 127 | C3_CRYPT_CWLO_ALG_AES 128 | C3_CRYPT_CWLO_ENCRYPT 129 | C3_CRYPT_CWLO_NORMAL), 130 }; 131 size_t nblocks = 1; 132 133 KASSERT(((uintptr_t)enc & 0xf) == 0); 134 KASSERT(((uintptr_t)in & 0xf) == 0); 135 KASSERT(((uintptr_t)out & 0xf) == 0); 136 137 asm volatile("rep xcryptecb" 138 : "+c"(nblocks), "+S"(in), "+D"(out) 139 : "b"(enc), "d"(cw) 140 : "memory", "cc"); 141 } 142 143 static inline void 144 aesvia_dec1(const struct aesdec *dec, const uint8_t in[static 16], 145 uint8_t out[static 16], uint32_t cw0) 146 { 147 const uint32_t cw[4] __aligned(16) = { 148 [0] = (cw0 149 | C3_CRYPT_CWLO_ALG_AES 150 | C3_CRYPT_CWLO_DECRYPT 151 | C3_CRYPT_CWLO_NORMAL), 152 }; 153 size_t nblocks = 1; 154 155 KASSERT(((uintptr_t)dec & 0xf) == 0); 156 KASSERT(((uintptr_t)in & 0xf) == 0); 157 KASSERT(((uintptr_t)out & 0xf) == 0); 158 159 asm volatile("rep xcryptecb" 160 : "+c"(nblocks), "+S"(in), "+D"(out) 161 : "b"(dec), "d"(cw) 162 : "memory", "cc"); 163 } 164 165 static struct evcnt enc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 166 NULL, "aesvia", "enc aligned"); 167 EVCNT_ATTACH_STATIC(enc_aligned_evcnt); 168 static struct evcnt enc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 169 NULL, "aesvia", "dec unaligned"); 170 EVCNT_ATTACH_STATIC(enc_unaligned_evcnt); 171 172 static void 173 aesvia_enc(const struct aesenc *enc, const uint8_t in[static 16], 174 uint8_t out[static 16], uint32_t nrounds) 175 { 176 const uint32_t cw0 = aesvia_keylen_cw0(nrounds); 177 178 fpu_kern_enter(); 179 aesvia_reload_keys(); 180 if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 && 181 ((uintptr_t)in & 0xff0) != 0xff0) { 182 enc_aligned_evcnt.ev_count++; 183 aesvia_enc1(enc, in, out, cw0); 184 } else { 185 enc_unaligned_evcnt.ev_count++; 186 /* 187 * VIA requires 16-byte/128-bit alignment, and 188 * xcrypt-ecb reads one block past the one we're 189 * working on -- which may go past the end of the page 190 * into unmapped territory. Use a bounce buffer if 191 * either constraint is violated. 192 */ 193 uint8_t inbuf[16] __aligned(16); 194 uint8_t outbuf[16] __aligned(16); 195 196 memcpy(inbuf, in, 16); 197 aesvia_enc1(enc, inbuf, outbuf, cw0); 198 memcpy(out, outbuf, 16); 199 200 explicit_memset(inbuf, 0, sizeof inbuf); 201 explicit_memset(outbuf, 0, sizeof outbuf); 202 } 203 fpu_kern_leave(); 204 } 205 206 static struct evcnt dec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 207 NULL, "aesvia", "dec aligned"); 208 EVCNT_ATTACH_STATIC(dec_aligned_evcnt); 209 static struct evcnt dec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 210 NULL, "aesvia", "dec unaligned"); 211 EVCNT_ATTACH_STATIC(dec_unaligned_evcnt); 212 213 static void 214 aesvia_dec(const struct aesdec *dec, const uint8_t in[static 16], 215 uint8_t out[static 16], uint32_t nrounds) 216 { 217 const uint32_t cw0 = aesvia_keylen_cw0(nrounds); 218 219 fpu_kern_enter(); 220 aesvia_reload_keys(); 221 if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 && 222 ((uintptr_t)in & 0xff0) != 0xff0) { 223 dec_aligned_evcnt.ev_count++; 224 aesvia_dec1(dec, in, out, cw0); 225 } else { 226 dec_unaligned_evcnt.ev_count++; 227 /* 228 * VIA requires 16-byte/128-bit alignment, and 229 * xcrypt-ecb reads one block past the one we're 230 * working on -- which may go past the end of the page 231 * into unmapped territory. Use a bounce buffer if 232 * either constraint is violated. 233 */ 234 uint8_t inbuf[16] __aligned(16); 235 uint8_t outbuf[16] __aligned(16); 236 237 memcpy(inbuf, in, 16); 238 aesvia_dec1(dec, inbuf, outbuf, cw0); 239 memcpy(out, outbuf, 16); 240 241 explicit_memset(inbuf, 0, sizeof inbuf); 242 explicit_memset(outbuf, 0, sizeof outbuf); 243 } 244 fpu_kern_leave(); 245 } 246 247 static inline void 248 aesvia_cbc_enc1(const struct aesenc *enc, const uint8_t in[static 16], 249 uint8_t out[static 16], size_t nblocks, uint8_t **ivp, uint32_t cw0) 250 { 251 const uint32_t cw[4] __aligned(16) = { 252 [0] = (cw0 253 | C3_CRYPT_CWLO_ALG_AES 254 | C3_CRYPT_CWLO_ENCRYPT 255 | C3_CRYPT_CWLO_NORMAL), 256 }; 257 258 KASSERT(((uintptr_t)enc & 0xf) == 0); 259 KASSERT(((uintptr_t)in & 0xf) == 0); 260 KASSERT(((uintptr_t)out & 0xf) == 0); 261 KASSERT(((uintptr_t)*ivp & 0xf) == 0); 262 263 /* 264 * Register effects: 265 * - Counts nblocks down to zero. 266 * - Advances in by nblocks (units of blocks). 267 * - Advances out by nblocks (units of blocks). 268 * - Updates *ivp to point at the last block of out. 269 */ 270 asm volatile("rep xcryptcbc" 271 : "+c"(nblocks), "+S"(in), "+D"(out), "+a"(*ivp) 272 : "b"(enc), "d"(cw) 273 : "memory", "cc"); 274 } 275 276 static inline void 277 aesvia_cbc_dec1(const struct aesdec *dec, const uint8_t in[static 16], 278 uint8_t out[static 16], size_t nblocks, uint8_t iv[static 16], 279 uint32_t cw0) 280 { 281 const uint32_t cw[4] __aligned(16) = { 282 [0] = (cw0 283 | C3_CRYPT_CWLO_ALG_AES 284 | C3_CRYPT_CWLO_DECRYPT 285 | C3_CRYPT_CWLO_NORMAL), 286 }; 287 288 KASSERT(((uintptr_t)dec & 0xf) == 0); 289 KASSERT(((uintptr_t)in & 0xf) == 0); 290 KASSERT(((uintptr_t)out & 0xf) == 0); 291 KASSERT(((uintptr_t)iv & 0xf) == 0); 292 293 /* 294 * Register effects: 295 * - Counts nblocks down to zero. 296 * - Advances in by nblocks (units of blocks). 297 * - Advances out by nblocks (units of blocks). 298 * Memory side effects: 299 * - Writes what was the last block of in at the address iv. 300 */ 301 asm volatile("rep xcryptcbc" 302 : "+c"(nblocks), "+S"(in), "+D"(out) 303 : "a"(iv), "b"(dec), "d"(cw) 304 : "memory", "cc"); 305 } 306 307 static inline void 308 xor128(void *x, const void *a, const void *b) 309 { 310 uint32_t *x32 = x; 311 const uint32_t *a32 = a; 312 const uint32_t *b32 = b; 313 314 x32[0] = a32[0] ^ b32[0]; 315 x32[1] = a32[1] ^ b32[1]; 316 x32[2] = a32[2] ^ b32[2]; 317 x32[3] = a32[3] ^ b32[3]; 318 } 319 320 static struct evcnt cbcenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 321 NULL, "aesvia", "cbcenc aligned"); 322 EVCNT_ATTACH_STATIC(cbcenc_aligned_evcnt); 323 static struct evcnt cbcenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 324 NULL, "aesvia", "cbcenc unaligned"); 325 EVCNT_ATTACH_STATIC(cbcenc_unaligned_evcnt); 326 327 static void 328 aesvia_cbc_enc(const struct aesenc *enc, const uint8_t in[static 16], 329 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 330 uint32_t nrounds) 331 { 332 const uint32_t cw0 = aesvia_keylen_cw0(nrounds); 333 334 KASSERT(nbytes % 16 == 0); 335 if (nbytes == 0) 336 return; 337 338 fpu_kern_enter(); 339 aesvia_reload_keys(); 340 if ((((uintptr_t)in | (uintptr_t)out | (uintptr_t)iv) & 0xf) == 0) { 341 cbcenc_aligned_evcnt.ev_count++; 342 uint8_t *ivp = iv; 343 aesvia_cbc_enc1(enc, in, out, nbytes/16, &ivp, cw0); 344 memcpy(iv, ivp, 16); 345 } else { 346 cbcenc_unaligned_evcnt.ev_count++; 347 uint8_t cv[16] __aligned(16); 348 uint8_t tmp[16] __aligned(16); 349 350 memcpy(cv, iv, 16); 351 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 352 memcpy(tmp, in, 16); 353 xor128(tmp, tmp, cv); 354 aesvia_enc1(enc, tmp, cv, cw0); 355 memcpy(out, cv, 16); 356 } 357 memcpy(iv, cv, 16); 358 } 359 fpu_kern_leave(); 360 } 361 362 static struct evcnt cbcdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 363 NULL, "aesvia", "cbcdec aligned"); 364 EVCNT_ATTACH_STATIC(cbcdec_aligned_evcnt); 365 static struct evcnt cbcdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 366 NULL, "aesvia", "cbcdec unaligned"); 367 EVCNT_ATTACH_STATIC(cbcdec_unaligned_evcnt); 368 369 static void 370 aesvia_cbc_dec(const struct aesdec *dec, const uint8_t in[static 16], 371 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16], 372 uint32_t nrounds) 373 { 374 const uint32_t cw0 = aesvia_keylen_cw0(nrounds); 375 376 KASSERT(nbytes % 16 == 0); 377 if (nbytes == 0) 378 return; 379 380 fpu_kern_enter(); 381 aesvia_reload_keys(); 382 if ((((uintptr_t)in | (uintptr_t)out | (uintptr_t)iv) & 0xf) == 0) { 383 cbcdec_aligned_evcnt.ev_count++; 384 aesvia_cbc_dec1(dec, in, out, nbytes/16, iv, cw0); 385 } else { 386 cbcdec_unaligned_evcnt.ev_count++; 387 uint8_t iv0[16] __aligned(16); 388 uint8_t cv[16] __aligned(16); 389 uint8_t tmp[16] __aligned(16); 390 391 memcpy(iv0, iv, 16); 392 memcpy(cv, in + nbytes - 16, 16); 393 memcpy(iv, cv, 16); 394 395 for (;;) { 396 aesvia_dec1(dec, cv, tmp, cw0); 397 if ((nbytes -= 16) == 0) 398 break; 399 memcpy(cv, in + nbytes - 16, 16); 400 xor128(tmp, tmp, cv); 401 memcpy(out + nbytes, tmp, 16); 402 } 403 404 xor128(tmp, tmp, iv0); 405 memcpy(out, tmp, 16); 406 explicit_memset(tmp, 0, sizeof tmp); 407 } 408 fpu_kern_leave(); 409 } 410 411 static inline void 412 aesvia_xts_update(uint32_t *t0, uint32_t *t1, uint32_t *t2, uint32_t *t3) 413 { 414 uint32_t s0, s1, s2, s3; 415 416 s0 = *t0 >> 31; 417 s1 = *t1 >> 31; 418 s2 = *t2 >> 31; 419 s3 = *t3 >> 31; 420 *t0 = (*t0 << 1) ^ (-s3 & 0x87); 421 *t1 = (*t1 << 1) ^ s0; 422 *t2 = (*t2 << 1) ^ s1; 423 *t3 = (*t3 << 1) ^ s2; 424 } 425 426 static int 427 aesvia_xts_update_selftest(void) 428 { 429 static const struct { 430 uint32_t in[4], out[4]; 431 } cases[] = { 432 { {1}, {2} }, 433 { {0x80000000U,0,0,0}, {0,1,0,0} }, 434 { {0,0x80000000U,0,0}, {0,0,1,0} }, 435 { {0,0,0x80000000U,0}, {0,0,0,1} }, 436 { {0,0,0,0x80000000U}, {0x87,0,0,0} }, 437 { {0,0x80000000U,0,0x80000000U}, {0x87,0,1,0} }, 438 }; 439 unsigned i; 440 uint32_t t0, t1, t2, t3; 441 442 for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) { 443 t0 = cases[i].in[0]; 444 t1 = cases[i].in[1]; 445 t2 = cases[i].in[2]; 446 t3 = cases[i].in[3]; 447 aesvia_xts_update(&t0, &t1, &t2, &t3); 448 if (t0 != cases[i].out[0] || 449 t1 != cases[i].out[1] || 450 t2 != cases[i].out[2] || 451 t3 != cases[i].out[3]) 452 return -1; 453 } 454 455 /* Success! */ 456 return 0; 457 } 458 459 static struct evcnt xtsenc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 460 NULL, "aesvia", "xtsenc aligned"); 461 EVCNT_ATTACH_STATIC(xtsenc_aligned_evcnt); 462 static struct evcnt xtsenc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 463 NULL, "aesvia", "xtsenc unaligned"); 464 EVCNT_ATTACH_STATIC(xtsenc_unaligned_evcnt); 465 466 static void 467 aesvia_xts_enc(const struct aesenc *enc, const uint8_t in[static 16], 468 uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16], 469 uint32_t nrounds) 470 { 471 const uint32_t cw0 = aesvia_keylen_cw0(nrounds); 472 uint32_t t[4]; 473 474 KASSERT(nbytes % 16 == 0); 475 476 memcpy(t, tweak, 16); 477 478 fpu_kern_enter(); 479 aesvia_reload_keys(); 480 if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0) { 481 xtsenc_aligned_evcnt.ev_count++; 482 unsigned lastblock = 0; 483 484 /* 485 * Make sure the last block is not the last block of a 486 * page. (Note that we store the AES input in `out' as 487 * a temporary buffer, rather than reading it directly 488 * from `in', since we have to combine the tweak 489 * first.) 490 */ 491 lastblock = 16*(((uintptr_t)(out + nbytes) & 0xfff) == 0); 492 nbytes -= lastblock; 493 494 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 495 xor128(out, in, t); 496 aesvia_enc1(enc, out, out, cw0); 497 xor128(out, out, t); 498 aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); 499 } 500 501 /* Handle the last block of a page, if necessary. */ 502 if (lastblock) { 503 uint8_t buf[16] __aligned(16); 504 xor128(buf, in, t); 505 aesvia_enc1(enc, buf, out, cw0); 506 explicit_memset(buf, 0, sizeof buf); 507 } 508 } else { 509 xtsenc_unaligned_evcnt.ev_count++; 510 uint8_t buf[16] __aligned(16); 511 512 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 513 memcpy(buf, in, 16); 514 xor128(buf, buf, t); 515 aesvia_enc1(enc, buf, buf, cw0); 516 xor128(buf, buf, t); 517 memcpy(out, buf, 16); 518 aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); 519 } 520 521 explicit_memset(buf, 0, sizeof buf); 522 } 523 fpu_kern_leave(); 524 525 memcpy(tweak, t, 16); 526 explicit_memset(t, 0, sizeof t); 527 } 528 529 static struct evcnt xtsdec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 530 NULL, "aesvia", "xtsdec aligned"); 531 EVCNT_ATTACH_STATIC(xtsdec_aligned_evcnt); 532 static struct evcnt xtsdec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, 533 NULL, "aesvia", "xtsdec unaligned"); 534 EVCNT_ATTACH_STATIC(xtsdec_unaligned_evcnt); 535 536 static void 537 aesvia_xts_dec(const struct aesdec *dec, const uint8_t in[static 16], 538 uint8_t out[static 16], size_t nbytes, uint8_t tweak[static 16], 539 uint32_t nrounds) 540 { 541 const uint32_t cw0 = aesvia_keylen_cw0(nrounds); 542 uint32_t t[4]; 543 544 KASSERT(nbytes % 16 == 0); 545 546 memcpy(t, tweak, 16); 547 548 fpu_kern_enter(); 549 aesvia_reload_keys(); 550 if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0) { 551 xtsdec_aligned_evcnt.ev_count++; 552 unsigned lastblock = 0; 553 554 /* 555 * Make sure the last block is not the last block of a 556 * page. (Note that we store the AES input in `out' as 557 * a temporary buffer, rather than reading it directly 558 * from `in', since we have to combine the tweak 559 * first.) 560 */ 561 lastblock = 16*(((uintptr_t)(out + nbytes) & 0xfff) == 0); 562 nbytes -= lastblock; 563 564 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 565 xor128(out, in, t); 566 aesvia_dec1(dec, out, out, cw0); 567 xor128(out, out, t); 568 aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); 569 } 570 571 /* Handle the last block of a page, if necessary. */ 572 if (lastblock) { 573 uint8_t buf[16] __aligned(16); 574 xor128(buf, in, t); 575 aesvia_dec1(dec, buf, out, cw0); 576 explicit_memset(buf, 0, sizeof buf); 577 } 578 } else { 579 xtsdec_unaligned_evcnt.ev_count++; 580 uint8_t buf[16] __aligned(16); 581 582 for (; nbytes; nbytes -= 16, in += 16, out += 16) { 583 memcpy(buf, in, 16); 584 xor128(buf, buf, t); 585 aesvia_dec1(dec, buf, buf, cw0); 586 xor128(buf, buf, t); 587 memcpy(out, buf, 16); 588 aesvia_xts_update(&t[0], &t[1], &t[2], &t[3]); 589 } 590 591 explicit_memset(buf, 0, sizeof buf); 592 } 593 fpu_kern_leave(); 594 595 memcpy(tweak, t, 16); 596 explicit_memset(t, 0, sizeof t); 597 } 598 599 static int 600 aesvia_probe(void) 601 { 602 603 /* Verify that the CPU advertises VIA ACE support. */ 604 if ((cpu_feature[4] & CPUID_VIA_HAS_ACE) == 0) 605 return -1; 606 607 /* Verify that our XTS tweak update logic works. */ 608 if (aesvia_xts_update_selftest()) 609 return -1; 610 611 /* Success! */ 612 return 0; 613 } 614 615 struct aes_impl aes_via_impl = { 616 .ai_name = "VIA ACE", 617 .ai_probe = aesvia_probe, 618 .ai_setenckey = aesvia_setenckey, 619 .ai_setdeckey = aesvia_setdeckey, 620 .ai_enc = aesvia_enc, 621 .ai_dec = aesvia_dec, 622 .ai_cbc_enc = aesvia_cbc_enc, 623 .ai_cbc_dec = aesvia_cbc_dec, 624 .ai_xts_enc = aesvia_xts_enc, 625 .ai_xts_dec = aesvia_xts_dec, 626 }; 627
Indexes created Tue Sep 30 20:09:53 GMT 2025