1 /* $OpenBSD: via.c,v 1.8 2006/11/17 07:47:56 tom Exp $ */ 2 /* $NetBSD: via_padlock.c,v 1.38 2026/04/29 16:29:59 kre Exp $ */ 3 4 /*- 5 * Copyright (c) 2003 Jason Wright 6 * Copyright (c) 2003, 2004 Theo de Raadt 7 * All rights reserved. 8 * 9 * Permission to use, copy, modify, and distribute this software for any 10 * purpose with or without fee is hereby granted, provided that the above 11 * copyright notice and this permission notice appear in all copies. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 20 */ 21 22 #include <sys/cdefs.h> 23 __KERNEL_RCSID(0, "$NetBSD: via_padlock.c,v 1.38 2026/04/29 16:29:59 kre Exp $"); 24 25 #include <sys/param.h> 26 #include <sys/systm.h> 27 #include <sys/signalvar.h> 28 #include <sys/kernel.h> 29 #include <sys/device.h> 30 #include <sys/module.h> 31 #include <sys/malloc.h> 32 #include <sys/mbuf.h> 33 #include <sys/cpu.h> 34 35 #include <x86/specialreg.h> 36 37 #include <machine/cpufunc.h> 38 #include <machine/cpuvar.h> 39 40 #include <crypto/aes/aes_keysched.h> 41 42 #include <opencrypto/cryptodev.h> 43 #include <opencrypto/cryptosoft.h> 44 #include <opencrypto/xform.h> 45 46 #include <opencrypto/cryptosoft_xform.c> 47 48 #include <x86/via_padlock.h> 49 50 static int via_padlock_match(device_t, cfdata_t, void *); 51 static void via_padlock_attach(device_t, device_t, void *); 52 static int via_padlock_detach(device_t, int); 53 static void via_padlock_attach_intr(device_t); 54 55 CFATTACH_DECL_NEW( 56 padlock, 57 sizeof(struct via_padlock_softc), 58 via_padlock_match, 59 via_padlock_attach, 60 via_padlock_detach, 61 NULL 62 ); 63 64 int via_padlock_crypto_newsession(void *, uint32_t *, struct cryptoini *); 65 int via_padlock_crypto_process(void *, struct cryptop *, int); 66 int via_padlock_crypto_swauth(struct cryptop *, struct cryptodesc *, 67 struct swcr_data *, void *); 68 int via_padlock_crypto_encdec(struct cryptop *, struct cryptodesc *, 69 struct via_padlock_session *, struct via_padlock_softc *, void *); 70 void via_padlock_crypto_freesession(void *, uint64_t); 71 static __inline void via_padlock_cbc(void *, void *, void *, void *, int, 72 void *); 73 74 static void 75 via_c3_ace_init(struct via_padlock_softc *sc) 76 { 77 /* 78 * There is no reason to call into the kernel to use this 79 * driver from userspace, because the crypto instructions can 80 * be directly accessed there. Setting CRYPTOCAP_F_SOFTWARE 81 * has approximately the right semantics though the name is 82 * confusing (however, consider that crypto via unprivileged 83 * instructions _is_ "just software" in some sense). 84 */ 85 sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_SOFTWARE); 86 if (sc->sc_cid < 0) { 87 aprint_error_dev(sc->sc_dev, 88 "could not get a crypto driver ID\n"); 89 return; 90 } 91 92 sc->sc_cid_attached = true; 93 94 /* 95 * Ask the opencrypto subsystem to register ourselves. Although 96 * we don't support hardware offloading for various HMAC algorithms, 97 * we will handle them, because opencrypto prefers drivers that 98 * support all requested algorithms. 99 * 100 * 101 * XXX We should actually implement the HMAC modes this hardware 102 * XXX can accelerate (wrap its plain SHA1/SHA2 as HMAC) and 103 * XXX strongly consider removing those passed through to cryptosoft. 104 * XXX As it stands, we can "steal" sessions from drivers which could 105 * XXX better accelerate them. 106 * 107 * XXX Note the ordering dependency between when this (or any 108 * XXX crypto driver) attaches and when cryptosoft does. We are 109 * XXX basically counting on the swcrypto pseudo-device to just 110 * XXX happen to attach last, or _it_ will steal every session 111 * XXX from _us_! 112 */ 113 #define REGISTER(alg) \ 114 crypto_register(sc->sc_cid, alg, 0, 0, \ 115 via_padlock_crypto_newsession, via_padlock_crypto_freesession, \ 116 via_padlock_crypto_process, sc); 117 118 REGISTER(CRYPTO_AES_CBC); 119 REGISTER(CRYPTO_MD5_HMAC_96); 120 REGISTER(CRYPTO_MD5_HMAC); 121 REGISTER(CRYPTO_SHA1_HMAC_96); 122 REGISTER(CRYPTO_SHA1_HMAC); 123 REGISTER(CRYPTO_RIPEMD160_HMAC_96); 124 REGISTER(CRYPTO_RIPEMD160_HMAC); 125 REGISTER(CRYPTO_SHA2_HMAC); 126 } 127 128 int 129 via_padlock_crypto_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri) 130 { 131 struct cryptoini *c; 132 struct via_padlock_softc *sc = arg; 133 struct via_padlock_session *ses = NULL; 134 const struct swcr_auth_hash *axf; 135 struct swcr_data *swd; 136 int sesn, cw0; 137 unsigned int i; 138 139 if (sc->sc_sessions == NULL) { 140 ses = sc->sc_sessions = malloc(sizeof(*ses), M_DEVBUF, 141 M_NOWAIT); 142 if (ses == NULL) 143 return (ENOMEM); 144 sesn = 0; 145 sc->sc_nsessions = 1; 146 } else { 147 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 148 if (sc->sc_sessions[sesn].ses_used == 0) { 149 ses = &sc->sc_sessions[sesn]; 150 break; 151 } 152 } 153 154 if (ses == NULL) { 155 sesn = sc->sc_nsessions; 156 ses = malloc((sesn + 1) * sizeof(*ses), M_DEVBUF, 157 M_NOWAIT); 158 if (ses == NULL) 159 return (ENOMEM); 160 memcpy(ses, sc->sc_sessions, sesn * sizeof(*ses)); 161 memset(sc->sc_sessions, 0, sesn * sizeof(*ses)); 162 free(sc->sc_sessions, M_DEVBUF); 163 sc->sc_sessions = ses; 164 ses = &sc->sc_sessions[sesn]; 165 sc->sc_nsessions++; 166 } 167 } 168 169 memset(ses, 0, sizeof(*ses)); 170 ses->ses_used = 1; 171 172 for (c = cri; c != NULL; c = c->cri_next) { 173 switch (c->cri_alg) { 174 case CRYPTO_AES_CBC: 175 memset(ses->ses_ekey, 0, sizeof(ses->ses_ekey)); 176 memset(ses->ses_dkey, 0, sizeof(ses->ses_dkey)); 177 178 switch (c->cri_klen) { 179 case 128: 180 /* 181 * XXX Is this needed? For AES-128 the 182 * VIA padlock instructions usually 183 * compute the key schedule internally. 184 */ 185 aes_keysched_enc(ses->ses_ekey, 186 c->cri_key, 16); 187 aes_keysched_dec(ses->ses_dkey, 188 c->cri_key, 16); 189 cw0 = C3_CRYPT_CWLO_KEY128; 190 break; 191 case 192: 192 aes_keysched_enc(ses->ses_ekey, 193 c->cri_key, 24); 194 aes_keysched_dec(ses->ses_dkey, 195 c->cri_key, 24); 196 cw0 = C3_CRYPT_CWLO_KEY192; 197 break; 198 case 256: 199 aes_keysched_enc(ses->ses_ekey, 200 c->cri_key, 32); 201 aes_keysched_dec(ses->ses_dkey, 202 c->cri_key, 32); 203 cw0 = C3_CRYPT_CWLO_KEY256; 204 break; 205 default: 206 return (EINVAL); 207 } 208 cw0 |= C3_CRYPT_CWLO_ALG_AES | 209 C3_CRYPT_CWLO_KEYGEN_SW | 210 C3_CRYPT_CWLO_NORMAL; 211 212 ses->ses_klen = c->cri_klen; 213 ses->ses_cw0 = cw0; 214 215 /* Convert words to host byte order (???) */ 216 for (i = 0; i < 4*(AES_256_NROUNDS + 1); i++) { 217 ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]); 218 ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]); 219 } 220 break; 221 222 /* Use hashing implementations from the cryptosoft code. */ 223 case CRYPTO_MD5_HMAC: 224 axf = &swcr_auth_hash_hmac_md5; 225 goto authcommon; 226 case CRYPTO_MD5_HMAC_96: 227 axf = &swcr_auth_hash_hmac_md5_96; 228 goto authcommon; 229 case CRYPTO_SHA1_HMAC: 230 axf = &swcr_auth_hash_hmac_sha1; 231 goto authcommon; 232 case CRYPTO_SHA1_HMAC_96: 233 axf = &swcr_auth_hash_hmac_sha1_96; 234 goto authcommon; 235 case CRYPTO_RIPEMD160_HMAC: 236 axf = &swcr_auth_hash_hmac_ripemd_160; 237 goto authcommon; 238 case CRYPTO_RIPEMD160_HMAC_96: 239 axf = &swcr_auth_hash_hmac_ripemd_160_96; 240 goto authcommon; 241 case CRYPTO_SHA2_HMAC: 242 if (cri->cri_klen == 256) 243 axf = &swcr_auth_hash_hmac_sha2_256; 244 else if (cri->cri_klen == 384) 245 axf = &swcr_auth_hash_hmac_sha2_384; 246 else if (cri->cri_klen == 512) 247 axf = &swcr_auth_hash_hmac_sha2_512; 248 else { 249 return EINVAL; 250 } 251 authcommon: 252 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 253 M_NOWAIT|M_ZERO); 254 if (swd == NULL) { 255 via_padlock_crypto_freesession(sc, sesn); 256 return (ENOMEM); 257 } 258 ses->swd = swd; 259 260 swd->sw_ictx = malloc(axf->ctxsize, 261 M_CRYPTO_DATA, M_NOWAIT); 262 if (swd->sw_ictx == NULL) { 263 via_padlock_crypto_freesession(sc, sesn); 264 return (ENOMEM); 265 } 266 267 swd->sw_octx = malloc(axf->ctxsize, 268 M_CRYPTO_DATA, M_NOWAIT); 269 if (swd->sw_octx == NULL) { 270 via_padlock_crypto_freesession(sc, sesn); 271 return (ENOMEM); 272 } 273 274 for (i = 0; i < c->cri_klen / 8; i++) 275 c->cri_key[i] ^= HMAC_IPAD_VAL; 276 277 axf->Init(swd->sw_ictx); 278 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 279 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 280 HMAC_BLOCK_LEN - (c->cri_klen / 8)); 281 282 for (i = 0; i < c->cri_klen / 8; i++) 283 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 284 HMAC_OPAD_VAL); 285 286 axf->Init(swd->sw_octx); 287 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 288 axf->Update(swd->sw_octx, hmac_opad_buffer, 289 HMAC_BLOCK_LEN - (c->cri_klen / 8)); 290 291 for (i = 0; i < c->cri_klen / 8; i++) 292 c->cri_key[i] ^= HMAC_OPAD_VAL; 293 294 swd->sw_axf = axf; 295 swd->sw_alg = c->cri_alg; 296 297 break; 298 default: 299 return (EINVAL); 300 } 301 } 302 303 *sidp = VIAC3_SID(0, sesn); 304 return (0); 305 } 306 307 void 308 via_padlock_crypto_freesession(void *arg, uint64_t tid) 309 { 310 struct via_padlock_softc *sc = arg; 311 struct swcr_data *swd; 312 const struct swcr_auth_hash *axf; 313 int sesn; 314 uint32_t sid = ((uint32_t)tid) & 0xffffffff; 315 316 sesn = VIAC3_SESSION(sid); 317 KASSERTMSG(sesn >= 0, "sesn=%d", sesn); 318 KASSERTMSG(sesn < sc->sc_nsessions, "sesn=%d nsessions=%d", 319 sesn, sc->sc_nsessions); 320 321 if (sc->sc_sessions[sesn].swd) { 322 swd = sc->sc_sessions[sesn].swd; 323 axf = swd->sw_axf; 324 325 if (swd->sw_ictx) { 326 memset(swd->sw_ictx, 0, axf->ctxsize); 327 free(swd->sw_ictx, M_CRYPTO_DATA); 328 } 329 if (swd->sw_octx) { 330 memset(swd->sw_octx, 0, axf->ctxsize); 331 free(swd->sw_octx, M_CRYPTO_DATA); 332 } 333 free(swd, M_CRYPTO_DATA); 334 } 335 336 memset(&sc->sc_sessions[sesn], 0, sizeof(sc->sc_sessions[sesn])); 337 } 338 339 static __inline void 340 via_padlock_cbc(void *cw, void *src, void *dst, void *key, int rep, 341 void *iv) 342 { 343 unsigned int cr0; 344 int s; 345 346 s = splhigh(); 347 348 cr0 = rcr0(); /* Permit access to SIMD/FPU path */ 349 lcr0(cr0 & ~(CR0_EM|CR0_TS)); 350 351 /* Do the deed */ 352 __asm __volatile("pushf; popf"); /* force key reload */ 353 __asm __volatile(".byte 0xf3, 0x0f, 0xa7, 0xd0" : /* rep xcrypt-cbc */ 354 : "a" (iv), "b" (key), "c" (rep), "d" (cw), "S" (src), "D" (dst) 355 : "memory", "cc"); 356 357 lcr0(cr0); 358 359 splx(s); 360 } 361 362 int 363 via_padlock_crypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 364 struct swcr_data *sw, void *buf) 365 { 366 int type; 367 368 if (crp->crp_flags & CRYPTO_F_IMBUF) 369 type = CRYPTO_BUF_MBUF; 370 else 371 type= CRYPTO_BUF_IOV; 372 373 return (swcr_authcompute(crp, crd, sw, buf, type)); 374 } 375 376 int 377 via_padlock_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd, 378 struct via_padlock_session *ses, struct via_padlock_softc *sc, void *buf) 379 { 380 uint32_t *key; 381 382 if ((crd->crd_len % 16) != 0) 383 return (EINVAL); 384 385 sc->op_buf = malloc(crd->crd_len, M_DEVBUF, M_NOWAIT); 386 if (sc->op_buf == NULL) 387 return (ENOMEM); 388 389 if (crd->crd_flags & CRD_F_ENCRYPT) { 390 sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_ENCRYPT; 391 key = ses->ses_ekey; 392 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 393 memcpy(sc->op_iv, crd->crd_iv, 16); 394 else 395 cprng_fast(sc->op_iv, 16); 396 397 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 398 if (crp->crp_flags & CRYPTO_F_IMBUF) 399 m_copyback((struct mbuf *)crp->crp_buf, 400 crd->crd_inject, 16, sc->op_iv); 401 else if (crp->crp_flags & CRYPTO_F_IOV) 402 cuio_copyback((struct uio *)crp->crp_buf, 403 crd->crd_inject, 16, sc->op_iv); 404 else 405 memcpy((char *)crp->crp_buf + crd->crd_inject, 406 sc->op_iv, 16); 407 } 408 } else { 409 sc->op_cw[0] = ses->ses_cw0 | C3_CRYPT_CWLO_DECRYPT; 410 key = ses->ses_dkey; 411 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 412 memcpy(sc->op_iv, crd->crd_iv, 16); 413 else { 414 if (crp->crp_flags & CRYPTO_F_IMBUF) 415 m_copydata((struct mbuf *)crp->crp_buf, 416 crd->crd_inject, 16, sc->op_iv); 417 else if (crp->crp_flags & CRYPTO_F_IOV) 418 cuio_copydata((struct uio *)crp->crp_buf, 419 crd->crd_inject, 16, sc->op_iv); 420 else 421 memcpy(sc->op_iv, (char *)crp->crp_buf + 422 crd->crd_inject, 16); 423 } 424 } 425 426 if (crp->crp_flags & CRYPTO_F_IMBUF) 427 m_copydata((struct mbuf *)crp->crp_buf, 428 crd->crd_skip, crd->crd_len, sc->op_buf); 429 else if (crp->crp_flags & CRYPTO_F_IOV) 430 cuio_copydata((struct uio *)crp->crp_buf, 431 crd->crd_skip, crd->crd_len, sc->op_buf); 432 else 433 memcpy(sc->op_buf, (char *)crp->crp_buf + crd->crd_skip, 434 crd->crd_len); 435 436 sc->op_cw[1] = sc->op_cw[2] = sc->op_cw[3] = 0; 437 via_padlock_cbc(&sc->op_cw, sc->op_buf, sc->op_buf, key, 438 crd->crd_len / 16, sc->op_iv); 439 440 if (crp->crp_flags & CRYPTO_F_IMBUF) 441 m_copyback((struct mbuf *)crp->crp_buf, 442 crd->crd_skip, crd->crd_len, sc->op_buf); 443 else if (crp->crp_flags & CRYPTO_F_IOV) 444 cuio_copyback((struct uio *)crp->crp_buf, 445 crd->crd_skip, crd->crd_len, sc->op_buf); 446 else 447 memcpy((char *)crp->crp_buf + crd->crd_skip, sc->op_buf, 448 crd->crd_len); 449 450 if (sc->op_buf != NULL) { 451 memset(sc->op_buf, 0, crd->crd_len); 452 free(sc->op_buf, M_DEVBUF); 453 sc->op_buf = NULL; 454 } 455 456 return 0; 457 } 458 459 int 460 via_padlock_crypto_process(void *arg, struct cryptop *crp, int hint) 461 { 462 struct via_padlock_softc *sc = arg; 463 struct via_padlock_session *ses; 464 struct cryptodesc *crd; 465 int sesn, err = 0; 466 467 KASSERT(sc != NULL /*, ("via_padlock_crypto_process: null softc")*/); 468 if (crp == NULL || crp->crp_callback == NULL) { 469 err = EINVAL; 470 goto out; 471 } 472 473 sesn = VIAC3_SESSION(crp->crp_sid); 474 if (sesn >= sc->sc_nsessions) { 475 err = EINVAL; 476 goto out; 477 } 478 ses = &sc->sc_sessions[sesn]; 479 480 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 481 switch (crd->crd_alg) { 482 case CRYPTO_AES_CBC: 483 if ((err = via_padlock_crypto_encdec(crp, crd, ses, 484 sc, crp->crp_buf)) != 0) 485 goto out; 486 break; 487 488 case CRYPTO_MD5_HMAC: 489 case CRYPTO_SHA1_HMAC: 490 case CRYPTO_RIPEMD160_HMAC: 491 case CRYPTO_SHA2_HMAC: 492 if ((err = via_padlock_crypto_swauth(crp, crd, 493 ses->swd, crp->crp_buf)) != 0) 494 goto out; 495 break; 496 497 default: 498 err = EINVAL; 499 goto out; 500 } 501 } 502 out: 503 crp->crp_etype = err; 504 crypto_done(crp); 505 return 0; 506 } 507 508 static int 509 via_padlock_match(device_t parent, cfdata_t cf, void *opaque) 510 { 511 struct cpufeature_attach_args *cfaa = opaque; 512 struct cpu_info *ci = cfaa->ci; 513 514 if (strcmp(cfaa->name, "padlock") != 0) 515 return 0; 516 if ((cpu_feature[4] & (CPUID_VIA_HAS_ACE|CPUID_VIA_HAS_RNG)) == 0) 517 return 0; 518 if ((ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) == 0) 519 return 0; 520 return 1; 521 } 522 523 static void 524 via_padlock_attach(device_t parent, device_t self, void *opaque) 525 { 526 struct via_padlock_softc *sc = device_private(self); 527 528 sc->sc_dev = self; 529 530 aprint_naive("\n"); 531 aprint_normal(": VIA PadLock\n"); 532 533 pmf_device_register(self, NULL, NULL); 534 535 config_interrupts(self, via_padlock_attach_intr); 536 } 537 538 static void 539 via_padlock_attach_intr(device_t self) 540 { 541 struct via_padlock_softc *sc = device_private(self); 542 543 aprint_normal("%s:", device_xname(self)); 544 if (cpu_feature[4] & CPUID_VIA_HAS_RNG) { 545 aprint_normal(" RNG"); 546 } 547 if (cpu_feature[4] & CPUID_VIA_HAS_ACE) { 548 via_c3_ace_init(sc); 549 aprint_normal(" ACE"); 550 } 551 aprint_normal("\n"); 552 } 553 554 static int 555 via_padlock_detach(device_t self, int flags) 556 { 557 struct via_padlock_softc *sc = device_private(self); 558 559 if (sc->sc_cid_attached) { 560 crypto_unregister(sc->sc_cid, CRYPTO_AES_CBC); 561 crypto_unregister(sc->sc_cid, CRYPTO_MD5_HMAC_96); 562 crypto_unregister(sc->sc_cid, CRYPTO_MD5_HMAC); 563 crypto_unregister(sc->sc_cid, CRYPTO_SHA1_HMAC_96); 564 crypto_unregister(sc->sc_cid, CRYPTO_SHA1_HMAC); 565 crypto_unregister(sc->sc_cid, CRYPTO_RIPEMD160_HMAC_96); 566 crypto_unregister(sc->sc_cid, CRYPTO_RIPEMD160_HMAC); 567 crypto_unregister(sc->sc_cid, CRYPTO_SHA2_HMAC); 568 sc->sc_cid_attached = false; 569 } 570 571 pmf_device_deregister(self); 572 573 return 0; 574 } 575 576 MODULE(MODULE_CLASS_DRIVER, padlock, NULL); 577 578 #ifdef _MODULE 579 #include "ioconf.c" 580 #endif 581 582 static int 583 padlock_modcmd(modcmd_t cmd, void *opaque) 584 { 585 int error = 0; 586 587 switch (cmd) { 588 case MODULE_CMD_INIT: 589 #ifdef _MODULE 590 error = config_init_component(cfdriver_ioconf_padlock, 591 cfattach_ioconf_padlock, cfdata_ioconf_padlock); 592 #endif 593 return error; 594 case MODULE_CMD_FINI: 595 #ifdef _MODULE 596 error = config_fini_component(cfdriver_ioconf_padlock, 597 cfattach_ioconf_padlock, cfdata_ioconf_padlock); 598 #endif 599 return error; 600 default: 601 return ENOTTY; 602 } 603 } 604
Indexes created Sun Oct 25 00:26:45 UTC 2026