1 /* $NetBSD: cryptodev.c,v 1.139 2026/09/22 14:46:27 riastradh Exp $ */ 2 /* $FreeBSD: src/sys/opencrypto/cryptodev.c,v 1.4.2.4 2003/06/03 00:09:02 sam Exp $ */ 3 /* $OpenBSD: cryptodev.c,v 1.53 2002/07/10 22:21:30 mickey Exp $ */ 4 5 /*- 6 * Copyright (c) 2008 The NetBSD Foundation, Inc. 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by Coyote Point Systems, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2001 Theo de Raadt 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. The name of the author may not be used to endorse or promote products 47 * derived from this software without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 50 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 51 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 52 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 53 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 54 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 55 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 56 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 57 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 58 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 59 * 60 * Effort sponsored in part by the Defense Advanced Research Projects 61 * Agency (DARPA) and Air Force Research Laboratory, Air Force 62 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 63 * 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: cryptodev.c,v 1.139 2026/09/22 14:46:27 riastradh Exp $"); 68 69 #include <sys/param.h> 70 71 #include <sys/atomic.h> 72 #include <sys/compat_stub.h> 73 #include <sys/conf.h> 74 #include <sys/device.h> 75 #include <sys/errno.h> 76 #include <sys/file.h> 77 #include <sys/filedesc.h> 78 #include <sys/kauth.h> 79 #include <sys/kmem.h> 80 #include <sys/malloc.h> 81 #include <sys/mbuf.h> 82 #include <sys/md5.h> 83 #include <sys/module.h> 84 #include <sys/poll.h> 85 #include <sys/pool.h> 86 #include <sys/sdt.h> 87 #include <sys/select.h> 88 #include <sys/sha1.h> 89 #include <sys/stat.h> 90 #include <sys/sysctl.h> 91 #include <sys/systm.h> 92 93 #include "ioconf.h" 94 95 #ifdef _KERNEL_OPT 96 #include "opt_ocf.h" 97 #include "opt_compat_netbsd.h" 98 #endif 99 100 #include <opencrypto/cryptodev.h> 101 #include <opencrypto/cryptodev_internal.h> 102 #include <opencrypto/ocryptodev.h> 103 #include <opencrypto/xform.h> 104 105 #include "ioconf.h" 106 107 #define CRYPTOP_IOV_MAX (256*1024 - 4) /* XXX ??? */ 108 109 static kmutex_t cryptodev_mtx; 110 111 struct csession { 112 TAILQ_ENTRY(csession) next; 113 kmutex_t lock; 114 uint64_t sid; 115 uint32_t ses; 116 volatile uint32_t refcnt; 117 118 uint32_t cipher; /* note: shares name space in crd_alg */ 119 const struct enc_xform *txform; 120 uint32_t mac; /* note: shares name space in crd_alg */ 121 const struct auth_hash *thash; 122 uint32_t comp_alg; /* note: shares name space in crd_alg */ 123 const struct comp_algo *tcomp; 124 125 void * key; 126 int keylen; 127 128 void * mackey; 129 int mackeylen; 130 }; 131 132 struct fcrypt { 133 TAILQ_HEAD(csessionlist, csession) csessions; 134 TAILQ_HEAD(crprethead, cryptop) crp_ret_mq; 135 TAILQ_HEAD(krprethead, cryptkop) crp_ret_mkq; 136 uint32_t sesn; 137 struct selinfo sinfo; 138 uint32_t requestid; 139 struct timespec atime; 140 struct timespec mtime; 141 struct timespec btime; 142 kmutex_t lock; 143 }; 144 145 /* For our fixed-size allocations */ 146 static struct pool fcrpl; 147 static struct pool csepl; 148 149 /* Declaration of master device (fd-cloning/ctxt-allocating) entrypoints */ 150 static int cryptoopen(dev_t dev, int flag, int mode, struct lwp *l); 151 static int cryptoread(dev_t dev, struct uio *uio, int ioflag); 152 static int cryptowrite(dev_t dev, struct uio *uio, int ioflag); 153 static int cryptoselect(dev_t dev, int rw, struct lwp *l); 154 155 static bool crypto_detaching = false; 156 static int crypto_refcount = 0; /* Prevent detaching while in use */ 157 158 static int 159 crypto_refcount_get(void) 160 { 161 int error; 162 163 mutex_enter(&cryptodev_mtx); 164 if (crypto_detaching) { 165 error = SET_ERROR(ENXIO); 166 } else if (crypto_refcount == INT_MAX) { 167 error = SET_ERROR(EBUSY); 168 } else { 169 crypto_refcount++; 170 error = 0; 171 } 172 mutex_exit(&cryptodev_mtx); 173 174 return error; 175 } 176 177 static void 178 crypto_refcount_put(void) 179 { 180 181 mutex_enter(&cryptodev_mtx); 182 KASSERT(crypto_refcount > 0); 183 crypto_refcount--; 184 mutex_exit(&cryptodev_mtx); 185 } 186 187 /* Declaration of cloned-device (per-ctxt) entrypoints */ 188 static int cryptof_read(struct file *, off_t *, struct uio *, 189 kauth_cred_t, int); 190 static int cryptof_write(struct file *, off_t *, struct uio *, 191 kauth_cred_t, int); 192 static int cryptof_ioctl(struct file *, u_long, void *); 193 static int cryptof_close(struct file *); 194 static int cryptof_poll(struct file *, int); 195 static int cryptof_stat(struct file *, struct stat *); 196 197 static const struct fileops cryptofops = { 198 .fo_name = "cryptof", 199 .fo_read = cryptof_read, 200 .fo_write = cryptof_write, 201 .fo_ioctl = cryptof_ioctl, 202 .fo_fcntl = fnullop_fcntl, 203 .fo_poll = cryptof_poll, 204 .fo_stat = cryptof_stat, 205 .fo_close = cryptof_close, 206 .fo_kqfilter = fnullop_kqfilter, 207 .fo_restart = fnullop_restart, 208 }; 209 210 static struct csession *cse_find(struct fcrypt *, uint32_t); 211 static int cse_delete(struct fcrypt *, uint32_t); 212 static struct csession *cse_add(struct fcrypt *, struct csession *); 213 static struct csession *cse_create(struct fcrypt *, uint64_t, void *, 214 uint64_t, void *, uint64_t, uint32_t, uint32_t, uint32_t, 215 const struct enc_xform *, const struct auth_hash *, 216 const struct comp_algo *); 217 static void cse_free(struct csession *); 218 219 static int cryptodev_key(struct crypt_kop *); 220 static int cryptodev_mkey(struct fcrypt *, struct crypt_n_kop *, size_t); 221 static void cryptodev_msessionfin(struct fcrypt *, size_t, uint32_t *); 222 223 static void cryptodev_cb(struct cryptop *); 224 static void cryptodevkey_cb(struct cryptkop *); 225 226 static void cryptodev_mcb(struct cryptop *); 227 static void cryptodevkey_mcb(struct cryptkop *); 228 229 static int cryptodev_getmstatus(struct fcrypt *, struct crypt_result *, 230 int); 231 static int cryptodev_getstatus(struct fcrypt *, struct crypt_result *); 232 233 static struct cryptop_data * 234 cod_ctor(struct cryptop_data *cod, struct csession *cse, size_t iov_len) 235 { 236 237 KASSERT(iov_len <= CRYPTOP_IOV_MAX); 238 239 memset(cod, 0, sizeof(*cod)); 240 cod->cse = cse; 241 cod->iov_len = iov_len; 242 cod->uio.uio_iovcnt = 1; 243 cod->uio.uio_resid = 0; 244 cod->uio.uio_rw = UIO_WRITE; 245 cod->uio.uio_iov = cod->iovec; 246 UIO_SETUP_SYSSPACE(&cod->uio); 247 248 /* the iov needs to be big enough to handle the uncompressed data.... */ 249 cod->uio.uio_iov[0].iov_len = cod->iov_len; 250 if (cod->iov_len > 0) 251 cod->uio.uio_iov[0].iov_base = kmem_zalloc(iov_len, KM_SLEEP); 252 cod->uio.uio_resid = cod->uio.uio_iov[0].iov_len; 253 DPRINTF("lid[%u]: uio.iov_base %p malloced %zu bytes\n", 254 CRYPTO_SESID2LID(cse->sid), 255 cod->uio.uio_iov[0].iov_base, cod->iov_len); 256 return cod; 257 } 258 259 static void 260 cod_dtor(struct cryptop_data *cod) 261 { 262 DPRINTF("lid[%u]: uio.iov_base %p freeing %zu bytes\n", 263 CRYPTO_SESID2LID(cod->cse->sid), 264 cod->uio.uio_iov[0].iov_base, cod->iov_len); 265 if (cod->iov_len && cod->uio.uio_iov[0].iov_base) { 266 kmem_free(cod->uio.uio_iov[0].iov_base, cod->iov_len); 267 cod->uio.uio_iov[0].iov_base = NULL; 268 } 269 } 270 271 static void 272 fcrypt_settime(struct fcrypt *fcr) 273 { 274 mutex_enter(&fcr->lock); 275 fcr->mtime = fcr->atime; 276 mutex_exit(&fcr->lock); 277 } 278 279 static struct fcrypt * 280 fcrypt_ctor(void) 281 { 282 struct fcrypt *fcr; 283 284 fcr = pool_get(&fcrpl, PR_WAITOK); 285 getnanotime(&fcr->btime); 286 fcr->atime = fcr->mtime = fcr->btime; 287 TAILQ_INIT(&fcr->csessions); 288 TAILQ_INIT(&fcr->crp_ret_mq); 289 TAILQ_INIT(&fcr->crp_ret_mkq); 290 selinit(&fcr->sinfo); 291 /* 292 * Don't ever return session 0, to allow detection of 293 * failed creation attempts with multi-create ioctl. 294 */ 295 fcr->sesn = 1; 296 fcr->requestid = 1; 297 mutex_init(&fcr->lock, MUTEX_DEFAULT, IPL_NONE); 298 299 return fcr; 300 } 301 302 static void 303 fcrypt_dtor(struct fcrypt *fcr) 304 { 305 struct csession *cse; 306 307 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 308 TAILQ_REMOVE(&fcr->csessions, cse, next); 309 KASSERT(atomic_load_relaxed(&cse->refcnt) == 1); 310 cse_free(cse); 311 } 312 seldestroy(&fcr->sinfo); 313 mutex_destroy(&fcr->lock); 314 315 pool_put(&fcrpl, fcr); 316 } 317 318 /* 319 * sysctl-able control variables for /dev/crypto now defined in crypto.c: 320 * crypto_usercrypto, crypto_userasymcrypto, crypto_devallowsoft. 321 */ 322 323 /* ARGSUSED */ 324 int 325 cryptof_read(file_t *fp, off_t *poff, 326 struct uio *uio, kauth_cred_t cred, int flags) 327 { 328 return SET_ERROR(EIO); 329 } 330 331 /* ARGSUSED */ 332 int 333 cryptof_write(file_t *fp, off_t *poff, 334 struct uio *uio, kauth_cred_t cred, int flags) 335 { 336 return SET_ERROR(EIO); 337 } 338 339 /* ARGSUSED */ 340 int 341 cryptof_ioctl(struct file *fp, u_long cmd, void *data) 342 { 343 struct fcrypt *fcr = fp->f_fcrypt; 344 struct csession *cse; 345 struct session_op *sop; 346 struct session_n_op *snop; 347 struct crypt_op *cop; 348 struct crypt_mop *mop; 349 struct crypt_mkop *mkop; 350 struct crypt_n_op *cnop; 351 struct crypt_n_kop *knop; 352 struct crypt_sgop *sgop; 353 struct crypt_sfop *sfop; 354 struct cryptret *crypt_ret; 355 struct crypt_result *crypt_res; 356 uint32_t ses; 357 uint32_t *sesid; 358 int error = 0; 359 size_t count, len; 360 361 /* backwards compatibility */ 362 file_t *criofp; 363 struct fcrypt *criofcr; 364 int criofd; 365 366 mutex_enter(&fcr->lock); 367 getnanotime(&fcr->atime); 368 mutex_exit(&fcr->lock); 369 370 switch (cmd) { 371 case CRIOGET: /* XXX deprecated, remove after 5.0 */ 372 error = crypto_refcount_get(); 373 if (error) 374 return error; 375 if ((error = fd_allocfile(&criofp, &criofd)) != 0) { 376 crypto_refcount_put(); 377 return error; 378 } 379 criofcr = fcrypt_ctor(); 380 error = fd_clone(criofp, criofd, (FREAD|FWRITE), 381 &cryptofops, criofcr); 382 KASSERTMSG(error == EMOVEFD, "error=%d", error); 383 *(uint32_t *)data = criofd; 384 return 0; 385 case CIOCGSESSION: 386 sop = (struct session_op *)data; 387 error = cryptodev_session(fcr, sop); 388 DPRINTF("create(%#x) error = %d\n", sop->ses, error); 389 break; 390 case CIOCNGSESSION: 391 sgop = (struct crypt_sgop *)data; 392 if (sgop->count <= 0 || sgop->count > 393 MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*snop))) { 394 error = SET_ERROR(EINVAL); 395 break; 396 } 397 len = sgop->count * sizeof(*snop); 398 snop = kmem_alloc(len, KM_SLEEP); 399 error = copyin(sgop->sessions, snop, len); 400 if (!error) { 401 fcrypt_settime(fcr); 402 error = cryptodev_msession(fcr, snop, sgop->count); 403 if (!error) { 404 error = copyout(snop, sgop->sessions, len); 405 } 406 } 407 kmem_free(snop, len); 408 break; 409 case CIOCFSESSION: 410 fcrypt_settime(fcr); 411 ses = *(uint32_t *)data; 412 mutex_enter(&fcr->lock); 413 error = cse_delete(fcr, ses); 414 mutex_exit(&fcr->lock); 415 DPRINTF("destroy(%#x) error = %d\n", ses, error); 416 break; 417 case CIOCNFSESSION: 418 fcrypt_settime(fcr); 419 sfop = (struct crypt_sfop *)data; 420 if (sfop->count <= 0 || sfop->count > 421 MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*sesid))) { 422 error = SET_ERROR(EINVAL); 423 break; 424 } 425 len = sfop->count * sizeof(*sesid); 426 sesid = kmem_alloc(len, KM_SLEEP); 427 error = copyin(sfop->sesid, sesid, len); 428 if (!error) { 429 cryptodev_msessionfin(fcr, sfop->count, sesid); 430 } 431 kmem_free(sesid, len); 432 break; 433 case CIOCCRYPT: 434 fcrypt_settime(fcr); 435 cop = (struct crypt_op *)data; 436 cse = cse_find(fcr, cop->ses); 437 if (cse == NULL) { 438 DPRINTF("cse_find failed %#x\n", cop->ses); 439 return SET_ERROR(EINVAL); 440 } 441 error = cryptodev_op(cse, cop, curlwp); 442 cse_free(cse); 443 DPRINTF("cryptodev_op error = %d\n", error); 444 break; 445 case CIOCNCRYPTM: 446 fcrypt_settime(fcr); 447 mop = (struct crypt_mop *)data; 448 if (mop->count <= 0 || mop->count > 449 MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*cnop))) { 450 error = SET_ERROR(EINVAL); 451 break; 452 } 453 len = mop->count * sizeof(*cnop); 454 cnop = kmem_alloc(len, KM_SLEEP); 455 error = copyin(mop->reqs, cnop, len); 456 if (!error) { 457 error = cryptodev_mop(fcr, cnop, mop->count, curlwp); 458 if (!error) { 459 error = copyout(cnop, mop->reqs, len); 460 } 461 } 462 kmem_free(cnop, len); 463 break; 464 case CIOCKEY: 465 error = cryptodev_key((struct crypt_kop *)data); 466 DPRINTF("cryptodev_key error = %d\n", error); 467 break; 468 case CIOCNFKEYM: 469 fcrypt_settime(fcr); 470 mkop = (struct crypt_mkop *)data; 471 if (mkop->count <= 0 || mkop->count > 472 MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*knop))) { 473 error = SET_ERROR(EINVAL); 474 break; 475 } 476 len = mkop->count * sizeof(*knop); 477 knop = kmem_alloc(len, KM_SLEEP); 478 error = copyin(mkop->reqs, knop, len); 479 if (!error) { 480 error = cryptodev_mkey(fcr, knop, mkop->count); 481 if (!error) 482 error = copyout(knop, mkop->reqs, len); 483 } 484 kmem_free(knop, len); 485 break; 486 case CIOCASYMFEAT: 487 error = crypto_getfeat((int *)data); 488 break; 489 case CIOCNCRYPTRETM: 490 fcrypt_settime(fcr); 491 crypt_ret = (struct cryptret *)data; 492 if (crypt_ret->count <= 0 || crypt_ret->count > 493 MIN(CRYPTODEV_OPS_MAX, SIZE_MAX/sizeof(*crypt_res))) { 494 error = SET_ERROR(EINVAL); 495 break; 496 } 497 len = crypt_ret->count * sizeof(*crypt_res); 498 crypt_res = kmem_alloc(len, KM_SLEEP); 499 error = copyin(crypt_ret->results, crypt_res, len); 500 if (error) 501 goto reterr; 502 count = crypt_ret->count; 503 crypt_ret->count = cryptodev_getmstatus(fcr, crypt_res, 504 crypt_ret->count); 505 /* sanity check count */ 506 if (crypt_ret->count > count) { 507 printf("%s.%d: error returned count %zd > original " 508 " count %zd\n", 509 __FILE__, __LINE__, crypt_ret->count, count); 510 crypt_ret->count = count; 511 512 } 513 error = copyout(crypt_res, crypt_ret->results, len); 514 reterr: 515 kmem_free(crypt_res, len); 516 break; 517 case CIOCNCRYPTRET: 518 error = cryptodev_getstatus(fcr, (struct crypt_result *)data); 519 break; 520 default: 521 /* Check for backward compatible commands */ 522 523 MODULE_HOOK_CALL(ocryptof_50_hook, (fp, cmd, data), 524 enosys(), error); 525 if (error == ENOSYS) 526 error = SET_ERROR(EINVAL); 527 return error; 528 } 529 return error; 530 } 531 532 int 533 cryptodev_op(struct csession *cse, struct crypt_op *cop, struct lwp *l) 534 { 535 struct cryptop *crp = NULL; 536 struct cryptop_data *cod = NULL; 537 struct cryptodesc *crde = NULL, *crda = NULL, *crdc = NULL; 538 int error; 539 size_t iov_len = cop->len; 540 int flags=0; 541 size_t dst_len; /* copyout size */ 542 543 if (cop->len > CRYPTOP_IOV_MAX || cop->dst_len > CRYPTOP_IOV_MAX) 544 return E2BIG; 545 546 if (cse->txform) { 547 if (cop->len < cse->txform->blocksize 548 + (cop->iv ? 0 : cse->txform->ivsize) || 549 (cop->len - (cop->iv ? 0 : cse->txform->ivsize)) 550 % cse->txform->blocksize != 0) 551 return SET_ERROR(EINVAL); 552 } 553 554 if (cse->tcomp == NULL && cse->txform == NULL && cse->thash == NULL) 555 return SET_ERROR(EINVAL); 556 557 DPRINTF("[%u]: iov_len %zu\n", 558 CRYPTO_SESID2LID(cse->sid), iov_len); 559 if ((cse->tcomp) && cop->dst_len) { 560 if (iov_len < cop->dst_len) { 561 /* Need larger iov to deal with decompress */ 562 iov_len = cop->dst_len; 563 } 564 DPRINTF("iov_len -> %zu for decompress\n", iov_len); 565 } 566 567 568 crp = crypto_getreq((cse->tcomp != NULL) 569 + (cse->txform != NULL) + (cse->thash != NULL)); 570 if (crp == NULL) { 571 error = SET_ERROR(ENOMEM); 572 goto bail; 573 } 574 cod = cod_ctor(&crp->cod, cse, iov_len); 575 DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(cse->sid), crp); 576 577 /* crds are always ordered tcomp, thash, then txform */ 578 /* with optional missing links */ 579 580 /* XXX: If we're going to compress then hash or encrypt, we need 581 * to be able to pass on the new size of the data. 582 */ 583 584 if (cse->tcomp) { 585 crdc = crp->crp_desc; 586 } 587 588 if (cse->thash) { 589 crda = crdc ? crdc->crd_next : crp->crp_desc; 590 if (cse->txform && crda) 591 crde = crda->crd_next; 592 } else { 593 if (cse->txform) { 594 crde = crdc ? crdc->crd_next : crp->crp_desc; 595 } else if (!cse->tcomp) { 596 error = SET_ERROR(EINVAL); 597 goto bail; 598 } 599 } 600 601 DPRINTF("ocf[%u]: iov_len %zu, cop->len %u\n", 602 CRYPTO_SESID2LID(cse->sid), 603 cod->uio.uio_iov[0].iov_len, 604 cop->len); 605 606 if ((error = copyin(cop->src, cod->uio.uio_iov[0].iov_base, cop->len))) 607 { 608 printf("copyin failed %s %d \n", (char *)cop->src, error); 609 goto bail; 610 } 611 612 if (crdc) { 613 switch (cop->op) { 614 case COP_COMP: 615 crdc->crd_flags |= CRD_F_COMP; 616 break; 617 case COP_DECOMP: 618 crdc->crd_flags &= ~CRD_F_COMP; 619 break; 620 default: 621 break; 622 } 623 /* more data to follow? */ 624 if (cop->flags & COP_F_MORE) { 625 flags |= CRYPTO_F_MORE; 626 } 627 crdc->crd_len = cop->len; 628 crdc->crd_inject = 0; 629 630 crdc->crd_alg = cse->comp_alg; 631 crdc->crd_key = NULL; 632 crdc->crd_klen = 0; 633 DPRINTF("lid[%u]: crdc setup for comp_alg %d.\n", 634 CRYPTO_SESID2LID(cse->sid), crdc->crd_alg); 635 } 636 637 if (crda) { 638 crda->crd_skip = 0; 639 crda->crd_len = cop->len; 640 crda->crd_inject = 0; /* ??? */ 641 642 crda->crd_alg = cse->mac; 643 crda->crd_key = cse->mackey; 644 crda->crd_klen = cse->mackeylen * 8; 645 DPRINTF("crda setup for mac %d.\n", crda->crd_alg); 646 } 647 648 if (crde) { 649 switch (cop->op) { 650 case COP_ENCRYPT: 651 crde->crd_flags |= CRD_F_ENCRYPT; 652 break; 653 case COP_DECRYPT: 654 crde->crd_flags &= ~CRD_F_ENCRYPT; 655 break; 656 default: 657 break; 658 } 659 crde->crd_len = cop->len; 660 crde->crd_inject = 0; 661 662 if (cse->cipher == CRYPTO_AES_GCM_16 && crda) 663 crda->crd_len = 0; 664 else if (cse->cipher == CRYPTO_AES_GMAC) 665 crde->crd_len = 0; 666 667 crde->crd_alg = cse->cipher; 668 crde->crd_key = cse->key; 669 crde->crd_klen = cse->keylen * 8; 670 DPRINTF("crde setup for cipher %d.\n", crde->crd_alg); 671 } 672 673 674 crp->crp_ilen = cop->len; 675 crp->crp_flags = CRYPTO_F_IOV | (cop->flags & COP_F_BATCH) | flags; 676 crp->crp_buf = (void *)&cod->uio; 677 crp->crp_callback = cryptodev_cb; 678 crp->crp_sid = cse->sid; 679 crp->crp_opaque = cod; 680 681 if (cop->iv) { 682 if (crde == NULL) { 683 error = SET_ERROR(EINVAL); 684 goto bail; 685 } 686 if (cse->txform->ivsize == 0) { 687 error = SET_ERROR(EINVAL); 688 goto bail; 689 } 690 if ((error = copyin(cop->iv, crp->tmp_iv, 691 cse->txform->ivsize))) 692 goto bail; 693 (void)memcpy(crde->crd_iv, crp->tmp_iv, cse->txform->ivsize); 694 crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 695 crde->crd_skip = 0; 696 } else if (crde) { 697 if (cse->txform->ivsize == 0) { 698 crde->crd_skip = 0; 699 } else { 700 if (!(crde->crd_flags & CRD_F_ENCRYPT)) 701 crde->crd_flags |= CRD_F_IV_PRESENT; 702 crde->crd_skip = cse->txform->ivsize; 703 crde->crd_len -= cse->txform->ivsize; 704 } 705 } 706 707 if (cop->mac) { 708 if (crda == NULL) { 709 error = SET_ERROR(EINVAL); 710 goto bail; 711 } 712 crp->crp_mac = crp->tmp_mac; 713 } 714 715 cv_init(&crp->crp_cv, "crydev"); 716 crypto_dispatch(crp); 717 mutex_enter(&cse->lock); 718 while (!(crp->crp_devflags & CRYPTODEV_F_RET)) { 719 DPRINTF("cse->sid[%u]: sleeping on cv %p for crp %p\n", 720 (uint32_t)cse->sid, &crp->crp_cv, crp); 721 cv_wait(&crp->crp_cv, &cse->lock); /* XXX cv_wait_sig? */ 722 } 723 mutex_exit(&cse->lock); 724 cv_destroy(&crp->crp_cv); 725 726 if (crp->crp_etype != 0) { 727 DPRINTF("crp_etype %d\n", crp->crp_etype); 728 error = crp->crp_etype; 729 goto bail; 730 } 731 732 dst_len = crp->crp_ilen; 733 /* let the user know how much data was returned */ 734 if (crp->crp_olen) { 735 if (crp->crp_olen > (cop->dst_len ? cop->dst_len : cop->len)) { 736 error = SET_ERROR(ENOSPC); 737 goto bail; 738 } 739 dst_len = cop->dst_len = crp->crp_olen; 740 } 741 742 if (cop->dst) { 743 DPRINTF("copyout %zu bytes to %p\n", dst_len, cop->dst); 744 } 745 if (cop->dst && 746 (error = copyout(cod->uio.uio_iov[0].iov_base, cop->dst, dst_len))) 747 { 748 DPRINTF("copyout error %d\n", error); 749 goto bail; 750 } 751 752 if (cop->mac && 753 (error = copyout(crp->crp_mac, cop->mac, cse->thash->authsize))) { 754 DPRINTF("mac copyout error %d\n", error); 755 goto bail; 756 } 757 758 759 bail: 760 if (cod) 761 cod_dtor(cod); 762 if (crp) 763 crypto_freereq(crp); 764 return error; 765 } 766 767 static void 768 cryptodev_cb(struct cryptop *crp) 769 { 770 struct cryptop_data *cod = crp->crp_opaque; 771 DPRINTF("cod = %p\n", cod); 772 773 mutex_enter(&cod->cse->lock); 774 crp->crp_devflags |= CRYPTODEV_F_RET; 775 cv_signal(&crp->crp_cv); 776 mutex_exit(&cod->cse->lock); 777 } 778 779 static void 780 cryptodev_mcb(struct cryptop *crp) 781 { 782 struct cryptop_data *cod = crp->crp_opaque; 783 784 mutex_enter(&cod->cse->lock); 785 TAILQ_INSERT_TAIL(&crp->fcrp->crp_ret_mq, crp, crp_next); 786 selnotify(&crp->fcrp->sinfo, 0, 0); 787 mutex_exit(&cod->cse->lock); 788 } 789 790 static void 791 cryptodevkey_cb(struct cryptkop *krp) 792 { 793 794 mutex_enter(&krp->krp_lock); 795 krp->krp_devflags |= CRYPTODEV_F_RET; 796 cv_signal(&krp->krp_cv); 797 mutex_exit(&krp->krp_lock); 798 } 799 800 static void 801 cryptodevkey_mcb(struct cryptkop *krp) 802 { 803 804 mutex_enter(&krp->krp_lock); 805 cv_signal(&krp->krp_cv); 806 TAILQ_INSERT_TAIL(&krp->fcrp->crp_ret_mkq, krp, krp_next); 807 selnotify(&krp->fcrp->sinfo, 0, 0); 808 mutex_exit(&krp->krp_lock); 809 } 810 811 static int 812 cryptodev_key(struct crypt_kop *kop) 813 { 814 struct cryptkop *krp = NULL; 815 int error = EINVAL; 816 size_t in, out, size, i; 817 818 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) 819 return SET_ERROR(EFBIG); 820 821 in = kop->crk_iparams; 822 out = kop->crk_oparams; 823 switch (kop->crk_op) { 824 case CRK_MOD_EXP: 825 if (in == 3 && out == 1) 826 break; 827 return SET_ERROR(EINVAL); 828 case CRK_MOD_EXP_CRT: 829 if (in == 6 && out == 1) 830 break; 831 return SET_ERROR(EINVAL); 832 case CRK_DSA_SIGN: 833 if (in == 5 && out == 2) 834 break; 835 return SET_ERROR(EINVAL); 836 case CRK_DSA_VERIFY: 837 if (in == 7 && out == 0) 838 break; 839 return SET_ERROR(EINVAL); 840 case CRK_DH_COMPUTE_KEY: 841 if (in == 3 && out == 1) 842 break; 843 return SET_ERROR(EINVAL); 844 case CRK_MOD_ADD: 845 if (in == 3 && out == 1) 846 break; 847 return SET_ERROR(EINVAL); 848 case CRK_MOD_ADDINV: 849 if (in == 2 && out == 1) 850 break; 851 return SET_ERROR(EINVAL); 852 case CRK_MOD_SUB: 853 if (in == 3 && out == 1) 854 break; 855 return SET_ERROR(EINVAL); 856 case CRK_MOD_MULT: 857 if (in == 3 && out == 1) 858 break; 859 return SET_ERROR(EINVAL); 860 case CRK_MOD_MULTINV: 861 if (in == 2 && out == 1) 862 break; 863 return SET_ERROR(EINVAL); 864 case CRK_MOD: 865 if (in == 2 && out == 1) 866 break; 867 return SET_ERROR(EINVAL); 868 default: 869 return SET_ERROR(EINVAL); 870 } 871 872 krp = crypto_kgetreq(1, PR_WAITOK); 873 if (krp == NULL) { 874 /* limited by opencrypto.crypto_ret_kq.maxlen */ 875 return SET_ERROR(ENOMEM); 876 } 877 (void)memset(krp, 0, sizeof *krp); 878 cv_init(&krp->krp_cv, "crykdev"); 879 mutex_init(&krp->krp_lock, MUTEX_DEFAULT, IPL_NONE); 880 krp->krp_op = kop->crk_op; 881 krp->krp_status = kop->crk_status; 882 krp->krp_iparams = kop->crk_iparams; 883 krp->krp_oparams = kop->crk_oparams; 884 krp->krp_status = 0; 885 krp->krp_callback = cryptodevkey_cb; 886 887 for (i = 0; i < CRK_MAXPARAM; i++) 888 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 889 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 890 size = (krp->krp_param[i].crp_nbits + 7) / 8; 891 if (size == 0) 892 continue; 893 krp->krp_param[i].crp_p = kmem_alloc(size, KM_SLEEP); 894 if (i >= krp->krp_iparams) 895 continue; 896 error = copyin(kop->crk_param[i].crp_p, 897 krp->krp_param[i].crp_p, size); 898 if (error) 899 goto fail; 900 } 901 902 crypto_kdispatch(krp); 903 904 mutex_enter(&krp->krp_lock); 905 while (!(krp->krp_devflags & CRYPTODEV_F_RET)) { 906 cv_wait(&krp->krp_cv, &krp->krp_lock); /* XXX cv_wait_sig? */ 907 } 908 mutex_exit(&krp->krp_lock); 909 910 if (krp->krp_status != 0) { 911 DPRINTF("krp->krp_status 0x%08x\n", krp->krp_status); 912 error = krp->krp_status; 913 goto fail; 914 } 915 916 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; 917 i++) { 918 size = (krp->krp_param[i].crp_nbits + 7) / 8; 919 if (size == 0) 920 continue; 921 error = copyout(krp->krp_param[i].crp_p, 922 kop->crk_param[i].crp_p, size); 923 if (error) { 924 DPRINTF("copyout oparam %zu failed, " 925 "error=%d\n", i-krp->krp_iparams, error); 926 goto fail; 927 } 928 } 929 930 fail: 931 kop->crk_status = krp->krp_status; 932 for (i = 0; i < CRK_MAXPARAM; i++) { 933 struct crparam *kp = &(krp->krp_param[i]); 934 if (krp->krp_param[i].crp_p) { 935 size = (kp->crp_nbits + 7) / 8; 936 KASSERT(size > 0); 937 (void)memset(kp->crp_p, 0, size); 938 kmem_free(kp->crp_p, size); 939 } 940 } 941 cv_destroy(&krp->krp_cv); 942 crypto_kfreereq(krp); 943 DPRINTF("error=0x%08x\n", error); 944 return error; 945 } 946 947 948 /* ARGSUSED */ 949 static int 950 cryptof_close(struct file *fp) 951 { 952 struct fcrypt *fcr = fp->f_fcrypt; 953 954 fp->f_fcrypt = NULL; 955 fcrypt_dtor(fcr); 956 crypto_refcount_put(); 957 return 0; 958 } 959 960 /* needed for compatibility module */ 961 struct csession * 962 cryptodev_cse_find(struct fcrypt *fcr, uint32_t ses) 963 { 964 return cse_find(fcr, ses); 965 } 966 967 /* needed for compatibility module */ 968 void 969 cryptodev_cse_free(struct csession *cse) 970 { 971 cse_free(cse); 972 } 973 974 static struct csession * 975 cse_find(struct fcrypt *fcr, uint32_t ses) 976 { 977 struct csession *cse, *cnext; 978 979 mutex_enter(&fcr->lock); 980 TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext) 981 if (cse->ses == ses) { 982 if (atomic_inc_uint_nv(&cse->refcnt) == UINT_MAX) { 983 atomic_dec_uint(&cse->refcnt); 984 cse = NULL; 985 } 986 mutex_exit(&fcr->lock); 987 return cse; 988 } 989 mutex_exit(&fcr->lock); 990 991 return NULL; 992 } 993 994 static int 995 cse_delete(struct fcrypt *fcr, uint32_t ses) 996 { 997 struct csession *cse, *cnext; 998 999 TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext) { 1000 if (cse->ses == ses) { 1001 if (atomic_load_relaxed(&cse->refcnt) != 1) 1002 return SET_ERROR(EBUSY); 1003 TAILQ_REMOVE(&fcr->csessions, cse, next); 1004 cse_free(cse); 1005 return 0; 1006 } 1007 } 1008 return SET_ERROR(EINVAL); 1009 } 1010 1011 static struct csession * 1012 cse_add(struct fcrypt *fcr, struct csession *cse) 1013 { 1014 mutex_enter(&fcr->lock); 1015 /* don't let session ID wrap! */ 1016 if (fcr->sesn + 1 == 0) { 1017 mutex_exit(&fcr->lock); 1018 return NULL; 1019 } 1020 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 1021 cse->ses = fcr->sesn++; 1022 mutex_exit(&fcr->lock); 1023 return cse; 1024 } 1025 1026 static struct csession * 1027 cse_create(struct fcrypt *fcr, uint64_t sid, void *key, uint64_t keylen, 1028 void *mackey, uint64_t mackeylen, uint32_t cipher, uint32_t mac, 1029 uint32_t comp_alg, const struct enc_xform *txform, 1030 const struct auth_hash *thash, const struct comp_algo *tcomp) 1031 { 1032 struct csession *cse; 1033 1034 cse = pool_get(&csepl, PR_NOWAIT); 1035 if (cse == NULL) 1036 return NULL; 1037 cse->key = key; 1038 mutex_init(&cse->lock, MUTEX_DEFAULT, IPL_NONE); 1039 cse->refcnt = 1; 1040 cse->keylen = keylen/8; 1041 cse->mackey = mackey; 1042 cse->mackeylen = mackeylen/8; 1043 cse->sid = sid; 1044 cse->cipher = cipher; 1045 cse->mac = mac; 1046 cse->comp_alg = comp_alg; 1047 cse->txform = txform; 1048 cse->thash = thash; 1049 cse->tcomp = tcomp; 1050 if (cse_add(fcr, cse)) { 1051 DPRINTF("created %p\n", cse); 1052 return cse; 1053 } 1054 1055 mutex_destroy(&cse->lock); 1056 pool_put(&csepl, cse); 1057 return NULL; 1058 } 1059 1060 static void 1061 cse_free(struct csession *cse) 1062 { 1063 1064 DPRINTF("destroying %p %d\n", cse, atomic_load_relaxed(&cse->refcnt)); 1065 membar_release(); 1066 if (atomic_dec_uint_nv(&cse->refcnt) > 0) 1067 return; 1068 membar_acquire(); 1069 mutex_destroy(&cse->lock); 1070 crypto_freesession(cse->sid); 1071 if (cse->key) 1072 free(cse->key, M_XDATA); 1073 if (cse->mackey) 1074 free(cse->mackey, M_XDATA); 1075 pool_put(&csepl, cse); 1076 } 1077 1078 static int 1079 cryptoopen(dev_t dev, int flag, int mode, 1080 struct lwp *l) 1081 { 1082 file_t *fp; 1083 struct fcrypt *fcr; 1084 int fd, error; 1085 1086 if (crypto_usercrypto == 0) 1087 return SET_ERROR(ENXIO); 1088 1089 if ((error = crypto_refcount_get()) != 0) 1090 return error; 1091 1092 if ((error = fd_allocfile(&fp, &fd)) != 0) { 1093 crypto_refcount_put(); 1094 return error; 1095 } 1096 1097 fcr = fcrypt_ctor(); 1098 error = fd_clone(fp, fd, flag, &cryptofops, fcr); 1099 KASSERTMSG(error == EMOVEFD, "error=%d", error); 1100 return error; 1101 } 1102 1103 static int 1104 cryptoread(dev_t dev, struct uio *uio, int ioflag) 1105 { 1106 return SET_ERROR(EIO); 1107 } 1108 1109 static int 1110 cryptowrite(dev_t dev, struct uio *uio, int ioflag) 1111 { 1112 return SET_ERROR(EIO); 1113 } 1114 1115 int 1116 cryptoselect(dev_t dev, int rw, struct lwp *l) 1117 { 1118 return 0; 1119 } 1120 1121 /*static*/ 1122 struct cdevsw crypto_cdevsw = { 1123 .d_open = cryptoopen, 1124 .d_close = noclose, 1125 .d_read = cryptoread, 1126 .d_write = cryptowrite, 1127 .d_ioctl = noioctl, 1128 .d_stop = nostop, 1129 .d_tty = notty, 1130 .d_poll = cryptoselect /*nopoll*/, 1131 .d_mmap = nommap, 1132 .d_kqfilter = nokqfilter, 1133 .d_discard = nodiscard, 1134 .d_flag = D_OTHER 1135 }; 1136 1137 int 1138 cryptodev_mop(struct fcrypt *fcr, 1139 struct crypt_n_op * cnop, 1140 size_t count, struct lwp *l) 1141 { 1142 struct cryptop *crp = NULL; 1143 struct cryptop_data *cod = NULL; 1144 struct cryptodesc *crde = NULL, *crda = NULL, *crdc = NULL; 1145 int error = 0; 1146 struct csession *cse; 1147 int flags=0; 1148 size_t req, iov_len; 1149 1150 1151 for (req = 0; req < count; req++) { 1152 cse = cse_find(fcr, cnop[req].ses); 1153 if (cse == NULL) { 1154 DPRINTF("cse_find failed %#x\n", cnop[req].ses); 1155 cnop[req].status = SET_ERROR(EINVAL); 1156 goto bail; 1157 } 1158 1159 if (cnop[req].len > CRYPTOP_IOV_MAX || 1160 cnop[req].dst_len > CRYPTOP_IOV_MAX) { 1161 DPRINTF("length failed\n"); 1162 cnop[req].status = SET_ERROR(EINVAL); 1163 goto bail; 1164 } 1165 if (cse->txform) { 1166 if (cnop[req].len < cse->txform->blocksize + 1167 (cnop[req].iv ? 0 : cse->txform->ivsize) || 1168 (cnop[req].len - 1169 (cnop[req].iv ? 0 : cse->txform->ivsize)) 1170 % cse->txform->blocksize) { 1171 cnop[req].status = SET_ERROR(EINVAL); 1172 goto bail; 1173 } 1174 } 1175 1176 if (cse->txform == NULL && 1177 cse->thash == NULL && 1178 cse->tcomp == NULL) { 1179 cnop[req].status = SET_ERROR(EINVAL); 1180 goto bail; 1181 } 1182 1183 /* sanitize */ 1184 if (cnop[req].len <= 0) { 1185 cnop[req].status = SET_ERROR(ENOMEM); 1186 goto bail; 1187 } 1188 1189 crp = crypto_getreq((cse->txform != NULL) + 1190 (cse->thash != NULL) + 1191 (cse->tcomp != NULL)); 1192 if (crp == NULL) { 1193 cnop[req].status = SET_ERROR(ENOMEM); 1194 goto bail; 1195 } 1196 1197 iov_len = cnop[req].len; 1198 /* got a compression/decompression max size? */ 1199 if ((cse->tcomp) && cnop[req].dst_len) { 1200 if (iov_len < cnop[req].dst_len) { 1201 /* Need larger iov to deal with decompress */ 1202 iov_len = cnop[req].dst_len; 1203 } 1204 DPRINTF("iov_len -> %zu for decompress\n", iov_len); 1205 } 1206 1207 cod = cod_ctor(&crp->cod, cse, iov_len); 1208 1209 if (cse->tcomp) { 1210 crdc = crp->crp_desc; 1211 } 1212 1213 if (cse->thash) { 1214 crda = crdc ? crdc->crd_next : crp->crp_desc; 1215 if (cse->txform && crda) 1216 crde = crda->crd_next; 1217 } else { 1218 if (cse->txform) { 1219 crde = crdc ? crdc->crd_next : crp->crp_desc; 1220 } else if (!cse->tcomp) { 1221 error = SET_ERROR(EINVAL); 1222 goto bail; 1223 } 1224 } 1225 1226 if ((copyin(cnop[req].src, 1227 cod->uio.uio_iov[0].iov_base, cnop[req].len))) { 1228 cnop[req].status = SET_ERROR(EINVAL); 1229 goto bail; 1230 } 1231 1232 if (crdc) { 1233 switch (cnop[req].op) { 1234 case COP_COMP: 1235 crdc->crd_flags |= CRD_F_COMP; 1236 break; 1237 case COP_DECOMP: 1238 crdc->crd_flags &= ~CRD_F_COMP; 1239 break; 1240 default: 1241 break; 1242 } 1243 /* more data to follow? */ 1244 if (cnop[req].flags & COP_F_MORE) { 1245 flags |= CRYPTO_F_MORE; 1246 } 1247 crdc->crd_len = cnop[req].len; 1248 crdc->crd_inject = 0; 1249 1250 crdc->crd_alg = cse->comp_alg; 1251 crdc->crd_key = NULL; 1252 crdc->crd_klen = 0; 1253 DPRINTF("cse->sid[%d]: crdc setup for comp_alg %d" 1254 " len %d.\n", 1255 (uint32_t)cse->sid, crdc->crd_alg, 1256 crdc->crd_len); 1257 } 1258 1259 if (crda) { 1260 crda->crd_skip = 0; 1261 crda->crd_len = cnop[req].len; 1262 crda->crd_inject = 0; /* ??? */ 1263 1264 crda->crd_alg = cse->mac; 1265 crda->crd_key = cse->mackey; 1266 crda->crd_klen = cse->mackeylen * 8; 1267 } 1268 1269 if (crde) { 1270 if (cnop[req].op == COP_ENCRYPT) 1271 crde->crd_flags |= CRD_F_ENCRYPT; 1272 else 1273 crde->crd_flags &= ~CRD_F_ENCRYPT; 1274 crde->crd_len = cnop[req].len; 1275 crde->crd_inject = 0; 1276 1277 crde->crd_alg = cse->cipher; 1278 #ifdef notyet /* XXX must notify h/w driver new key, drain */ 1279 if(cnop[req].key && cnop[req].keylen) { 1280 crde->crd_key = malloc(cnop[req].keylen, 1281 M_XDATA, M_WAITOK); 1282 if((error = copyin(cnop[req].key, 1283 crde->crd_key, cnop[req].keylen))) { 1284 cnop[req].status = SET_ERROR(EINVAL); 1285 goto bail; 1286 } 1287 crde->crd_klen = cnop[req].keylen * 8; 1288 } else { ... } 1289 #endif 1290 crde->crd_key = cse->key; 1291 crde->crd_klen = cse->keylen * 8; 1292 } 1293 1294 crp->crp_ilen = cnop[req].len; 1295 crp->crp_flags = CRYPTO_F_IOV | 1296 (cnop[req].flags & COP_F_BATCH) | flags; 1297 crp->crp_buf = (void *)&cod->uio; 1298 crp->crp_callback = cryptodev_mcb; 1299 crp->crp_sid = cse->sid; 1300 crp->crp_opaque = cod; 1301 crp->fcrp = fcr; 1302 crp->dst = cnop[req].dst; 1303 crp->len = cnop[req].len; /* input len, iov may be larger */ 1304 crp->mac = cnop[req].mac; 1305 DPRINTF("iov_base %p dst %p len %d mac %p\n", 1306 cod->uio.uio_iov[0].iov_base, crp->dst, crp->len, 1307 crp->mac); 1308 1309 if (cnop[req].iv) { 1310 if (crde == NULL) { 1311 cnop[req].status = SET_ERROR(EINVAL); 1312 goto bail; 1313 } 1314 if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ 1315 cnop[req].status = SET_ERROR(EINVAL); 1316 goto bail; 1317 } 1318 if ((error = copyin(cnop[req].iv, crp->tmp_iv, 1319 cse->txform->ivsize))) { 1320 cnop[req].status = SET_ERROR(EINVAL); 1321 goto bail; 1322 } 1323 (void)memcpy(crde->crd_iv, crp->tmp_iv, 1324 cse->txform->ivsize); 1325 crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 1326 crde->crd_skip = 0; 1327 } else if (crde) { 1328 if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ 1329 crde->crd_skip = 0; 1330 } else { 1331 if (!(crde->crd_flags & CRD_F_ENCRYPT)) 1332 crde->crd_flags |= CRD_F_IV_PRESENT; 1333 crde->crd_skip = cse->txform->ivsize; 1334 crde->crd_len -= cse->txform->ivsize; 1335 } 1336 } 1337 1338 if (cnop[req].mac) { 1339 if (crda == NULL) { 1340 cnop[req].status = SET_ERROR(EINVAL); 1341 goto bail; 1342 } 1343 crp->crp_mac = crp->tmp_mac; 1344 } 1345 cnop[req].reqid = atomic_inc_32_nv(&(fcr->requestid)); 1346 crp->crp_reqid = cnop[req].reqid; 1347 crp->crp_usropaque = cnop[req].opaque; 1348 cv_init(&crp->crp_cv, "crydev"); 1349 crypto_dispatch(crp); 1350 cnop[req].status = 0; 1351 cv_destroy(&crp->crp_cv); 1352 bail: 1353 if (cnop[req].status) { 1354 DPRINTF("%zu status %d\n", req, cnop[req].status); 1355 if (cse) 1356 cse_free(cse); 1357 if (cod) 1358 cod_dtor(cod); 1359 if (crp) 1360 crypto_freereq(crp); 1361 error = 0; 1362 } 1363 } 1364 return error; 1365 } 1366 1367 static int 1368 cryptodev_mkey(struct fcrypt *fcr, struct crypt_n_kop *kop, size_t count) 1369 { 1370 struct cryptkop *krp = NULL; 1371 int error = EINVAL; 1372 size_t in, out, size, i, req; 1373 1374 for (req = 0; req < count; req++) { 1375 if (kop[req].crk_iparams + kop[req].crk_oparams > CRK_MAXPARAM) 1376 return SET_ERROR(EFBIG); 1377 1378 in = kop[req].crk_iparams; 1379 out = kop[req].crk_oparams; 1380 switch (kop[req].crk_op) { 1381 case CRK_MOD_EXP: 1382 if (in == 3 && out == 1) 1383 break; 1384 kop[req].crk_status = SET_ERROR(EINVAL); 1385 continue; 1386 case CRK_MOD_EXP_CRT: 1387 if (in == 6 && out == 1) 1388 break; 1389 kop[req].crk_status = SET_ERROR(EINVAL); 1390 continue; 1391 case CRK_DSA_SIGN: 1392 if (in == 5 && out == 2) 1393 break; 1394 kop[req].crk_status = SET_ERROR(EINVAL); 1395 continue; 1396 case CRK_DSA_VERIFY: 1397 if (in == 7 && out == 0) 1398 break; 1399 kop[req].crk_status = SET_ERROR(EINVAL); 1400 continue; 1401 case CRK_DH_COMPUTE_KEY: 1402 if (in == 3 && out == 1) 1403 break; 1404 kop[req].crk_status = SET_ERROR(EINVAL); 1405 continue; 1406 case CRK_MOD_ADD: 1407 if (in == 3 && out == 1) 1408 break; 1409 kop[req].crk_status = SET_ERROR(EINVAL); 1410 continue; 1411 case CRK_MOD_ADDINV: 1412 if (in == 2 && out == 1) 1413 break; 1414 kop[req].crk_status = SET_ERROR(EINVAL); 1415 continue; 1416 case CRK_MOD_SUB: 1417 if (in == 3 && out == 1) 1418 break; 1419 kop[req].crk_status = SET_ERROR(EINVAL); 1420 continue; 1421 case CRK_MOD_MULT: 1422 if (in == 3 && out == 1) 1423 break; 1424 kop[req].crk_status = SET_ERROR(EINVAL); 1425 continue; 1426 case CRK_MOD_MULTINV: 1427 if (in == 2 && out == 1) 1428 break; 1429 kop[req].crk_status = SET_ERROR(EINVAL); 1430 continue; 1431 case CRK_MOD: 1432 if (in == 2 && out == 1) 1433 break; 1434 kop[req].crk_status = SET_ERROR(EINVAL); 1435 continue; 1436 default: 1437 kop[req].crk_status = SET_ERROR(EINVAL); 1438 continue; 1439 } 1440 1441 krp = crypto_kgetreq(1, PR_WAITOK); 1442 if (krp == NULL) { 1443 /* limited by opencrypto.crypto_ret_kq.maxlen */ 1444 continue; 1445 } 1446 (void)memset(krp, 0, sizeof *krp); 1447 cv_init(&krp->krp_cv, "crykdev"); 1448 krp->krp_op = kop[req].crk_op; 1449 krp->krp_status = kop[req].crk_status; 1450 krp->krp_iparams = kop[req].crk_iparams; 1451 krp->krp_oparams = kop[req].crk_oparams; 1452 krp->krp_status = 0; 1453 krp->krp_callback = cryptodevkey_mcb; 1454 (void)memcpy(krp->crk_param, kop[req].crk_param, 1455 sizeof(kop[req].crk_param)); 1456 1457 krp->krp_flags = 0; 1458 1459 for (i = 0; i < CRK_MAXPARAM; i++) 1460 krp->krp_param[i].crp_nbits = 1461 kop[req].crk_param[i].crp_nbits; 1462 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1463 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1464 if (size == 0) 1465 continue; 1466 krp->krp_param[i].crp_p = 1467 kmem_alloc(size, KM_SLEEP); 1468 if (i >= krp->krp_iparams) 1469 continue; 1470 kop[req].crk_status = 1471 copyin(kop[req].crk_param[i].crp_p, 1472 krp->krp_param[i].crp_p, size); 1473 if (kop[req].crk_status) 1474 goto fail; 1475 } 1476 krp->fcrp = fcr; 1477 1478 kop[req].crk_reqid = atomic_inc_32_nv(&(fcr->requestid)); 1479 krp->krp_reqid = kop[req].crk_reqid; 1480 krp->krp_usropaque = kop[req].crk_opaque; 1481 1482 crypto_kdispatch(krp); 1483 kop[req].crk_status = 0; 1484 fail: 1485 if (kop[req].crk_status) { 1486 if (krp) { 1487 kop[req].crk_status = krp->krp_status; 1488 for (i = 0; i < CRK_MAXPARAM; i++) { 1489 struct crparam *kp = 1490 &(krp->krp_param[i]); 1491 if (kp->crp_p) { 1492 size = (kp->crp_nbits + 7) / 8; 1493 KASSERT(size > 0); 1494 memset(kp->crp_p, 0, size); 1495 kmem_free(kp->crp_p, size); 1496 } 1497 } 1498 cv_destroy(&krp->krp_cv); 1499 crypto_kfreereq(krp); 1500 } 1501 } 1502 error = 0; 1503 } 1504 DPRINTF("error=0x%08x\n", error); 1505 return error; 1506 } 1507 1508 int 1509 cryptodev_session(struct fcrypt *fcr, struct session_op *sop) 1510 { 1511 struct cryptoini cria, crie; 1512 struct cryptoini cric; /* compressor */ 1513 struct cryptoini *crihead = NULL; 1514 const struct enc_xform *txform = NULL; 1515 const struct auth_hash *thash = NULL; 1516 const struct comp_algo *tcomp = NULL; 1517 struct csession *cse; 1518 uint64_t sid; 1519 int error = 0; 1520 1521 DPRINTF("cipher=%d, mac=%d\n", sop->cipher, sop->mac); 1522 1523 /* XXX there must be a way to not embed the list of xforms here */ 1524 switch (sop->cipher) { 1525 case 0: 1526 break; 1527 case CRYPTO_DES_CBC: 1528 txform = &enc_xform_des; 1529 break; 1530 case CRYPTO_3DES_CBC: 1531 txform = &enc_xform_3des; 1532 break; 1533 case CRYPTO_BLF_CBC: 1534 txform = &enc_xform_blf; 1535 break; 1536 case CRYPTO_CAST_CBC: 1537 txform = &enc_xform_cast5; 1538 break; 1539 case CRYPTO_SKIPJACK_CBC: 1540 txform = &enc_xform_skipjack; 1541 break; 1542 case CRYPTO_AES_CBC: 1543 txform = &enc_xform_aes; 1544 break; 1545 case CRYPTO_CAMELLIA_CBC: 1546 txform = &enc_xform_camellia; 1547 break; 1548 case CRYPTO_AES_CTR: 1549 txform = &enc_xform_aes_ctr; 1550 break; 1551 case CRYPTO_AES_GCM_16: 1552 txform = &enc_xform_aes_gcm; 1553 break; 1554 case CRYPTO_AES_GMAC: 1555 txform = &enc_xform_aes_gmac; 1556 break; 1557 case CRYPTO_NULL_CBC: 1558 txform = &enc_xform_null; 1559 break; 1560 case CRYPTO_ARC4: 1561 txform = &enc_xform_arc4; 1562 break; 1563 default: 1564 DPRINTF("Invalid cipher %d\n", sop->cipher); 1565 return SET_ERROR(EINVAL); 1566 } 1567 1568 switch (sop->comp_alg) { 1569 case 0: 1570 break; 1571 case CRYPTO_DEFLATE_COMP: 1572 tcomp = &comp_algo_deflate; 1573 break; 1574 case CRYPTO_GZIP_COMP: 1575 tcomp = &comp_algo_gzip; 1576 DPRINTF("tcomp for GZIP\n"); 1577 break; 1578 default: 1579 DPRINTF("Invalid compression alg %d\n", sop->comp_alg); 1580 return SET_ERROR(EINVAL); 1581 } 1582 1583 switch (sop->mac) { 1584 case 0: 1585 break; 1586 case CRYPTO_MD5_HMAC: 1587 thash = &auth_hash_hmac_md5; 1588 break; 1589 case CRYPTO_SHA1_HMAC: 1590 thash = &auth_hash_hmac_sha1; 1591 break; 1592 case CRYPTO_MD5_HMAC_96: 1593 thash = &auth_hash_hmac_md5_96; 1594 break; 1595 case CRYPTO_SHA1_HMAC_96: 1596 thash = &auth_hash_hmac_sha1_96; 1597 break; 1598 case CRYPTO_SHA2_HMAC: 1599 /* XXX switching on key length seems questionable */ 1600 if (sop->mackeylen == auth_hash_hmac_sha2_256.keysize) { 1601 thash = &auth_hash_hmac_sha2_256; 1602 } else if (sop->mackeylen == auth_hash_hmac_sha2_384.keysize) { 1603 thash = &auth_hash_hmac_sha2_384; 1604 } else if (sop->mackeylen == auth_hash_hmac_sha2_512.keysize) { 1605 thash = &auth_hash_hmac_sha2_512; 1606 } else { 1607 DPRINTF("Invalid mackeylen %d\n", sop->mackeylen); 1608 return SET_ERROR(EINVAL); 1609 } 1610 break; 1611 case CRYPTO_SHA2_384_HMAC: 1612 thash = &auth_hash_hmac_sha2_384; 1613 break; 1614 case CRYPTO_SHA2_512_HMAC: 1615 thash = &auth_hash_hmac_sha2_512; 1616 break; 1617 case CRYPTO_RIPEMD160_HMAC: 1618 thash = &auth_hash_hmac_ripemd_160; 1619 break; 1620 case CRYPTO_RIPEMD160_HMAC_96: 1621 thash = &auth_hash_hmac_ripemd_160_96; 1622 break; 1623 case CRYPTO_MD5: 1624 thash = &auth_hash_md5; 1625 break; 1626 case CRYPTO_SHA1: 1627 thash = &auth_hash_sha1; 1628 break; 1629 case CRYPTO_AES_XCBC_MAC_96: 1630 thash = &auth_hash_aes_xcbc_mac_96; 1631 break; 1632 case CRYPTO_AES_128_GMAC: 1633 thash = &auth_hash_gmac_aes_128; 1634 break; 1635 case CRYPTO_AES_192_GMAC: 1636 thash = &auth_hash_gmac_aes_192; 1637 break; 1638 case CRYPTO_AES_256_GMAC: 1639 thash = &auth_hash_gmac_aes_256; 1640 break; 1641 case CRYPTO_NULL_HMAC: 1642 thash = &auth_hash_null; 1643 break; 1644 default: 1645 DPRINTF("Invalid mac %d\n", sop->mac); 1646 return SET_ERROR(EINVAL); 1647 } 1648 1649 memset(&crie, 0, sizeof(crie)); 1650 memset(&cria, 0, sizeof(cria)); 1651 memset(&cric, 0, sizeof(cric)); 1652 1653 if (tcomp) { 1654 cric.cri_alg = tcomp->type; 1655 cric.cri_klen = 0; 1656 DPRINTF("tcomp->type = %d\n", tcomp->type); 1657 1658 crihead = &cric; 1659 if (txform) { 1660 cric.cri_next = &crie; 1661 } else if (thash) { 1662 cric.cri_next = &cria; 1663 } 1664 } 1665 1666 if (txform) { 1667 crie.cri_alg = txform->type; 1668 crie.cri_klen = sop->keylen * 8; 1669 if (sop->keylen > txform->maxkey || 1670 sop->keylen < txform->minkey) { 1671 DPRINTF("keylen %d not in [%d,%d]\n", 1672 sop->keylen, txform->minkey, txform->maxkey); 1673 error = SET_ERROR(EINVAL); 1674 goto bail; 1675 } 1676 1677 crie.cri_key = malloc(crie.cri_klen / 8, M_XDATA, M_WAITOK); 1678 if ((error = copyin(sop->key, crie.cri_key, crie.cri_klen / 8))) 1679 goto bail; 1680 if (!crihead) { 1681 crihead = &crie; 1682 } 1683 if (thash) 1684 crie.cri_next = &cria; 1685 } 1686 1687 if (thash) { 1688 cria.cri_alg = thash->type; 1689 cria.cri_klen = sop->mackeylen * 8; 1690 if (sop->mackeylen != thash->keysize) { 1691 DPRINTF("mackeylen %d != keysize %d\n", 1692 sop->mackeylen, thash->keysize); 1693 error = SET_ERROR(EINVAL); 1694 goto bail; 1695 } 1696 if (cria.cri_klen) { 1697 cria.cri_key = malloc(cria.cri_klen / 8, M_XDATA, 1698 M_WAITOK); 1699 if ((error = copyin(sop->mackey, cria.cri_key, 1700 cria.cri_klen / 8))) { 1701 goto bail; 1702 } 1703 } 1704 if (!crihead) { 1705 crihead = &cria; 1706 } 1707 } 1708 1709 error = crypto_newsession(&sid, crihead, crypto_devallowsoft); 1710 if (!error) { 1711 DPRINTF("got session %d\n", (uint32_t)sid); 1712 cse = cse_create(fcr, sid, crie.cri_key, crie.cri_klen, 1713 cria.cri_key, cria.cri_klen, (txform ? sop->cipher : 0), sop->mac, 1714 (tcomp ? sop->comp_alg : 0), txform, thash, tcomp); 1715 if (cse != NULL) { 1716 sop->ses = cse->ses; 1717 } else { 1718 DPRINTF("csecreate failed\n"); 1719 crypto_freesession(sid); 1720 error = SET_ERROR(EINVAL); 1721 } 1722 } else { 1723 DPRINTF("SIOCSESSION violates kernel parameters %d\n", error); 1724 } 1725 bail: 1726 if (error) { 1727 if (crie.cri_key) { 1728 memset(crie.cri_key, 0, crie.cri_klen / 8); 1729 free(crie.cri_key, M_XDATA); 1730 } 1731 if (cria.cri_key) { 1732 memset(cria.cri_key, 0, cria.cri_klen / 8); 1733 free(cria.cri_key, M_XDATA); 1734 } 1735 } 1736 return error; 1737 } 1738 1739 int 1740 cryptodev_msession(struct fcrypt *fcr, struct session_n_op *sn_ops, 1741 size_t count) 1742 { 1743 size_t i; 1744 1745 for (i = 0; i < count; i++, sn_ops++) { 1746 struct session_op s_op; 1747 s_op.cipher = sn_ops->cipher; 1748 s_op.mac = sn_ops->mac; 1749 s_op.comp_alg = sn_ops->comp_alg; 1750 s_op.keylen = sn_ops->keylen; 1751 s_op.key = sn_ops->key; 1752 s_op.mackeylen = sn_ops->mackeylen; 1753 s_op.mackey = sn_ops->mackey; 1754 s_op.ses = ~0; 1755 1756 sn_ops->status = cryptodev_session(fcr, &s_op); 1757 if (sn_ops->status) { 1758 DPRINTF("%zu: create failed %d\n", i, sn_ops->status); 1759 } 1760 sn_ops->ses = s_op.ses; 1761 } 1762 1763 return 0; 1764 } 1765 1766 static void 1767 cryptodev_msessionfin(struct fcrypt *fcr, size_t count, uint32_t *sesid) 1768 { 1769 mutex_enter(&fcr->lock); 1770 for (size_t req = 0; req < count; req++) { 1771 // XXX: What to do if busy? 1772 cse_delete(fcr, sesid[req]); 1773 } 1774 mutex_exit(&fcr->lock); 1775 } 1776 1777 /* 1778 * collect as many completed requests as are available, or count completed 1779 * requests, whichever is less. 1780 * return the number of requests. 1781 */ 1782 static int 1783 cryptodev_getmstatus(struct fcrypt *fcr, struct crypt_result *crypt_res, 1784 int count) 1785 { 1786 struct cryptop *crp = NULL; 1787 struct cryptkop *krp = NULL; 1788 int i, size, req = 0; 1789 int completed=0; 1790 1791 /* On queue so nobody else can grab them 1792 * and copyout can be delayed-- no locking */ 1793 TAILQ_HEAD(, cryptop) crp_delfree_q = 1794 TAILQ_HEAD_INITIALIZER(crp_delfree_q); 1795 TAILQ_HEAD(, cryptkop) krp_delfree_q = 1796 TAILQ_HEAD_INITIALIZER(krp_delfree_q); 1797 1798 /* at this point we do not know which response user is requesting for 1799 * (symmetric or asymmetric) so we copyout one from each i.e if the 1800 * count is 2 then 1 from symmetric and 1 from asymmetric queue and 1801 * if 3 then 2 symmetric and 1 asymmetric and so on */ 1802 1803 /* pull off a list of requests while protected from changes */ 1804 mutex_enter(&fcr->lock); 1805 while (req < count) { 1806 crp = TAILQ_FIRST(&fcr->crp_ret_mq); 1807 if (crp) { 1808 TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next); 1809 TAILQ_INSERT_TAIL(&crp_delfree_q, crp, crp_next); 1810 crypt_res[req].status = 0; 1811 req++; 1812 } 1813 if (req < count) { 1814 crypt_res[req].status = 0; 1815 krp = TAILQ_FIRST(&fcr->crp_ret_mkq); 1816 if (krp) { 1817 TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next); 1818 TAILQ_INSERT_TAIL(&krp_delfree_q, krp, krp_next); 1819 req++; 1820 } 1821 } 1822 } 1823 mutex_exit(&fcr->lock); 1824 1825 /* now do all the work outside the mutex */ 1826 for(req=0; req < count ;) { 1827 crp = TAILQ_FIRST(&crp_delfree_q); 1828 if (crp) { 1829 struct cryptop_data *cod = crp->crp_opaque; 1830 if (crypt_res[req].status != 0) { 1831 /* cse_find failed during collection */ 1832 goto bail; 1833 } 1834 crypt_res[req].reqid = crp->crp_reqid; 1835 crypt_res[req].opaque = crp->crp_usropaque; 1836 completed++; 1837 1838 if (crp->crp_etype != 0) { 1839 crypt_res[req].status = crp->crp_etype; 1840 goto bail; 1841 } 1842 1843 if (crp->dst && (crypt_res[req].status = 1844 copyout(cod->uio.uio_iov[0].iov_base, crp->dst, 1845 crp->len))) 1846 goto bail; 1847 1848 if (crp->mac && (crypt_res[req].status = 1849 copyout(crp->crp_mac, crp->mac, 1850 cod->cse->thash->authsize))) 1851 goto bail; 1852 1853 bail: 1854 TAILQ_REMOVE(&crp_delfree_q, crp, crp_next); 1855 cse_free(cod->cse); 1856 cod_dtor(cod); 1857 crypto_freereq(crp); 1858 req++; 1859 } 1860 1861 if (req < count) { 1862 krp = TAILQ_FIRST(&krp_delfree_q); 1863 if (krp) { 1864 crypt_res[req].reqid = krp->krp_reqid; 1865 crypt_res[req].opaque = krp->krp_usropaque; 1866 completed++; 1867 if (krp->krp_status != 0) { 1868 DPRINTF("krp->krp_status 0x%08x\n", 1869 krp->krp_status); 1870 crypt_res[req].status = krp->krp_status; 1871 goto fail; 1872 } 1873 1874 for (i = krp->krp_iparams; i < krp->krp_iparams 1875 + krp->krp_oparams; i++) { 1876 size = (krp->krp_param[i].crp_nbits 1877 + 7) / 8; 1878 if (size == 0) 1879 continue; 1880 crypt_res[req].status = copyout 1881 (krp->krp_param[i].crp_p, 1882 krp->crk_param[i].crp_p, size); 1883 if (crypt_res[req].status) { 1884 DPRINTF("copyout oparam %d failed, " 1885 "error=%d\n", 1886 i - krp->krp_iparams, 1887 crypt_res[req].status); 1888 goto fail; 1889 } 1890 } 1891 fail: 1892 TAILQ_REMOVE(&krp_delfree_q, krp, krp_next); 1893 /* not sure what to do for this */ 1894 /* kop[req].crk_status = krp->krp_status; */ 1895 for (i = 0; i < CRK_MAXPARAM; i++) { 1896 struct crparam *kp = &(krp->krp_param[i]); 1897 if (kp->crp_p) { 1898 size = (kp->crp_nbits + 7) / 8; 1899 KASSERT(size > 0); 1900 (void)memset(kp->crp_p, 0, size); 1901 kmem_free(kp->crp_p, size); 1902 } 1903 } 1904 cv_destroy(&krp->krp_cv); 1905 crypto_kfreereq(krp); 1906 req++; 1907 } 1908 } 1909 } 1910 1911 return completed; 1912 } 1913 1914 static int 1915 cryptodev_getstatus (struct fcrypt *fcr, struct crypt_result *crypt_res) 1916 { 1917 struct cryptop *crp = NULL, *cnext; 1918 struct cryptkop *krp = NULL, *knext; 1919 int i, size, req = 0; 1920 1921 mutex_enter(&fcr->lock); 1922 /* Here we dont know for which request the user is requesting the 1923 * response so checking in both the queues */ 1924 TAILQ_FOREACH_SAFE(crp, &fcr->crp_ret_mq, crp_next, cnext) { 1925 if(crp && (crp->crp_reqid == crypt_res->reqid)) { 1926 struct cryptop_data *cod = crp->crp_opaque; 1927 crypt_res->opaque = crp->crp_usropaque; 1928 if (crp->crp_etype != 0) { 1929 crypt_res->status = crp->crp_etype; 1930 goto bail; 1931 } 1932 1933 if (crp->dst && (crypt_res->status = 1934 copyout(cod->uio.uio_iov[0].iov_base, 1935 crp->dst, crp->len))) 1936 goto bail; 1937 1938 if (crp->mac && (crypt_res->status = 1939 copyout(crp->crp_mac, crp->mac, 1940 cod->cse->thash->authsize))) 1941 goto bail; 1942 bail: 1943 TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next); 1944 mutex_exit(&fcr->lock); 1945 cse_free(cod->cse); 1946 cod_dtor(cod); 1947 crypto_freereq(crp); 1948 return 0; 1949 } 1950 } 1951 1952 TAILQ_FOREACH_SAFE(krp, &fcr->crp_ret_mkq, krp_next, knext) { 1953 if(krp && (krp->krp_reqid == crypt_res->reqid)) { 1954 crypt_res[req].opaque = krp->krp_usropaque; 1955 if (krp->krp_status != 0) { 1956 DPRINTF("krp->krp_status 0x%08x\n", 1957 krp->krp_status); 1958 crypt_res[req].status = krp->krp_status; 1959 goto fail; 1960 } 1961 1962 for (i = krp->krp_iparams; i < krp->krp_iparams + 1963 krp->krp_oparams; i++) { 1964 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1965 if (size == 0) 1966 continue; 1967 crypt_res[req].status = copyout( 1968 krp->krp_param[i].crp_p, 1969 krp->crk_param[i].crp_p, size); 1970 if (crypt_res[req].status) { 1971 DPRINTF("copyout oparam " 1972 "%d failed, error=%d\n", 1973 i - krp->krp_iparams, 1974 crypt_res[req].status); 1975 goto fail; 1976 } 1977 } 1978 fail: 1979 TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next); 1980 mutex_exit(&fcr->lock); 1981 /* not sure what to do for this */ 1982 /* kop[req].crk_status = krp->krp_status; */ 1983 for (i = 0; i < CRK_MAXPARAM; i++) { 1984 struct crparam *kp = &(krp->krp_param[i]); 1985 if (kp->crp_p) { 1986 size = (kp->crp_nbits + 7) / 8; 1987 KASSERT(size > 0); 1988 memset(kp->crp_p, 0, size); 1989 kmem_free(kp->crp_p, size); 1990 } 1991 } 1992 cv_destroy(&krp->krp_cv); 1993 crypto_kfreereq(krp); 1994 return 0; 1995 } 1996 } 1997 mutex_exit(&fcr->lock); 1998 return SET_ERROR(EINPROGRESS); 1999 } 2000 2001 static int 2002 cryptof_stat(struct file *fp, struct stat *st) 2003 { 2004 struct fcrypt *fcr = fp->f_fcrypt; 2005 2006 (void)memset(st, 0, sizeof(*st)); 2007 2008 mutex_enter(&fcr->lock); 2009 st->st_dev = makedev(cdevsw_lookup_major(&crypto_cdevsw), fcr->sesn); 2010 st->st_atimespec = fcr->atime; 2011 st->st_mtimespec = fcr->mtime; 2012 st->st_ctimespec = st->st_birthtimespec = fcr->btime; 2013 st->st_uid = kauth_cred_geteuid(fp->f_cred); 2014 st->st_gid = kauth_cred_getegid(fp->f_cred); 2015 mutex_exit(&fcr->lock); 2016 2017 return 0; 2018 } 2019 2020 static int 2021 cryptof_poll(struct file *fp, int events) 2022 { 2023 struct fcrypt *fcr = fp->f_fcrypt; 2024 int revents = 0; 2025 2026 if (!(events & (POLLIN | POLLRDNORM))) { 2027 /* only support read and POLLIN */ 2028 return 0; 2029 } 2030 2031 mutex_enter(&fcr->lock); 2032 if (TAILQ_EMPTY(&fcr->crp_ret_mq) && TAILQ_EMPTY(&fcr->crp_ret_mkq)) { 2033 /* no completed requests pending, save the poll for later */ 2034 selrecord(curlwp, &fcr->sinfo); 2035 } else { 2036 /* let the app(s) know that there are completed requests */ 2037 revents = events & (POLLIN | POLLRDNORM); 2038 } 2039 mutex_exit(&fcr->lock); 2040 2041 return revents; 2042 } 2043 2044 static void 2045 cryptodev_init(void) 2046 { 2047 2048 mutex_init(&cryptodev_mtx, MUTEX_DEFAULT, IPL_NONE); 2049 2050 pool_init(&fcrpl, sizeof(struct fcrypt), 0, 0, 0, "fcrpl", 2051 NULL, IPL_NONE); 2052 pool_init(&csepl, sizeof(struct csession), 0, 0, 0, "csepl", 2053 NULL, IPL_NONE); 2054 2055 /* 2056 * Preallocate space for 64 users, with 5 sessions each. 2057 * (consider that a TLS protocol session requires at least 2058 * 3DES, MD5, and SHA1 (both hashes are used in the PRF) for 2059 * the negotiation, plus HMAC_SHA1 for the actual SSL records, 2060 * consuming one session here for each algorithm. 2061 */ 2062 pool_prime(&fcrpl, 64); 2063 pool_prime(&csepl, 64 * 5); 2064 } 2065 2066 static void 2067 cryptodev_fini(void) 2068 { 2069 2070 KASSERTMSG(crypto_refcount == 0, 2071 "crypto_refcount=%d", crypto_refcount); 2072 2073 pool_destroy(&csepl); 2074 pool_destroy(&fcrpl); 2075 2076 mutex_destroy(&cryptodev_mtx); 2077 } 2078 2079 /* 2080 * Pseudo-device initialization routine for /dev/crypto 2081 */ 2082 void 2083 cryptoattach(int num) 2084 { 2085 #ifndef _MODULE 2086 crypto_init(); 2087 cryptodev_init(); 2088 #endif 2089 } 2090 2091 MODULE(MODULE_CLASS_DRIVER, crypto, "opencrypto"); 2092 2093 static int 2094 crypto_modcmd(modcmd_t cmd, void *arg) 2095 { 2096 #ifdef _MODULE 2097 int error = 0; 2098 devmajor_t cmajor = NODEVMAJOR, bmajor = NODEVMAJOR; 2099 #endif 2100 2101 switch (cmd) { 2102 case MODULE_CMD_INIT: 2103 #ifdef _MODULE 2104 cryptodev_init(); 2105 error = devsw_attach("crypto", NULL, &bmajor, 2106 &crypto_cdevsw, &cmajor); 2107 if (error) { 2108 aprint_error("crypto: unable to register devsw," 2109 " error %d\n", error); 2110 cryptodev_fini(); 2111 return error; 2112 } 2113 #endif 2114 return 0; 2115 case MODULE_CMD_FINI: 2116 /* 2117 * Can't unload safely because there is no way to wait 2118 * for all files with a given struct fileops to be 2119 * closed. 2120 */ 2121 return SET_ERROR(ENOTTY); 2122 2123 mutex_enter(&cryptodev_mtx); 2124 if (crypto_refcount != 0) { 2125 mutex_exit(&cryptodev_mtx); 2126 return SET_ERROR(EBUSY); 2127 } 2128 crypto_detaching = true; 2129 mutex_exit(&cryptodev_mtx); 2130 devsw_detach(NULL, &crypto_cdevsw); 2131 cryptodev_fini(); 2132 return 0; 2133 default: 2134 return SET_ERROR(ENOTTY); 2135 } 2136 } 2137
Indexes created Sun Oct 25 00:26:45 UTC 2026