cryptodev.c revision 1.44.4.1
1 /* $NetBSD: cryptodev.c,v 1.44.4.1 2008/12/13 01:15:33 haad Exp $ */ 2 /* $FreeBSD: src/sys/opencrypto/cryptodev.c,v 1.4.2.4 2003/06/03 00:09:02 sam Exp $ */ 3 /* $OpenBSD: cryptodev.c,v 1.53 2002/07/10 22:21:30 mickey Exp $ */ 4 5 /*- 6 * Copyright (c) 2008 The NetBSD Foundation, Inc. 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by Coyote Point Systems, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * Copyright (c) 2001 Theo de Raadt 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. The name of the author may not be used to endorse or promote products 47 * derived from this software without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 50 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 51 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 52 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 53 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 54 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 55 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 56 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 57 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 58 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 59 * 60 * Effort sponsored in part by the Defense Advanced Research Projects 61 * Agency (DARPA) and Air Force Research Laboratory, Air Force 62 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 63 * 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: cryptodev.c,v 1.44.4.1 2008/12/13 01:15:33 haad Exp $"); 68 69 #include <sys/param.h> 70 #include <sys/systm.h> 71 #include <sys/kmem.h> 72 #include <sys/malloc.h> 73 #include <sys/mbuf.h> 74 #include <sys/pool.h> 75 #include <sys/sysctl.h> 76 #include <sys/file.h> 77 #include <sys/filedesc.h> 78 #include <sys/errno.h> 79 #include <sys/md5.h> 80 #include <sys/sha1.h> 81 #include <sys/conf.h> 82 #include <sys/device.h> 83 #include <sys/kauth.h> 84 #include <sys/select.h> 85 #include <sys/poll.h> 86 #include <sys/atomic.h> 87 88 #include "opt_ocf.h" 89 #include <opencrypto/cryptodev.h> 90 #include <opencrypto/xform.h> 91 92 struct csession { 93 TAILQ_ENTRY(csession) next; 94 u_int64_t sid; 95 u_int32_t ses; 96 97 u_int32_t cipher; 98 struct enc_xform *txform; 99 u_int32_t mac; 100 struct auth_hash *thash; 101 102 void * key; 103 int keylen; 104 u_char tmp_iv[EALG_MAX_BLOCK_LEN]; 105 106 void * mackey; 107 int mackeylen; 108 u_char tmp_mac[CRYPTO_MAX_MAC_LEN]; 109 110 struct iovec iovec[1]; /* user requests never have more */ 111 struct uio uio; 112 int error; 113 }; 114 115 struct fcrypt { 116 TAILQ_HEAD(csessionlist, csession) csessions; 117 TAILQ_HEAD(crprethead, cryptop) crp_ret_mq; 118 TAILQ_HEAD(krprethead, cryptkop) crp_ret_mkq; 119 int sesn; 120 struct selinfo sinfo; 121 u_int32_t requestid; 122 }; 123 124 /* For our fixed-size allocations */ 125 static struct pool fcrpl; 126 static struct pool csepl; 127 128 /* Declaration of master device (fd-cloning/ctxt-allocating) entrypoints */ 129 static int cryptoopen(dev_t dev, int flag, int mode, struct lwp *l); 130 static int cryptoread(dev_t dev, struct uio *uio, int ioflag); 131 static int cryptowrite(dev_t dev, struct uio *uio, int ioflag); 132 static int cryptoselect(dev_t dev, int rw, struct lwp *l); 133 134 /* Declaration of cloned-device (per-ctxt) entrypoints */ 135 static int cryptof_read(struct file *, off_t *, struct uio *, 136 kauth_cred_t, int); 137 static int cryptof_write(struct file *, off_t *, struct uio *, 138 kauth_cred_t, int); 139 static int cryptof_ioctl(struct file *, u_long, void *); 140 static int cryptof_close(struct file *); 141 static int cryptof_poll(struct file *, int); 142 143 static const struct fileops cryptofops = { 144 cryptof_read, 145 cryptof_write, 146 cryptof_ioctl, 147 fnullop_fcntl, 148 cryptof_poll, 149 fbadop_stat, 150 cryptof_close, 151 fnullop_kqfilter 152 }; 153 154 static struct csession *csefind(struct fcrypt *, u_int); 155 static int csedelete(struct fcrypt *, struct csession *); 156 static struct csession *cseadd(struct fcrypt *, struct csession *); 157 static struct csession *csecreate(struct fcrypt *, u_int64_t, void *, 158 u_int64_t, void *, u_int64_t, u_int32_t, u_int32_t, struct enc_xform *, 159 struct auth_hash *); 160 static int csefree(struct csession *); 161 162 static int cryptodev_op(struct csession *, struct crypt_op *, 163 struct lwp *); 164 static int cryptodev_mop(struct fcrypt *, struct crypt_n_op *, int, 165 struct lwp *); 166 static int cryptodev_key(struct crypt_kop *); 167 static int cryptodev_mkey(struct fcrypt *, struct crypt_n_kop *, int); 168 static int cryptodev_session(struct fcrypt *, struct session_op *); 169 static int cryptodev_msession(struct fcrypt *, struct session_n_op *, int); 170 static int cryptodev_msessionfin(struct fcrypt *, int, u_int32_t *); 171 172 static int cryptodev_cb(void *); 173 static int cryptodevkey_cb(void *); 174 175 static int cryptodev_mcb(void *); 176 static int cryptodevkey_mcb(void *); 177 178 static int cryptodev_getmstatus(struct fcrypt *, struct crypt_result *, 179 int); 180 static int cryptodev_getstatus(struct fcrypt *, struct crypt_result *); 181 182 /* 183 * sysctl-able control variables for /dev/crypto now defined in crypto.c: 184 * crypto_usercrypto, crypto_userasmcrypto, crypto_devallowsoft. 185 */ 186 187 /* ARGSUSED */ 188 int 189 cryptof_read(file_t *fp, off_t *poff, 190 struct uio *uio, kauth_cred_t cred, int flags) 191 { 192 return EIO; 193 } 194 195 /* ARGSUSED */ 196 int 197 cryptof_write(file_t *fp, off_t *poff, 198 struct uio *uio, kauth_cred_t cred, int flags) 199 { 200 return EIO; 201 } 202 203 /* ARGSUSED */ 204 int 205 cryptof_ioctl(struct file *fp, u_long cmd, void *data) 206 { 207 struct fcrypt *fcr = fp->f_data; 208 struct csession *cse; 209 struct session_op *sop; 210 struct session_n_op *snop; 211 struct crypt_op *cop; 212 struct crypt_mop *mop; 213 struct crypt_mkop *mkop; 214 struct crypt_n_op *cnop; 215 struct crypt_n_kop *knop; 216 struct crypt_sgop *sgop; 217 struct crypt_sfop *sfop; 218 struct cryptret *crypt_ret; 219 struct crypt_result *crypt_res; 220 u_int32_t ses; 221 u_int32_t *sesid; 222 int error = 0; 223 size_t count; 224 225 /* backwards compatibility */ 226 file_t *criofp; 227 struct fcrypt *criofcr; 228 int criofd; 229 230 switch (cmd) { 231 case CRIOGET: /* XXX deprecated, remove after 5.0 */ 232 if ((error = fd_allocfile(&criofp, &criofd)) != 0) 233 return error; 234 criofcr = pool_get(&fcrpl, PR_WAITOK); 235 mutex_spin_enter(&crypto_mtx); 236 TAILQ_INIT(&criofcr->csessions); 237 TAILQ_INIT(&criofcr->crp_ret_mq); 238 TAILQ_INIT(&criofcr->crp_ret_mkq); 239 selinit(&criofcr->sinfo); 240 241 /* 242 * Don't ever return session 0, to allow detection of 243 * failed creation attempts with multi-create ioctl. 244 */ 245 criofcr->sesn = 1; 246 criofcr->requestid = 1; 247 mutex_spin_exit(&crypto_mtx); 248 (void)fd_clone(criofp, criofd, (FREAD|FWRITE), 249 &cryptofops, criofcr); 250 *(u_int32_t *)data = criofd; 251 return error; 252 break; 253 case CIOCGSESSION: 254 sop = (struct session_op *)data; 255 error = cryptodev_session(fcr, sop); 256 break; 257 case CIOCNGSESSION: 258 sgop = (struct crypt_sgop *)data; 259 snop = kmem_alloc((sgop->count * 260 sizeof(struct session_n_op)), KM_SLEEP); 261 error = copyin(sgop->sessions, snop, sgop->count * 262 sizeof(struct session_n_op)); 263 if (error) { 264 goto mbail; 265 } 266 267 error = cryptodev_msession(fcr, snop, sgop->count); 268 if (error) { 269 goto mbail; 270 } 271 272 error = copyout(snop, sgop->sessions, sgop->count * 273 sizeof(struct session_n_op)); 274 mbail: 275 kmem_free(snop, sgop->count * sizeof(struct session_n_op)); 276 break; 277 case CIOCFSESSION: 278 mutex_spin_enter(&crypto_mtx); 279 ses = *(u_int32_t *)data; 280 cse = csefind(fcr, ses); 281 if (cse == NULL) 282 return EINVAL; 283 csedelete(fcr, cse); 284 error = csefree(cse); 285 mutex_spin_exit(&crypto_mtx); 286 break; 287 case CIOCNFSESSION: 288 sfop = (struct crypt_sfop *)data; 289 sesid = kmem_alloc((sfop->count * sizeof(u_int32_t)), 290 KM_SLEEP); 291 error = copyin(sfop->sesid, sesid, 292 (sfop->count * sizeof(u_int32_t))); 293 if (!error) { 294 error = cryptodev_msessionfin(fcr, sfop->count, sesid); 295 } 296 kmem_free(sesid, (sfop->count * sizeof(u_int32_t))); 297 break; 298 case CIOCCRYPT: 299 mutex_spin_enter(&crypto_mtx); 300 cop = (struct crypt_op *)data; 301 cse = csefind(fcr, cop->ses); 302 mutex_spin_exit(&crypto_mtx); 303 if (cse == NULL) { 304 DPRINTF(("csefind failed\n")); 305 return EINVAL; 306 } 307 error = cryptodev_op(cse, cop, curlwp); 308 DPRINTF(("cryptodev_op error = %d\n", error)); 309 break; 310 case CIOCNCRYPTM: 311 mop = (struct crypt_mop *)data; 312 cnop = kmem_alloc((mop->count * sizeof(struct crypt_n_op)), 313 KM_SLEEP); 314 error = copyin(mop->reqs, cnop, 315 (mop->count * sizeof(struct crypt_n_op))); 316 if(!error) { 317 error = cryptodev_mop(fcr, cnop, mop->count, curlwp); 318 if (!error) { 319 error = copyout(cnop, mop->reqs, 320 (mop->count * sizeof(struct crypt_n_op))); 321 } 322 } 323 kmem_free(cnop, (mop->count * sizeof(struct crypt_n_op))); 324 break; 325 case CIOCKEY: 326 error = cryptodev_key((struct crypt_kop *)data); 327 DPRINTF(("cryptodev_key error = %d\n", error)); 328 break; 329 case CIOCNFKEYM: 330 mkop = (struct crypt_mkop *)data; 331 knop = kmem_alloc((mkop->count * sizeof(struct crypt_n_kop)), 332 KM_SLEEP); 333 error = copyin(mkop->reqs, knop, 334 (mkop->count * sizeof(struct crypt_n_kop))); 335 if (!error) { 336 error = cryptodev_mkey(fcr, knop, mkop->count); 337 if (!error) 338 error = copyout(knop, mkop->reqs, 339 (mkop->count * sizeof(struct crypt_n_kop))); 340 } 341 kmem_free(knop, (mkop->count * sizeof(struct crypt_n_kop))); 342 break; 343 case CIOCASYMFEAT: 344 error = crypto_getfeat((int *)data); 345 break; 346 case CIOCNCRYPTRETM: 347 crypt_ret = (struct cryptret *)data; 348 count = crypt_ret->count; 349 crypt_res = kmem_alloc((count * sizeof(struct crypt_result)), 350 KM_SLEEP); 351 error = copyin(crypt_ret->results, crypt_res, 352 (count * sizeof(struct crypt_result))); 353 if (error) 354 goto reterr; 355 crypt_ret->count = cryptodev_getmstatus(fcr, crypt_res, 356 crypt_ret->count); 357 /* sanity check count */ 358 if (crypt_ret->count > count) { 359 printf("%s.%d: error returned count %zd > original " 360 " count %zd\n", 361 __FILE__, __LINE__, crypt_ret->count, count); 362 crypt_ret->count = count; 363 364 } 365 error = copyout(crypt_res, crypt_ret->results, 366 (crypt_ret->count * sizeof(struct crypt_result))); 367 reterr: 368 kmem_free(crypt_res, (count * sizeof(struct crypt_result))); 369 break; 370 case CIOCNCRYPTRET: 371 error = cryptodev_getstatus(fcr, (struct crypt_result *)data); 372 break; 373 default: 374 DPRINTF(("invalid ioctl cmd %ld\n", cmd)); 375 error = EINVAL; 376 } 377 return error; 378 } 379 380 static int 381 cryptodev_op(struct csession *cse, struct crypt_op *cop, struct lwp *l) 382 { 383 struct cryptop *crp = NULL; 384 struct cryptodesc *crde = NULL, *crda = NULL; 385 int error; 386 387 if (cop->len > 256*1024-4) 388 return E2BIG; 389 390 if (cse->txform) { 391 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) 392 return EINVAL; 393 } 394 395 (void)memset(&cse->uio, 0, sizeof(cse->uio)); 396 cse->uio.uio_iovcnt = 1; 397 cse->uio.uio_resid = 0; 398 cse->uio.uio_rw = UIO_WRITE; 399 cse->uio.uio_iov = cse->iovec; 400 UIO_SETUP_SYSSPACE(&cse->uio); 401 memset(&cse->iovec, 0, sizeof(cse->iovec)); 402 cse->uio.uio_iov[0].iov_len = cop->len; 403 cse->uio.uio_iov[0].iov_base = kmem_alloc(cop->len, KM_SLEEP); 404 cse->uio.uio_resid = cse->uio.uio_iov[0].iov_len; 405 406 crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL)); 407 if (crp == NULL) { 408 error = ENOMEM; 409 goto bail; 410 } 411 412 if (cse->thash) { 413 crda = crp->crp_desc; 414 if (cse->txform && crda) 415 crde = crda->crd_next; 416 } else { 417 if (cse->txform) 418 crde = crp->crp_desc; 419 else { 420 error = EINVAL; 421 goto bail; 422 } 423 } 424 425 if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, cop->len))) 426 { 427 printf("copyin failed %s %d \n", (char *)cop->src, error); 428 goto bail; 429 } 430 431 if (crda) { 432 crda->crd_skip = 0; 433 crda->crd_len = cop->len; 434 crda->crd_inject = 0; /* ??? */ 435 436 crda->crd_alg = cse->mac; 437 crda->crd_key = cse->mackey; 438 crda->crd_klen = cse->mackeylen * 8; 439 } 440 441 if (crde) { 442 if (cop->op == COP_ENCRYPT) 443 crde->crd_flags |= CRD_F_ENCRYPT; 444 else 445 crde->crd_flags &= ~CRD_F_ENCRYPT; 446 crde->crd_len = cop->len; 447 crde->crd_inject = 0; 448 449 crde->crd_alg = cse->cipher; 450 crde->crd_key = cse->key; 451 crde->crd_klen = cse->keylen * 8; 452 } 453 454 crp->crp_ilen = cop->len; 455 /* The reqest is flagged as CRYPTO_F_USER as long as it is running 456 * in the user IOCTL thread. This flag lets us skip using the retq for 457 * the request if it completes immediately. If the request ends up being 458 * delayed or is not completed immediately the flag is removed. 459 */ 460 crp->crp_flags = CRYPTO_F_IOV | (cop->flags & COP_F_BATCH) | CRYPTO_F_USER; 461 crp->crp_buf = (void *)&cse->uio; 462 crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb; 463 crp->crp_sid = cse->sid; 464 crp->crp_opaque = (void *)cse; 465 466 if (cop->iv) { 467 if (crde == NULL) { 468 error = EINVAL; 469 goto bail; 470 } 471 if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ 472 error = EINVAL; 473 goto bail; 474 } 475 if ((error = copyin(cop->iv, cse->tmp_iv, 476 cse->txform->blocksize))) 477 goto bail; 478 (void)memcpy(crde->crd_iv, cse->tmp_iv, cse->txform->blocksize); 479 crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 480 crde->crd_skip = 0; 481 } else if (crde) { 482 if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ 483 crde->crd_skip = 0; 484 } else { 485 crde->crd_flags |= CRD_F_IV_PRESENT; 486 crde->crd_skip = cse->txform->blocksize; 487 crde->crd_len -= cse->txform->blocksize; 488 } 489 } 490 491 if (cop->mac) { 492 if (crda == NULL) { 493 error = EINVAL; 494 goto bail; 495 } 496 crp->crp_mac=cse->tmp_mac; 497 } 498 499 /* 500 * XXX there was a comment here which said that we went to 501 * XXX splcrypto() but needed to only if CRYPTO_F_CBIMM, 502 * XXX disabled on NetBSD since 1.6O due to a race condition. 503 * XXX But crypto_dispatch went to splcrypto() itself! (And 504 * XXX now takes the crypto_mtx mutex itself). We do, however, 505 * XXX need to hold the mutex across the call to cv_wait(). 506 * XXX (should we arrange for crypto_dispatch to return to 507 * XXX us with it held? it seems quite ugly to do so.) 508 */ 509 #ifdef notyet 510 eagain: 511 #endif 512 error = crypto_dispatch(crp); 513 mutex_spin_enter(&crypto_mtx); 514 515 /* 516 * If the request was going to be completed by the 517 * ioctl thread then it would have been done by now. 518 * Remove the F_USER flag it so crypto_done() is not confused 519 * if the crypto device calls it after this point. 520 */ 521 crp->crp_flags &= ~(CRYPTO_F_USER); 522 523 switch (error) { 524 #ifdef notyet /* don't loop forever -- but EAGAIN not possible here yet */ 525 case EAGAIN: 526 mutex_spin_exit(&crypto_mtx); 527 goto eagain; 528 break; 529 #endif 530 case 0: 531 break; 532 default: 533 DPRINTF(("cryptodev_op: not waiting, error.\n")); 534 mutex_spin_exit(&crypto_mtx); 535 goto bail; 536 } 537 538 while (!(crp->crp_flags & CRYPTO_F_DONE)) { 539 DPRINTF(("cryptodev_op: sleeping on cv %08x for crp %08x\n", \ 540 (uint32_t)&crp->crp_cv, (uint32_t)crp)); 541 cv_wait(&crp->crp_cv, &crypto_mtx); /* XXX cv_wait_sig? */ 542 } 543 if (crp->crp_flags & CRYPTO_F_ONRETQ) { 544 /* XXX this should never happen now with the CRYPTO_F_USER flag 545 * changes. 546 */ 547 DPRINTF(("cryptodev_op: DONE, not woken by cryptoret.\n")); 548 (void)crypto_ret_q_remove(crp); 549 } 550 mutex_spin_exit(&crypto_mtx); 551 552 if (crp->crp_etype != 0) { 553 DPRINTF(("cryptodev_op: crp_etype %d\n", crp->crp_etype)); 554 error = crp->crp_etype; 555 goto bail; 556 } 557 558 if (cse->error) { 559 DPRINTF(("cryptodev_op: cse->error %d\n", cse->error)); 560 error = cse->error; 561 goto bail; 562 } 563 564 if (cop->dst && 565 (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, cop->len))) 566 { 567 DPRINTF(("cryptodev_op: copyout error %d\n", error)); 568 goto bail; 569 } 570 571 if (cop->mac && 572 (error = copyout(crp->crp_mac, cop->mac, cse->thash->authsize))) { 573 DPRINTF(("cryptodev_op: mac copyout error %d\n", error)); 574 goto bail; 575 } 576 577 bail: 578 if (crp) { 579 crypto_freereq(crp); 580 } 581 if (cse->uio.uio_iov[0].iov_base) 582 kmem_free(cse->uio.uio_iov[0].iov_base, 583 cse->uio.uio_iov[0].iov_len); 584 585 return error; 586 } 587 588 static int 589 cryptodev_cb(void *op) 590 { 591 struct cryptop *crp = (struct cryptop *) op; 592 struct csession *cse = (struct csession *)crp->crp_opaque; 593 int error = 0; 594 595 mutex_spin_enter(&crypto_mtx); 596 cse->error = crp->crp_etype; 597 if (crp->crp_etype == EAGAIN) { 598 /* always drop mutex to call dispatch routine */ 599 mutex_spin_exit(&crypto_mtx); 600 error = crypto_dispatch(crp); 601 mutex_spin_enter(&crypto_mtx); 602 } 603 if (error != 0 || (crp->crp_flags & CRYPTO_F_DONE)) { 604 cv_signal(&crp->crp_cv); 605 } 606 mutex_spin_exit(&crypto_mtx); 607 return 0; 608 } 609 610 static int 611 cryptodev_mcb(void *op) 612 { 613 struct cryptop *crp = (struct cryptop *) op; 614 struct csession *cse = (struct csession *)crp->crp_opaque; 615 int error=0; 616 617 mutex_spin_enter(&crypto_mtx); 618 cse->error = crp->crp_etype; 619 if (crp->crp_etype == EAGAIN) { 620 mutex_spin_exit(&crypto_mtx); 621 error = crypto_dispatch(crp); 622 mutex_spin_enter(&crypto_mtx); 623 } 624 if (error != 0 || (crp->crp_flags & CRYPTO_F_DONE)) { 625 cv_signal(&crp->crp_cv); 626 } 627 628 TAILQ_INSERT_TAIL(&crp->fcrp->crp_ret_mq, crp, crp_next); 629 selnotify(&crp->fcrp->sinfo, 0, 0); 630 mutex_spin_exit(&crypto_mtx); 631 return 0; 632 } 633 634 static int 635 cryptodevkey_cb(void *op) 636 { 637 struct cryptkop *krp = op; 638 639 mutex_spin_enter(&crypto_mtx); 640 cv_signal(&krp->krp_cv); 641 mutex_spin_exit(&crypto_mtx); 642 return 0; 643 } 644 645 static int 646 cryptodevkey_mcb(void *op) 647 { 648 struct cryptkop *krp = op; 649 650 mutex_spin_enter(&crypto_mtx); 651 cv_signal(&krp->krp_cv); 652 TAILQ_INSERT_TAIL(&krp->fcrp->crp_ret_mkq, krp, krp_next); 653 selnotify(&krp->fcrp->sinfo, 0, 0); 654 mutex_spin_exit(&crypto_mtx); 655 return 0; 656 } 657 658 static int 659 cryptodev_key(struct crypt_kop *kop) 660 { 661 struct cryptkop *krp = NULL; 662 int error = EINVAL; 663 int in, out, size, i; 664 665 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) 666 return EFBIG; 667 668 in = kop->crk_iparams; 669 out = kop->crk_oparams; 670 switch (kop->crk_op) { 671 case CRK_MOD_EXP: 672 if (in == 3 && out == 1) 673 break; 674 return EINVAL; 675 case CRK_MOD_EXP_CRT: 676 if (in == 6 && out == 1) 677 break; 678 return EINVAL; 679 case CRK_DSA_SIGN: 680 if (in == 5 && out == 2) 681 break; 682 return EINVAL; 683 case CRK_DSA_VERIFY: 684 if (in == 7 && out == 0) 685 break; 686 return EINVAL; 687 case CRK_DH_COMPUTE_KEY: 688 if (in == 3 && out == 1) 689 break; 690 return EINVAL; 691 case CRK_MOD_ADD: 692 if (in == 3 && out == 1) 693 break; 694 return EINVAL; 695 case CRK_MOD_ADDINV: 696 if (in == 2 && out == 1) 697 break; 698 return EINVAL; 699 case CRK_MOD_SUB: 700 if (in == 3 && out == 1) 701 break; 702 return EINVAL; 703 case CRK_MOD_MULT: 704 if (in == 3 && out == 1) 705 break; 706 return EINVAL; 707 case CRK_MOD_MULTINV: 708 if (in == 2 && out == 1) 709 break; 710 return EINVAL; 711 case CRK_MOD: 712 if (in == 2 && out == 1) 713 break; 714 return EINVAL; 715 default: 716 return EINVAL; 717 } 718 719 krp = pool_get(&cryptkop_pool, PR_WAITOK); 720 (void)memset(krp, 0, sizeof *krp); 721 cv_init(&krp->krp_cv, "crykdev"); 722 krp->krp_op = kop->crk_op; 723 krp->krp_status = kop->crk_status; 724 krp->krp_iparams = kop->crk_iparams; 725 krp->krp_oparams = kop->crk_oparams; 726 krp->krp_status = 0; 727 krp->krp_callback = (int (*) (struct cryptkop *)) cryptodevkey_cb; 728 729 for (i = 0; i < CRK_MAXPARAM; i++) 730 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 731 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 732 size = (krp->krp_param[i].crp_nbits + 7) / 8; 733 if (size == 0) 734 continue; 735 krp->krp_param[i].crp_p = kmem_alloc(size, KM_SLEEP); 736 if (i >= krp->krp_iparams) 737 continue; 738 error = copyin(kop->crk_param[i].crp_p, 739 krp->krp_param[i].crp_p, size); 740 if (error) 741 goto fail; 742 } 743 744 error = crypto_kdispatch(krp); 745 if (error != 0) { 746 goto fail; 747 } 748 749 mutex_spin_enter(&crypto_mtx); 750 while (!(krp->krp_flags & CRYPTO_F_DONE)) { 751 cv_wait(&krp->krp_cv, &crypto_mtx); /* XXX cv_wait_sig? */ 752 } 753 if (krp->krp_flags & CRYPTO_F_ONRETQ) { 754 DPRINTF(("cryptodev_key: DONE early, not via cryptoret.\n")); 755 (void)crypto_ret_kq_remove(krp); 756 } 757 mutex_spin_exit(&crypto_mtx); 758 759 if (krp->krp_status != 0) { 760 DPRINTF(("cryptodev_key: krp->krp_status 0x%08x\n", 761 krp->krp_status)); 762 error = krp->krp_status; 763 goto fail; 764 } 765 766 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; 767 i++) { 768 size = (krp->krp_param[i].crp_nbits + 7) / 8; 769 if (size == 0) 770 continue; 771 error = copyout(krp->krp_param[i].crp_p, 772 kop->crk_param[i].crp_p, size); 773 if (error) { 774 DPRINTF(("cryptodev_key: copyout oparam %d failed, " 775 "error=%d\n", i-krp->krp_iparams, error)); 776 goto fail; 777 } 778 } 779 780 fail: 781 kop->crk_status = krp->krp_status; 782 for (i = 0; i < CRK_MAXPARAM; i++) { 783 struct crparam *kp = &(krp->krp_param[i]); 784 if (krp->krp_param[i].crp_p) { 785 size = (kp->crp_nbits + 7) / 8; 786 KASSERT(size > 0); 787 (void)memset(kp->crp_p, 0, size); 788 kmem_free(kp->crp_p, size); 789 } 790 } 791 cv_destroy(&krp->krp_cv); 792 pool_put(&cryptkop_pool, krp); 793 DPRINTF(("cryptodev_key: error=0x%08x\n", error)); 794 return error; 795 } 796 797 /* ARGSUSED */ 798 static int 799 cryptof_close(struct file *fp) 800 { 801 struct fcrypt *fcr = fp->f_data; 802 struct csession *cse; 803 804 mutex_spin_enter(&crypto_mtx); 805 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 806 TAILQ_REMOVE(&fcr->csessions, cse, next); 807 (void)csefree(cse); 808 } 809 seldestroy(&fcr->sinfo); 810 fp->f_data = NULL; 811 mutex_spin_exit(&crypto_mtx); 812 813 pool_put(&fcrpl, fcr); 814 return 0; 815 } 816 817 /* csefind: call with crypto_mtx held. */ 818 static struct csession * 819 csefind(struct fcrypt *fcr, u_int ses) 820 { 821 struct csession *cse, *cnext, *ret = NULL; 822 823 KASSERT(mutex_owned(&crypto_mtx)); 824 TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext) 825 if (cse->ses == ses) 826 ret = cse; 827 828 return ret; 829 } 830 831 /* csedelete: call with crypto_mtx held. */ 832 static int 833 csedelete(struct fcrypt *fcr, struct csession *cse_del) 834 { 835 struct csession *cse, *cnext; 836 int ret = 0; 837 838 KASSERT(mutex_owned(&crypto_mtx)); 839 TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext) { 840 if (cse == cse_del) { 841 TAILQ_REMOVE(&fcr->csessions, cse, next); 842 ret = 1; 843 } 844 } 845 return ret; 846 } 847 848 /* cseadd: call with crypto_mtx held. */ 849 static struct csession * 850 cseadd(struct fcrypt *fcr, struct csession *cse) 851 { 852 KASSERT(mutex_owned(&crypto_mtx)); 853 /* don't let session ID wrap! */ 854 if (fcr->sesn + 1 == 0) return NULL; 855 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 856 cse->ses = fcr->sesn++; 857 return cse; 858 } 859 860 /* csecreate: call with crypto_mtx held. */ 861 static struct csession * 862 csecreate(struct fcrypt *fcr, u_int64_t sid, void *key, u_int64_t keylen, 863 void *mackey, u_int64_t mackeylen, u_int32_t cipher, u_int32_t mac, 864 struct enc_xform *txform, struct auth_hash *thash) 865 { 866 struct csession *cse; 867 868 KASSERT(mutex_owned(&crypto_mtx)); 869 cse = pool_get(&csepl, PR_NOWAIT); 870 if (cse == NULL) 871 return NULL; 872 cse->key = key; 873 cse->keylen = keylen/8; 874 cse->mackey = mackey; 875 cse->mackeylen = mackeylen/8; 876 cse->sid = sid; 877 cse->cipher = cipher; 878 cse->mac = mac; 879 cse->txform = txform; 880 cse->thash = thash; 881 cse->error = 0; 882 if (cseadd(fcr, cse)) 883 return cse; 884 else { 885 pool_put(&csepl, cse); 886 return NULL; 887 } 888 } 889 890 /* csefree: call with crypto_mtx held. */ 891 static int 892 csefree(struct csession *cse) 893 { 894 int error; 895 896 KASSERT(mutex_owned(&crypto_mtx)); 897 error = crypto_freesession(cse->sid); 898 if (cse->key) 899 free(cse->key, M_XDATA); 900 if (cse->mackey) 901 free(cse->mackey, M_XDATA); 902 pool_put(&csepl, cse); 903 return error; 904 } 905 906 static int 907 cryptoopen(dev_t dev, int flag, int mode, 908 struct lwp *l) 909 { 910 file_t *fp; 911 struct fcrypt *fcr; 912 int fd, error; 913 914 if (crypto_usercrypto == 0) 915 return ENXIO; 916 917 if ((error = fd_allocfile(&fp, &fd)) != 0) 918 return error; 919 920 fcr = pool_get(&fcrpl, PR_WAITOK); 921 mutex_spin_enter(&crypto_mtx); 922 TAILQ_INIT(&fcr->csessions); 923 TAILQ_INIT(&fcr->crp_ret_mq); 924 TAILQ_INIT(&fcr->crp_ret_mkq); 925 selinit(&fcr->sinfo); 926 /* 927 * Don't ever return session 0, to allow detection of 928 * failed creation attempts with multi-create ioctl. 929 */ 930 fcr->sesn = 1; 931 fcr->requestid = 1; 932 mutex_spin_exit(&crypto_mtx); 933 return fd_clone(fp, fd, flag, &cryptofops, fcr); 934 } 935 936 static int 937 cryptoread(dev_t dev, struct uio *uio, int ioflag) 938 { 939 return EIO; 940 } 941 942 static int 943 cryptowrite(dev_t dev, struct uio *uio, int ioflag) 944 { 945 return EIO; 946 } 947 948 int 949 cryptoselect(dev_t dev, int rw, struct lwp *l) 950 { 951 return 0; 952 } 953 954 /*static*/ 955 struct cdevsw crypto_cdevsw = { 956 /* open */ cryptoopen, 957 /* close */ noclose, 958 /* read */ cryptoread, 959 /* write */ cryptowrite, 960 /* ioctl */ noioctl, 961 /* ttstop?*/ nostop, 962 /* ??*/ notty, 963 /* poll */ cryptoselect /*nopoll*/, 964 /* mmap */ nommap, 965 /* kqfilter */ nokqfilter, 966 /* type */ D_OTHER, 967 }; 968 969 static int 970 cryptodev_mop(struct fcrypt *fcr, 971 struct crypt_n_op * cnop, 972 int count, struct lwp *l) 973 { 974 struct cryptop *crp = NULL; 975 struct cryptodesc *crde = NULL, *crda = NULL; 976 int req, error=0; 977 struct csession *cse; 978 979 for (req = 0; req < count; req++) { 980 mutex_spin_enter(&crypto_mtx); 981 cse = csefind(fcr, cnop[req].ses); 982 if (cse == NULL) { 983 DPRINTF(("csefind failed\n")); 984 cnop[req].status = EINVAL; 985 mutex_spin_exit(&crypto_mtx); 986 continue; 987 } 988 mutex_spin_exit(&crypto_mtx); 989 990 if (cnop[req].len > 256*1024-4) { 991 DPRINTF(("length failed\n")); 992 cnop[req].status = EINVAL; 993 continue; 994 } 995 if (cse->txform) { 996 if (cnop[req].len == 0 || 997 (cnop[req].len % cse->txform->blocksize) != 0) { 998 cnop[req].status = EINVAL; 999 continue; 1000 } 1001 } 1002 1003 crp = crypto_getreq((cse->txform != NULL) + 1004 (cse->thash != NULL)); 1005 if (crp == NULL) { 1006 cnop[req].status = ENOMEM; 1007 goto bail; 1008 } 1009 1010 (void)memset(&crp->uio, 0, sizeof(crp->uio)); 1011 crp->uio.uio_iovcnt = 1; 1012 crp->uio.uio_resid = 0; 1013 crp->uio.uio_rw = UIO_WRITE; 1014 crp->uio.uio_iov = crp->iovec; 1015 UIO_SETUP_SYSSPACE(&crp->uio); 1016 memset(&crp->iovec, 0, sizeof(crp->iovec)); 1017 crp->uio.uio_iov[0].iov_len = cnop[req].len; 1018 crp->uio.uio_iov[0].iov_base = kmem_alloc(cnop[req].len, 1019 KM_SLEEP); 1020 crp->uio.uio_resid = crp->uio.uio_iov[0].iov_len; 1021 1022 if (cse->thash) { 1023 crda = crp->crp_desc; 1024 if (cse->txform && crda) 1025 crde = crda->crd_next; 1026 } else { 1027 if (cse->txform) 1028 crde = crp->crp_desc; 1029 else { 1030 cnop[req].status = EINVAL; 1031 goto bail; 1032 } 1033 } 1034 1035 if ((copyin(cnop[req].src, 1036 crp->uio.uio_iov[0].iov_base, cnop[req].len))) { 1037 cnop[req].status = EINVAL; 1038 goto bail; 1039 } 1040 1041 if (crda) { 1042 crda->crd_skip = 0; 1043 crda->crd_len = cnop[req].len; 1044 crda->crd_inject = 0; /* ??? */ 1045 1046 crda->crd_alg = cse->mac; 1047 crda->crd_key = cse->mackey; 1048 crda->crd_klen = cse->mackeylen * 8; 1049 } 1050 1051 if (crde) { 1052 if (cnop[req].op == COP_ENCRYPT) 1053 crde->crd_flags |= CRD_F_ENCRYPT; 1054 else 1055 crde->crd_flags &= ~CRD_F_ENCRYPT; 1056 crde->crd_len = cnop[req].len; 1057 crde->crd_inject = 0; 1058 1059 crde->crd_alg = cse->cipher; 1060 #ifdef notyet /* XXX must notify h/w driver new key, drain */ 1061 if(cnop[req].key && cnop[req].keylen) { 1062 crde->crd_key = malloc(cnop[req].keylen, 1063 M_XDATA, M_WAITOK); 1064 if((error = copyin(cnop[req].key, 1065 crde->crd_key, cnop[req].keylen))) { 1066 cnop[req].status = EINVAL; 1067 goto bail; 1068 } 1069 crde->crd_klen = cnop[req].keylen * 8; 1070 } else { ... } 1071 #endif 1072 crde->crd_key = cse->key; 1073 crde->crd_klen = cse->keylen * 8; 1074 } 1075 1076 crp->crp_ilen = cnop[req].len; 1077 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM | 1078 (cnop[req].flags & COP_F_BATCH); 1079 crp->crp_buf = (void *)&crp->uio; 1080 crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_mcb; 1081 crp->crp_sid = cse->sid; 1082 crp->crp_opaque = (void *)cse; 1083 crp->fcrp = fcr; 1084 crp->dst = cnop[req].dst; 1085 /* we can use the crp_ilen in cryptop(crp) for this */ 1086 crp->len = cnop[req].len; 1087 crp->mac = cnop[req].mac; 1088 1089 if (cnop[req].iv) { 1090 if (crde == NULL) { 1091 cnop[req].status = EINVAL; 1092 goto bail; 1093 } 1094 if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ 1095 cnop[req].status = EINVAL; 1096 goto bail; 1097 } 1098 if ((error = copyin(cnop[req].iv, crp->tmp_iv, 1099 cse->txform->blocksize))) { 1100 cnop[req].status = EINVAL; 1101 goto bail; 1102 } 1103 (void)memcpy(crde->crd_iv, crp->tmp_iv, 1104 cse->txform->blocksize); 1105 crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 1106 crde->crd_skip = 0; 1107 } else if (crde) { 1108 if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ 1109 crde->crd_skip = 0; 1110 } else { 1111 crde->crd_flags |= CRD_F_IV_PRESENT; 1112 crde->crd_skip = cse->txform->blocksize; 1113 crde->crd_len -= cse->txform->blocksize; 1114 } 1115 } 1116 1117 if (cnop[req].mac) { 1118 if (crda == NULL) { 1119 cnop[req].status = EINVAL; 1120 goto bail; 1121 } 1122 crp->crp_mac=cse->tmp_mac; 1123 } 1124 cnop[req].reqid = atomic_inc_32_nv(&(fcr->requestid)); 1125 crp->crp_reqid = cnop[req].reqid; 1126 crp->crp_usropaque = cnop[req].opaque; 1127 #ifdef notyet 1128 eagain: 1129 #endif 1130 cnop[req].status = crypto_dispatch(crp); 1131 mutex_spin_enter(&crypto_mtx); /* XXX why mutex? */ 1132 1133 switch (cnop[req].status) { 1134 #ifdef notyet /* don't loop forever -- but EAGAIN not possible here yet */ 1135 case EAGAIN: 1136 mutex_spin_exit(&crypto_mtx); 1137 goto eagain; 1138 break; 1139 #endif 1140 case 0: 1141 break; 1142 default: 1143 DPRINTF(("cryptodev_op: not waiting, error.\n")); 1144 mutex_spin_exit(&crypto_mtx); 1145 goto bail; 1146 } 1147 1148 mutex_spin_exit(&crypto_mtx); 1149 bail: 1150 if (cnop[req].status) { 1151 if (crp) { 1152 crypto_freereq(crp); 1153 if(cse->uio.uio_iov[0].iov_base) { 1154 kmem_free(cse->uio.uio_iov[0].iov_base, 1155 cse->uio.uio_iov[0].iov_len); 1156 } 1157 } 1158 error = 0; 1159 } 1160 } 1161 return error; 1162 } 1163 1164 static int 1165 cryptodev_mkey(struct fcrypt *fcr, struct crypt_n_kop *kop, int count) 1166 { 1167 struct cryptkop *krp = NULL; 1168 int error = EINVAL; 1169 int in, out, size, i, req; 1170 1171 for (req = 0; req < count; req++) { 1172 if (kop[req].crk_iparams + kop[req].crk_oparams > CRK_MAXPARAM) 1173 return EFBIG; 1174 1175 in = kop[req].crk_iparams; 1176 out = kop[req].crk_oparams; 1177 switch (kop[req].crk_op) { 1178 case CRK_MOD_EXP: 1179 if (in == 3 && out == 1) 1180 break; 1181 kop[req].crk_status = EINVAL; 1182 continue; 1183 case CRK_MOD_EXP_CRT: 1184 if (in == 6 && out == 1) 1185 break; 1186 kop[req].crk_status = EINVAL; 1187 continue; 1188 case CRK_DSA_SIGN: 1189 if (in == 5 && out == 2) 1190 break; 1191 kop[req].crk_status = EINVAL; 1192 continue; 1193 case CRK_DSA_VERIFY: 1194 if (in == 7 && out == 0) 1195 break; 1196 kop[req].crk_status = EINVAL; 1197 continue; 1198 case CRK_DH_COMPUTE_KEY: 1199 if (in == 3 && out == 1) 1200 break; 1201 kop[req].crk_status = EINVAL; 1202 continue; 1203 case CRK_MOD_ADD: 1204 if (in == 3 && out == 1) 1205 break; 1206 kop[req].crk_status = EINVAL; 1207 continue; 1208 case CRK_MOD_ADDINV: 1209 if (in == 2 && out == 1) 1210 break; 1211 kop[req].crk_status = EINVAL; 1212 continue; 1213 case CRK_MOD_SUB: 1214 if (in == 3 && out == 1) 1215 break; 1216 kop[req].crk_status = EINVAL; 1217 continue; 1218 case CRK_MOD_MULT: 1219 if (in == 3 && out == 1) 1220 break; 1221 kop[req].crk_status = EINVAL; 1222 continue; 1223 case CRK_MOD_MULTINV: 1224 if (in == 2 && out == 1) 1225 break; 1226 kop[req].crk_status = EINVAL; 1227 continue; 1228 case CRK_MOD: 1229 if (in == 2 && out == 1) 1230 break; 1231 kop[req].crk_status = EINVAL; 1232 continue; 1233 default: 1234 kop[req].crk_status = EINVAL; 1235 continue; 1236 } 1237 1238 krp = pool_get(&cryptkop_pool, PR_WAITOK); 1239 (void)memset(krp, 0, sizeof *krp); 1240 cv_init(&krp->krp_cv, "crykdev"); 1241 krp->krp_op = kop[req].crk_op; 1242 krp->krp_status = kop[req].crk_status; 1243 krp->krp_iparams = kop[req].crk_iparams; 1244 krp->krp_oparams = kop[req].crk_oparams; 1245 krp->krp_status = 0; 1246 krp->krp_callback = 1247 (int (*) (struct cryptkop *)) cryptodevkey_mcb; 1248 (void)memcpy(krp->crk_param, kop[req].crk_param, 1249 sizeof(kop[req].crk_param)); 1250 1251 krp->krp_flags = CRYPTO_F_CBIMM; 1252 1253 for (i = 0; i < CRK_MAXPARAM; i++) 1254 krp->krp_param[i].crp_nbits = 1255 kop[req].crk_param[i].crp_nbits; 1256 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1257 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1258 if (size == 0) 1259 continue; 1260 krp->krp_param[i].crp_p = 1261 kmem_alloc(size, KM_SLEEP); 1262 if (i >= krp->krp_iparams) 1263 continue; 1264 kop[req].crk_status = 1265 copyin(kop[req].crk_param[i].crp_p, 1266 krp->krp_param[i].crp_p, size); 1267 if (kop[req].crk_status) 1268 goto fail; 1269 } 1270 krp->fcrp = fcr; 1271 1272 kop[req].crk_reqid = atomic_inc_32_nv(&(fcr->requestid)); 1273 krp->krp_reqid = kop[req].crk_reqid; 1274 krp->krp_usropaque = kop[req].crk_opaque; 1275 1276 kop[req].crk_status = crypto_kdispatch(krp); 1277 if (kop[req].crk_status != 0) { 1278 goto fail; 1279 } 1280 1281 fail: 1282 if(kop[req].crk_status) { 1283 if (krp) { 1284 kop[req].crk_status = krp->krp_status; 1285 for (i = 0; i < CRK_MAXPARAM; i++) { 1286 struct crparam *kp = 1287 &(krp->krp_param[i]); 1288 if (kp->crp_p) { 1289 size = (kp->crp_nbits + 7) / 8; 1290 KASSERT(size > 0); 1291 memset(kp->crp_p, 0, size); 1292 kmem_free(kp->crp_p, size); 1293 } 1294 } 1295 cv_destroy(&krp->krp_cv); 1296 pool_put(&cryptkop_pool, krp); 1297 } 1298 } 1299 error = 0; 1300 } 1301 DPRINTF(("cryptodev_key: error=0x%08x\n", error)); 1302 return error; 1303 } 1304 1305 static int 1306 cryptodev_session(struct fcrypt *fcr, struct session_op *sop) 1307 { 1308 struct cryptoini cria, crie; 1309 struct enc_xform *txform = NULL; 1310 struct auth_hash *thash = NULL; 1311 struct csession *cse; 1312 u_int64_t sid; 1313 int error = 0; 1314 1315 /* XXX there must be a way to not embed the list of xforms here */ 1316 switch (sop->cipher) { 1317 case 0: 1318 break; 1319 case CRYPTO_DES_CBC: 1320 txform = &enc_xform_des; 1321 break; 1322 case CRYPTO_3DES_CBC: 1323 txform = &enc_xform_3des; 1324 break; 1325 case CRYPTO_BLF_CBC: 1326 txform = &enc_xform_blf; 1327 break; 1328 case CRYPTO_CAST_CBC: 1329 txform = &enc_xform_cast5; 1330 case CRYPTO_SKIPJACK_CBC: 1331 txform = &enc_xform_skipjack; 1332 break; 1333 case CRYPTO_AES_CBC: 1334 txform = &enc_xform_rijndael128; 1335 break; 1336 case CRYPTO_NULL_CBC: 1337 txform = &enc_xform_null; 1338 break; 1339 case CRYPTO_ARC4: 1340 txform = &enc_xform_arc4; 1341 break; 1342 default: 1343 DPRINTF(("Invalid cipher %d\n", sop->cipher)); 1344 return EINVAL; 1345 } 1346 1347 switch (sop->mac) { 1348 case 0: 1349 break; 1350 case CRYPTO_MD5_HMAC: 1351 thash = &auth_hash_hmac_md5; 1352 break; 1353 case CRYPTO_SHA1_HMAC: 1354 thash = &auth_hash_hmac_sha1; 1355 break; 1356 case CRYPTO_MD5_HMAC_96: 1357 thash = &auth_hash_hmac_md5_96; 1358 break; 1359 case CRYPTO_SHA1_HMAC_96: 1360 thash = &auth_hash_hmac_sha1_96; 1361 break; 1362 case CRYPTO_SHA2_HMAC: 1363 /* XXX switching on key length seems questionable */ 1364 if (sop->mackeylen == auth_hash_hmac_sha2_256.keysize) { 1365 thash = &auth_hash_hmac_sha2_256; 1366 } else if (sop->mackeylen == auth_hash_hmac_sha2_384.keysize) { 1367 thash = &auth_hash_hmac_sha2_384; 1368 } else if (sop->mackeylen == auth_hash_hmac_sha2_512.keysize) { 1369 thash = &auth_hash_hmac_sha2_512; 1370 } else { 1371 DPRINTF(("Invalid mackeylen %d\n", sop->mackeylen)); 1372 return EINVAL; 1373 } 1374 break; 1375 case CRYPTO_RIPEMD160_HMAC: 1376 thash = &auth_hash_hmac_ripemd_160; 1377 break; 1378 case CRYPTO_RIPEMD160_HMAC_96: 1379 thash = &auth_hash_hmac_ripemd_160_96; 1380 break; 1381 case CRYPTO_MD5: 1382 thash = &auth_hash_md5; 1383 break; 1384 case CRYPTO_SHA1: 1385 thash = &auth_hash_sha1; 1386 break; 1387 case CRYPTO_NULL_HMAC: 1388 thash = &auth_hash_null; 1389 break; 1390 default: 1391 DPRINTF(("Invalid mac %d\n", sop->mac)); 1392 return EINVAL; 1393 } 1394 1395 memset(&crie, 0, sizeof(crie)); 1396 memset(&cria, 0, sizeof(cria)); 1397 1398 if (txform) { 1399 crie.cri_alg = txform->type; 1400 crie.cri_klen = sop->keylen * 8; 1401 if (sop->keylen > txform->maxkey || 1402 sop->keylen < txform->minkey) { 1403 DPRINTF(("keylen %d not in [%d,%d]\n", 1404 sop->keylen, txform->minkey, txform->maxkey)); 1405 error = EINVAL; 1406 goto bail; 1407 } 1408 1409 crie.cri_key = malloc(crie.cri_klen / 8, M_XDATA, M_WAITOK); 1410 if ((error = copyin(sop->key, crie.cri_key, crie.cri_klen / 8))) 1411 goto bail; 1412 if (thash) 1413 crie.cri_next = &cria; /* XXX forces enc then hash? */ 1414 } 1415 1416 if (thash) { 1417 cria.cri_alg = thash->type; 1418 cria.cri_klen = sop->mackeylen * 8; 1419 if (sop->mackeylen != thash->keysize) { 1420 DPRINTF(("mackeylen %d != keysize %d\n", 1421 sop->mackeylen, thash->keysize)); 1422 error = EINVAL; 1423 goto bail; 1424 } 1425 if (cria.cri_klen) { 1426 cria.cri_key = malloc(cria.cri_klen / 8, M_XDATA, 1427 M_WAITOK); 1428 if ((error = copyin(sop->mackey, cria.cri_key, 1429 cria.cri_klen / 8))) { 1430 goto bail; 1431 } 1432 } 1433 } 1434 /* crypto_newsession requires that we hold the mutex. */ 1435 mutex_spin_enter(&crypto_mtx); 1436 error = crypto_newsession(&sid, (txform ? &crie : &cria), 1437 crypto_devallowsoft); 1438 if (!error) { 1439 cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen, 1440 cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, 1441 txform, thash); 1442 if (cse != NULL) { 1443 sop->ses = cse->ses; 1444 } else { 1445 DPRINTF(("csecreate failed\n")); 1446 crypto_freesession(sid); 1447 error = EINVAL; 1448 } 1449 } else { 1450 DPRINTF(("SIOCSESSION violates kernel parameters %d\n", 1451 error)); 1452 } 1453 mutex_spin_exit(&crypto_mtx); 1454 bail: 1455 if (error) { 1456 if (crie.cri_key) { 1457 memset(crie.cri_key, 0, crie.cri_klen / 8); 1458 free(crie.cri_key, M_XDATA); 1459 } 1460 if (cria.cri_key) { 1461 memset(cria.cri_key, 0, cria.cri_klen / 8); 1462 free(cria.cri_key, M_XDATA); 1463 } 1464 } 1465 return error; 1466 } 1467 1468 static int 1469 cryptodev_msession(struct fcrypt *fcr, struct session_n_op *sn_ops, 1470 int count) 1471 { 1472 int i; 1473 1474 for (i = 0; i < count; i++, sn_ops++) { 1475 struct session_op s_op; 1476 s_op.cipher = sn_ops->cipher; 1477 s_op.mac = sn_ops->mac; 1478 s_op.keylen = sn_ops->keylen; 1479 s_op.key = sn_ops->key; 1480 s_op.mackeylen = sn_ops->mackeylen; 1481 s_op.mackey = sn_ops->mackey; 1482 1483 sn_ops->status = cryptodev_session(fcr, &s_op); 1484 sn_ops->ses = s_op.ses; 1485 } 1486 1487 return 0; 1488 } 1489 1490 static int 1491 cryptodev_msessionfin(struct fcrypt *fcr, int count, u_int32_t *sesid) 1492 { 1493 struct csession *cse; 1494 int req, error = 0; 1495 1496 mutex_spin_enter(&crypto_mtx); 1497 for(req = 0; req < count; req++) { 1498 cse = csefind(fcr, sesid[req]); 1499 if (cse == NULL) 1500 continue; 1501 csedelete(fcr, cse); 1502 error = csefree(cse); 1503 } 1504 mutex_spin_exit(&crypto_mtx); 1505 return 0; 1506 } 1507 1508 /* 1509 * collect as many completed requests as are availble, or count completed 1510 * requests whichever is less. 1511 * return the number of requests. 1512 */ 1513 static int 1514 cryptodev_getmstatus(struct fcrypt *fcr, struct crypt_result *crypt_res, 1515 int count) 1516 { 1517 struct cryptop *crp = NULL; 1518 struct cryptkop *krp = NULL; 1519 struct csession *cse; 1520 int i, size, req = 0; 1521 int completed=0; 1522 1523 /* On stack so nobody else can grab them -- no locking */ 1524 SLIST_HEAD(, cryptop) crp_delfree_l = 1525 SLIST_HEAD_INITIALIZER(crp_delfree_l); 1526 SLIST_HEAD(, cryptkop) krp_delfree_l = 1527 SLIST_HEAD_INITIALIZER(krp_delfree_l); 1528 1529 mutex_spin_enter(&crypto_mtx); 1530 1531 /* at this point we do not know which response user is requesting for 1532 * (symmetric or asymmetric) so we copyout one from each i.e if the 1533 * count is 2 then 1 from symmetric and 1 from asymmetric queue and 1534 * if 3 then 2 symmetric and 1 asymmetric and so on */ 1535 for(; req < count ;) { 1536 crp = TAILQ_FIRST(&fcr->crp_ret_mq); 1537 if(crp) { 1538 cse = (struct csession *)crp->crp_opaque; 1539 crypt_res[req].reqid = crp->crp_reqid; 1540 crypt_res[req].opaque = crp->crp_usropaque; 1541 completed++; 1542 cse = csefind(fcr, cse->ses); 1543 if (cse == NULL) { 1544 DPRINTF(("csefind failed\n")); 1545 crypt_res[req].status = EINVAL; 1546 goto bail; 1547 } 1548 1549 if (crp->crp_etype != 0) { 1550 crypt_res[req].status = crp->crp_etype; 1551 goto bail; 1552 } 1553 1554 if (cse->error) { 1555 crypt_res[req].status = cse->error; 1556 goto bail; 1557 } 1558 1559 if (crp->dst && (crypt_res[req].status = 1560 copyout(crp->uio.uio_iov[0].iov_base, crp->dst, 1561 crp->len))) 1562 goto bail; 1563 1564 if (crp->mac && (crypt_res[req].status = 1565 copyout(crp->crp_mac, crp->mac, 1566 cse->thash->authsize))) 1567 goto bail; 1568 bail: 1569 TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next); 1570 SLIST_INSERT_HEAD(&crp_delfree_l, crp, 1571 crp_qun.crp_lnext); 1572 req++; 1573 } 1574 1575 if(req < count) { 1576 krp = TAILQ_FIRST(&fcr->crp_ret_mkq); 1577 if (krp) { 1578 crypt_res[req].reqid = krp->krp_reqid; 1579 crypt_res[req].opaque = krp->krp_usropaque; 1580 completed++; 1581 if (krp->krp_status != 0) { 1582 DPRINTF(("cryptodev_key: " 1583 "krp->krp_status 0x%08x\n", 1584 krp->krp_status)); 1585 crypt_res[req].status = krp->krp_status; 1586 goto fail; 1587 } 1588 1589 for (i = krp->krp_iparams; i < krp->krp_iparams 1590 + krp->krp_oparams; i++) { 1591 size = (krp->krp_param[i].crp_nbits 1592 + 7) / 8; 1593 if (size == 0) 1594 continue; 1595 crypt_res[req].status = copyout 1596 (krp->krp_param[i].crp_p, 1597 krp->crk_param[i].crp_p, size); 1598 if (crypt_res[req].status) { 1599 DPRINTF(("cryptodev_key: " 1600 "copyout oparam %d failed, " 1601 "error=%d\n", 1602 i - krp->krp_iparams, 1603 crypt_res[req].status)); 1604 goto fail; 1605 } 1606 } 1607 fail: 1608 TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next); 1609 /* not sure what to do for this */ 1610 /* kop[req].crk_status = krp->krp_status; */ 1611 SLIST_INSERT_HEAD(&krp_delfree_l, krp, 1612 krp_qun.krp_lnext); 1613 } 1614 req++; 1615 } 1616 } 1617 mutex_spin_exit(&crypto_mtx); 1618 1619 while(!SLIST_EMPTY(&crp_delfree_l)) { 1620 crp = SLIST_FIRST(&crp_delfree_l); 1621 SLIST_REMOVE_HEAD(&crp_delfree_l, crp_qun.crp_lnext); 1622 kmem_free(crp->uio.uio_iov[0].iov_base, 1623 crp->uio.uio_iov[0].iov_len); 1624 crypto_freereq(crp); 1625 } 1626 1627 while(!SLIST_EMPTY(&krp_delfree_l)) { 1628 krp = SLIST_FIRST(&krp_delfree_l); 1629 for (i = 0; i < CRK_MAXPARAM; i++) { 1630 struct crparam *kp = &(krp->krp_param[i]); 1631 if (kp->crp_p) { 1632 size = (kp->crp_nbits + 7) / 8; 1633 KASSERT(size > 0); 1634 (void)memset(kp->crp_p, 0, size); 1635 kmem_free(kp->crp_p, size); 1636 } 1637 } 1638 SLIST_REMOVE_HEAD(&krp_delfree_l, krp_qun.krp_lnext); 1639 cv_destroy(&krp->krp_cv); 1640 pool_put(&cryptkop_pool, krp); 1641 } 1642 return completed; 1643 } 1644 1645 static int 1646 cryptodev_getstatus (struct fcrypt *fcr, struct crypt_result *crypt_res) 1647 { 1648 struct cryptop *crp = NULL, *cnext; 1649 struct cryptkop *krp = NULL, *knext; 1650 struct csession *cse; 1651 int i, size, req = 0; 1652 1653 mutex_spin_enter(&crypto_mtx); 1654 /* Here we dont know for which request the user is requesting the 1655 * response so checking in both the queues */ 1656 TAILQ_FOREACH_SAFE(crp, &fcr->crp_ret_mq, crp_next, cnext) { 1657 if(crp && (crp->crp_reqid == crypt_res->reqid)) { 1658 cse = (struct csession *)crp->crp_opaque; 1659 crypt_res->opaque = crp->crp_usropaque; 1660 cse = csefind(fcr, cse->ses); 1661 if (cse == NULL) { 1662 DPRINTF(("csefind failed\n")); 1663 crypt_res->status = EINVAL; 1664 goto bail; 1665 } 1666 1667 if (crp->crp_etype != 0) { 1668 crypt_res->status = crp->crp_etype; 1669 goto bail; 1670 } 1671 1672 if (cse->error) { 1673 crypt_res->status = cse->error; 1674 goto bail; 1675 } 1676 1677 if (crp->dst && (crypt_res->status = 1678 copyout(crp->uio.uio_iov[0].iov_base, 1679 crp->dst, crp->len))) 1680 goto bail; 1681 1682 if (crp->mac && (crypt_res->status = 1683 copyout(crp->crp_mac, crp->mac, 1684 cse->thash->authsize))) 1685 goto bail; 1686 bail: 1687 TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next); 1688 1689 mutex_spin_exit(&crypto_mtx); 1690 crypto_freereq(crp); 1691 return 0; 1692 } 1693 } 1694 1695 TAILQ_FOREACH_SAFE(krp, &fcr->crp_ret_mkq, krp_next, knext) { 1696 if(krp && (krp->krp_reqid == crypt_res->reqid)) { 1697 crypt_res[req].opaque = krp->krp_usropaque; 1698 if (krp->krp_status != 0) { 1699 DPRINTF(("cryptodev_key: " 1700 "krp->krp_status 0x%08x\n", 1701 krp->krp_status)); 1702 crypt_res[req].status = krp->krp_status; 1703 goto fail; 1704 } 1705 1706 for (i = krp->krp_iparams; i < krp->krp_iparams + 1707 krp->krp_oparams; i++) { 1708 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1709 if (size == 0) 1710 continue; 1711 crypt_res[req].status = copyout( 1712 krp->krp_param[i].crp_p, 1713 krp->crk_param[i].crp_p, size); 1714 if (crypt_res[req].status) { 1715 DPRINTF(("cryptodev_key: copyout oparam" 1716 "%d failed, error=%d\n", 1717 i - krp->krp_iparams, 1718 crypt_res[req].status)); 1719 goto fail; 1720 } 1721 } 1722 fail: 1723 TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next); 1724 mutex_spin_exit(&crypto_mtx); 1725 /* not sure what to do for this */ 1726 /* kop[req].crk_status = krp->krp_status; */ 1727 for (i = 0; i < CRK_MAXPARAM; i++) { 1728 struct crparam *kp = &(krp->krp_param[i]); 1729 if (kp->crp_p) { 1730 size = (kp->crp_nbits + 7) / 8; 1731 KASSERT(size > 0); 1732 memset(kp->crp_p, 0, size); 1733 kmem_free(kp->crp_p, size); 1734 } 1735 } 1736 cv_destroy(&krp->krp_cv); 1737 pool_put(&cryptkop_pool, krp); 1738 return 0; 1739 } 1740 } 1741 mutex_spin_exit(&crypto_mtx); 1742 return EINPROGRESS; 1743 } 1744 1745 static int 1746 cryptof_poll(struct file *fp, int events) 1747 { 1748 struct fcrypt *fcr = (struct fcrypt *)fp->f_data; 1749 int revents = 0; 1750 1751 if (!(events & (POLLIN | POLLRDNORM))) { 1752 /* only support read and POLLIN */ 1753 return 0; 1754 } 1755 1756 mutex_spin_enter(&crypto_mtx); 1757 if (TAILQ_EMPTY(&fcr->crp_ret_mq) && TAILQ_EMPTY(&fcr->crp_ret_mkq)) { 1758 /* no completed requests pending, save the poll for later */ 1759 selrecord(curlwp, &fcr->sinfo); 1760 } else { 1761 /* let the app(s) know that there are completed requests */ 1762 revents = events & (POLLIN | POLLRDNORM); 1763 } 1764 mutex_spin_exit(&crypto_mtx); 1765 1766 return revents; 1767 } 1768 1769 /* 1770 * Pseudo-device initialization routine for /dev/crypto 1771 */ 1772 void cryptoattach(int); 1773 1774 void 1775 cryptoattach(int num) 1776 { 1777 pool_init(&fcrpl, sizeof(struct fcrypt), 0, 0, 0, "fcrpl", 1778 NULL, IPL_NET); /* XXX IPL_NET ("splcrypto") */ 1779 pool_init(&csepl, sizeof(struct csession), 0, 0, 0, "csepl", 1780 NULL, IPL_NET); /* XXX IPL_NET ("splcrypto") */ 1781 1782 /* 1783 * Preallocate space for 64 users, with 5 sessions each. 1784 * (consider that a TLS protocol session requires at least 1785 * 3DES, MD5, and SHA1 (both hashes are used in the PRF) for 1786 * the negotiation, plus HMAC_SHA1 for the actual SSL records, 1787 * consuming one session here for each algorithm. 1788 */ 1789 pool_prime(&fcrpl, 64); 1790 pool_prime(&csepl, 64 * 5); 1791 } 1792
Indexes created Thu Oct 02 14:10:14 GMT 2025