crypto.c revision 1.78.2.1
1 /* $NetBSD: crypto.c,v 1.78.2.1 2017/06/22 05:36:41 snj Exp $ */ 2 /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.5 2003/02/26 00:14:05 sam Exp $ */ 3 /* $OpenBSD: crypto.c,v 1.41 2002/07/17 23:52:38 art Exp $ */ 4 5 /*- 6 * Copyright (c) 2008 The NetBSD Foundation, Inc. 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by Coyote Point Systems, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * The author of this code is Angelos D. Keromytis (angelos (at) cis.upenn.edu) 36 * 37 * This code was written by Angelos D. Keromytis in Athens, Greece, in 38 * February 2000. Network Security Technologies Inc. (NSTI) kindly 39 * supported the development of this code. 40 * 41 * Copyright (c) 2000, 2001 Angelos D. Keromytis 42 * 43 * Permission to use, copy, and modify this software with or without fee 44 * is hereby granted, provided that this entire notice is included in 45 * all source code copies of any software which is or includes a copy or 46 * modification of this software. 47 * 48 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 49 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 50 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 51 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 52 * PURPOSE. 53 */ 54 55 #include <sys/cdefs.h> 56 __KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.78.2.1 2017/06/22 05:36:41 snj Exp $"); 57 58 #include <sys/param.h> 59 #include <sys/reboot.h> 60 #include <sys/systm.h> 61 #include <sys/malloc.h> 62 #include <sys/proc.h> 63 #include <sys/pool.h> 64 #include <sys/kthread.h> 65 #include <sys/once.h> 66 #include <sys/sysctl.h> 67 #include <sys/intr.h> 68 #include <sys/errno.h> 69 #include <sys/module.h> 70 71 #if defined(_KERNEL_OPT) 72 #include "opt_ocf.h" 73 #endif 74 75 #include <opencrypto/cryptodev.h> 76 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 77 78 static kmutex_t crypto_q_mtx; 79 static kmutex_t crypto_ret_q_mtx; 80 static kcondvar_t cryptoret_cv; 81 82 /* below are kludges for residual code wrtitten to FreeBSD interfaces */ 83 #define SWI_CRYPTO 17 84 #define register_swi(lvl, fn) \ 85 softint_establish(SOFTINT_NET|SOFTINT_MPSAFE, (void (*)(void *))fn, NULL) 86 #define unregister_swi(lvl, fn) softint_disestablish(softintr_cookie) 87 #define setsoftcrypto(x) \ 88 do{ \ 89 kpreempt_disable(); \ 90 softint_schedule(x); \ 91 kpreempt_enable(); \ 92 }while(0) 93 94 int crypto_ret_q_check(struct cryptop *); 95 96 /* 97 * Crypto drivers register themselves by allocating a slot in the 98 * crypto_drivers table with crypto_get_driverid() and then registering 99 * each algorithm they support with crypto_register() and crypto_kregister(). 100 */ 101 static kmutex_t crypto_drv_mtx; 102 /* Don't directly access crypto_drivers[i], use crypto_checkdriver(i). */ 103 static struct cryptocap *crypto_drivers; 104 static int crypto_drivers_num; 105 static void *softintr_cookie; 106 static int crypto_exit_flag; 107 108 /* 109 * There are two queues for crypto requests; one for symmetric (e.g. 110 * cipher) operations and one for asymmetric (e.g. MOD) operations. 111 * See below for how synchronization is handled. 112 */ 113 static TAILQ_HEAD(,cryptop) crp_q = /* request queues */ 114 TAILQ_HEAD_INITIALIZER(crp_q); 115 static TAILQ_HEAD(,cryptkop) crp_kq = 116 TAILQ_HEAD_INITIALIZER(crp_kq); 117 118 /* 119 * There are two queues for processing completed crypto requests; one 120 * for the symmetric and one for the asymmetric ops. We only need one 121 * but have two to avoid type futzing (cryptop vs. cryptkop). See below 122 * for how synchronization is handled. 123 */ 124 static TAILQ_HEAD(crprethead, cryptop) crp_ret_q = /* callback queues */ 125 TAILQ_HEAD_INITIALIZER(crp_ret_q); 126 static TAILQ_HEAD(krprethead, cryptkop) crp_ret_kq = 127 TAILQ_HEAD_INITIALIZER(crp_ret_kq); 128 129 #define DEFINIT_CRYPTO_Q_LEN(name) \ 130 static int crypto_##name##_len = 0 131 132 #define DEFINIT_CRYPTO_Q_DROPS(name) \ 133 static int crypto_##name##_drops = 0 134 135 #define DEFINIT_CRYPTO_Q_MAXLEN(name, defval) \ 136 static int crypto_##name##_maxlen = defval 137 138 #define CRYPTO_Q_INC(name) \ 139 do { \ 140 crypto_##name##_len++; \ 141 } while(0); 142 143 #define CRYPTO_Q_DEC(name) \ 144 do { \ 145 crypto_##name##_len--; \ 146 } while(0); 147 148 #define CRYPTO_Q_INC_DROPS(name) \ 149 do { \ 150 crypto_##name##_drops++; \ 151 } while(0); 152 153 #define CRYPTO_Q_IS_FULL(name) \ 154 (crypto_##name##_maxlen > 0 \ 155 && (crypto_##name##_len > crypto_##name##_maxlen)) 156 157 /* 158 * current queue length. 159 */ 160 DEFINIT_CRYPTO_Q_LEN(crp_ret_q); 161 DEFINIT_CRYPTO_Q_LEN(crp_ret_kq); 162 163 /* 164 * queue dropped count. 165 */ 166 DEFINIT_CRYPTO_Q_DROPS(crp_ret_q); 167 DEFINIT_CRYPTO_Q_DROPS(crp_ret_kq); 168 169 #ifndef CRYPTO_RET_Q_MAXLEN 170 #define CRYPTO_RET_Q_MAXLEN 0 171 #endif 172 #ifndef CRYPTO_RET_KQ_MAXLEN 173 #define CRYPTO_RET_KQ_MAXLEN 0 174 #endif 175 /* 176 * queue length limit. 177 * default value is 0. <=0 means unlimited. 178 */ 179 DEFINIT_CRYPTO_Q_MAXLEN(crp_ret_q, CRYPTO_RET_Q_MAXLEN); 180 DEFINIT_CRYPTO_Q_MAXLEN(crp_ret_kq, CRYPTO_RET_KQ_MAXLEN); 181 182 /* 183 * TODO: 184 * make percpu 185 */ 186 static int 187 sysctl_opencrypto_q_len(SYSCTLFN_ARGS) 188 { 189 int error; 190 191 error = sysctl_lookup(SYSCTLFN_CALL(rnode)); 192 if (error || newp == NULL) 193 return error; 194 195 return 0; 196 } 197 198 /* 199 * TODO: 200 * make percpu 201 */ 202 static int 203 sysctl_opencrypto_q_drops(SYSCTLFN_ARGS) 204 { 205 int error; 206 207 error = sysctl_lookup(SYSCTLFN_CALL(rnode)); 208 if (error || newp == NULL) 209 return error; 210 211 return 0; 212 } 213 214 /* 215 * need to make percpu? 216 */ 217 static int 218 sysctl_opencrypto_q_maxlen(SYSCTLFN_ARGS) 219 { 220 int error; 221 222 error = sysctl_lookup(SYSCTLFN_CALL(rnode)); 223 if (error || newp == NULL) 224 return error; 225 226 return 0; 227 } 228 229 /* 230 * Crypto op and desciptor data structures are allocated 231 * from separate private zones(FreeBSD)/pools(netBSD/OpenBSD) . 232 */ 233 struct pool cryptop_pool; 234 struct pool cryptodesc_pool; 235 struct pool cryptkop_pool; 236 237 int crypto_usercrypto = 1; /* userland may open /dev/crypto */ 238 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 239 /* 240 * cryptodevallowsoft is (intended to be) sysctl'able, controlling 241 * access to hardware versus software transforms as below: 242 * 243 * crypto_devallowsoft < 0: Force userlevel requests to use software 244 * transforms, always 245 * crypto_devallowsoft = 0: Use hardware if present, grant userlevel 246 * requests for non-accelerated transforms 247 * (handling the latter in software) 248 * crypto_devallowsoft > 0: Allow user requests only for transforms which 249 * are hardware-accelerated. 250 */ 251 int crypto_devallowsoft = 1; /* only use hardware crypto */ 252 253 static void 254 sysctl_opencrypto_setup(struct sysctllog **clog) 255 { 256 const struct sysctlnode *ocnode; 257 const struct sysctlnode *retqnode, *retkqnode; 258 259 sysctl_createv(clog, 0, NULL, NULL, 260 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 261 CTLTYPE_INT, "usercrypto", 262 SYSCTL_DESCR("Enable/disable user-mode access to " 263 "crypto support"), 264 NULL, 0, &crypto_usercrypto, 0, 265 CTL_KERN, CTL_CREATE, CTL_EOL); 266 sysctl_createv(clog, 0, NULL, NULL, 267 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 268 CTLTYPE_INT, "userasymcrypto", 269 SYSCTL_DESCR("Enable/disable user-mode access to " 270 "asymmetric crypto support"), 271 NULL, 0, &crypto_userasymcrypto, 0, 272 CTL_KERN, CTL_CREATE, CTL_EOL); 273 sysctl_createv(clog, 0, NULL, NULL, 274 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 275 CTLTYPE_INT, "cryptodevallowsoft", 276 SYSCTL_DESCR("Enable/disable use of software " 277 "asymmetric crypto support"), 278 NULL, 0, &crypto_devallowsoft, 0, 279 CTL_KERN, CTL_CREATE, CTL_EOL); 280 281 sysctl_createv(clog, 0, NULL, &ocnode, 282 CTLFLAG_PERMANENT, 283 CTLTYPE_NODE, "opencrypto", 284 SYSCTL_DESCR("opencrypto related entries"), 285 NULL, 0, NULL, 0, 286 CTL_CREATE, CTL_EOL); 287 288 sysctl_createv(clog, 0, &ocnode, &retqnode, 289 CTLFLAG_PERMANENT, 290 CTLTYPE_NODE, "crypto_ret_q", 291 SYSCTL_DESCR("crypto_ret_q related entries"), 292 NULL, 0, NULL, 0, 293 CTL_CREATE, CTL_EOL); 294 sysctl_createv(clog, 0, &retqnode, NULL, 295 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 296 CTLTYPE_INT, "len", 297 SYSCTL_DESCR("Current queue length"), 298 sysctl_opencrypto_q_len, 0, 299 (void *)&crypto_crp_ret_q_len, 0, 300 CTL_CREATE, CTL_EOL); 301 sysctl_createv(clog, 0, &retqnode, NULL, 302 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 303 CTLTYPE_INT, "drops", 304 SYSCTL_DESCR("Crypto requests dropped due to full ret queue"), 305 sysctl_opencrypto_q_drops, 0, 306 (void *)&crypto_crp_ret_q_drops, 0, 307 CTL_CREATE, CTL_EOL); 308 sysctl_createv(clog, 0, &retqnode, NULL, 309 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 310 CTLTYPE_INT, "maxlen", 311 SYSCTL_DESCR("Maximum allowed queue length"), 312 sysctl_opencrypto_q_maxlen, 0, 313 (void *)&crypto_crp_ret_q_maxlen, 0, 314 CTL_CREATE, CTL_EOL); 315 316 sysctl_createv(clog, 0, &ocnode, &retkqnode, 317 CTLFLAG_PERMANENT, 318 CTLTYPE_NODE, "crypto_ret_kq", 319 SYSCTL_DESCR("crypto_ret_kq related entries"), 320 NULL, 0, NULL, 0, 321 CTL_CREATE, CTL_EOL); 322 sysctl_createv(clog, 0, &retkqnode, NULL, 323 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 324 CTLTYPE_INT, "len", 325 SYSCTL_DESCR("Current queue length"), 326 sysctl_opencrypto_q_len, 0, 327 (void *)&crypto_crp_ret_kq_len, 0, 328 CTL_CREATE, CTL_EOL); 329 sysctl_createv(clog, 0, &retkqnode, NULL, 330 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 331 CTLTYPE_INT, "drops", 332 SYSCTL_DESCR("Crypto requests dropped due to full ret queue"), 333 sysctl_opencrypto_q_drops, 0, 334 (void *)&crypto_crp_ret_kq_drops, 0, 335 CTL_CREATE, CTL_EOL); 336 sysctl_createv(clog, 0, &retkqnode, NULL, 337 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 338 CTLTYPE_INT, "maxlen", 339 SYSCTL_DESCR("Maximum allowed queue length"), 340 sysctl_opencrypto_q_maxlen, 0, 341 (void *)&crypto_crp_ret_kq_maxlen, 0, 342 CTL_CREATE, CTL_EOL); 343 } 344 345 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 346 347 /* 348 * Synchronization: read carefully, this is non-trivial. 349 * 350 * Crypto requests are submitted via crypto_dispatch. Typically 351 * these come in from network protocols at spl0 (output path) or 352 * spl[,soft]net (input path). 353 * 354 * Requests are typically passed on the driver directly, but they 355 * may also be queued for processing by a software interrupt thread, 356 * cryptointr, that runs at splsoftcrypto. This thread dispatches 357 * the requests to crypto drivers (h/w or s/w) who call crypto_done 358 * when a request is complete. Hardware crypto drivers are assumed 359 * to register their IRQ's as network devices so their interrupt handlers 360 * and subsequent "done callbacks" happen at spl[imp,net]. 361 * 362 * Completed crypto ops are queued for a separate kernel thread that 363 * handles the callbacks at spl0. This decoupling insures the crypto 364 * driver interrupt service routine is not delayed while the callback 365 * takes place and that callbacks are delivered after a context switch 366 * (as opposed to a software interrupt that clients must block). 367 * 368 * This scheme is not intended for SMP machines. 369 */ 370 static void cryptointr(void); /* swi thread to dispatch ops */ 371 static void cryptoret(void); /* kernel thread for callbacks*/ 372 static struct lwp *cryptothread; 373 static int crypto_destroy(bool); 374 static int crypto_invoke(struct cryptop *crp, int hint); 375 static int crypto_kinvoke(struct cryptkop *krp, int hint); 376 377 static struct cryptocap *crypto_checkdriver_lock(u_int32_t); 378 static struct cryptocap *crypto_checkdriver_uninit(u_int32_t); 379 static struct cryptocap *crypto_checkdriver(u_int32_t); 380 static void crypto_driver_lock(struct cryptocap *); 381 static void crypto_driver_unlock(struct cryptocap *); 382 static void crypto_driver_clear(struct cryptocap *); 383 384 static struct cryptostats cryptostats; 385 #ifdef CRYPTO_TIMING 386 static int crypto_timing = 0; 387 #endif 388 389 static struct sysctllog *sysctl_opencrypto_clog; 390 391 static int 392 crypto_init0(void) 393 { 394 int error; 395 396 mutex_init(&crypto_drv_mtx, MUTEX_DEFAULT, IPL_NONE); 397 mutex_init(&crypto_q_mtx, MUTEX_DEFAULT, IPL_NONE); 398 mutex_init(&crypto_ret_q_mtx, MUTEX_DEFAULT, IPL_NET); 399 cv_init(&cryptoret_cv, "crypto_w"); 400 pool_init(&cryptop_pool, sizeof(struct cryptop), 0, 0, 401 0, "cryptop", NULL, IPL_NET); 402 pool_init(&cryptodesc_pool, sizeof(struct cryptodesc), 0, 0, 403 0, "cryptodesc", NULL, IPL_NET); 404 pool_init(&cryptkop_pool, sizeof(struct cryptkop), 0, 0, 405 0, "cryptkop", NULL, IPL_NET); 406 407 crypto_drivers = malloc(CRYPTO_DRIVERS_INITIAL * 408 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 409 if (crypto_drivers == NULL) { 410 printf("crypto_init: cannot malloc driver table\n"); 411 return ENOMEM; 412 } 413 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 414 415 softintr_cookie = register_swi(SWI_CRYPTO, cryptointr); 416 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 417 (void (*)(void *))cryptoret, NULL, &cryptothread, "cryptoret"); 418 if (error) { 419 printf("crypto_init: cannot start cryptoret thread; error %d", 420 error); 421 return crypto_destroy(false); 422 } 423 424 sysctl_opencrypto_setup(&sysctl_opencrypto_clog); 425 426 return 0; 427 } 428 429 int 430 crypto_init(void) 431 { 432 static ONCE_DECL(crypto_init_once); 433 434 return RUN_ONCE(&crypto_init_once, crypto_init0); 435 } 436 437 static int 438 crypto_destroy(bool exit_kthread) 439 { 440 int i; 441 442 if (exit_kthread) { 443 struct cryptocap *cap = NULL; 444 445 /* if we have any in-progress requests, don't unload */ 446 mutex_enter(&crypto_q_mtx); 447 if (!TAILQ_EMPTY(&crp_q) || !TAILQ_EMPTY(&crp_kq)) { 448 mutex_exit(&crypto_q_mtx); 449 return EBUSY; 450 } 451 mutex_exit(&crypto_q_mtx); 452 /* FIXME: 453 * prohibit enqueue to crp_q and crp_kq after here. 454 */ 455 456 mutex_enter(&crypto_drv_mtx); 457 for (i = 0; i < crypto_drivers_num; i++) { 458 cap = crypto_checkdriver(i); 459 if (cap == NULL) 460 continue; 461 if (cap->cc_sessions != 0) { 462 mutex_exit(&crypto_drv_mtx); 463 return EBUSY; 464 } 465 } 466 mutex_exit(&crypto_drv_mtx); 467 /* FIXME: 468 * prohibit touch crypto_drivers[] and each element after here. 469 */ 470 471 mutex_spin_enter(&crypto_ret_q_mtx); 472 /* kick the cryptoret thread and wait for it to exit */ 473 crypto_exit_flag = 1; 474 cv_signal(&cryptoret_cv); 475 476 while (crypto_exit_flag != 0) 477 cv_wait(&cryptoret_cv, &crypto_ret_q_mtx); 478 mutex_spin_exit(&crypto_ret_q_mtx); 479 } 480 481 if (sysctl_opencrypto_clog != NULL) 482 sysctl_teardown(&sysctl_opencrypto_clog); 483 484 unregister_swi(SWI_CRYPTO, cryptointr); 485 486 mutex_enter(&crypto_drv_mtx); 487 if (crypto_drivers != NULL) 488 free(crypto_drivers, M_CRYPTO_DATA); 489 mutex_exit(&crypto_drv_mtx); 490 491 pool_destroy(&cryptop_pool); 492 pool_destroy(&cryptodesc_pool); 493 pool_destroy(&cryptkop_pool); 494 495 cv_destroy(&cryptoret_cv); 496 497 mutex_destroy(&crypto_ret_q_mtx); 498 mutex_destroy(&crypto_q_mtx); 499 mutex_destroy(&crypto_drv_mtx); 500 501 return 0; 502 } 503 504 /* 505 * Create a new session. 506 */ 507 int 508 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 509 { 510 struct cryptoini *cr; 511 struct cryptocap *cap; 512 u_int32_t hid, lid; 513 int err = EINVAL; 514 515 mutex_enter(&crypto_drv_mtx); 516 517 /* 518 * The algorithm we use here is pretty stupid; just use the 519 * first driver that supports all the algorithms we need. 520 * 521 * XXX We need more smarts here (in real life too, but that's 522 * XXX another story altogether). 523 */ 524 525 for (hid = 0; hid < crypto_drivers_num; hid++) { 526 cap = crypto_checkdriver(hid); 527 if (cap == NULL) 528 continue; 529 530 crypto_driver_lock(cap); 531 532 /* 533 * If it's not initialized or has remaining sessions 534 * referencing it, skip. 535 */ 536 if (cap->cc_newsession == NULL || 537 (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) { 538 crypto_driver_unlock(cap); 539 continue; 540 } 541 542 /* Hardware required -- ignore software drivers. */ 543 if (hard > 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE)) { 544 crypto_driver_unlock(cap); 545 continue; 546 } 547 /* Software required -- ignore hardware drivers. */ 548 if (hard < 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) { 549 crypto_driver_unlock(cap); 550 continue; 551 } 552 553 /* See if all the algorithms are supported. */ 554 for (cr = cri; cr; cr = cr->cri_next) 555 if (cap->cc_alg[cr->cri_alg] == 0) { 556 DPRINTF("alg %d not supported\n", cr->cri_alg); 557 break; 558 } 559 560 if (cr == NULL) { 561 /* Ok, all algorithms are supported. */ 562 563 /* 564 * Can't do everything in one session. 565 * 566 * XXX Fix this. We need to inject a "virtual" session layer right 567 * XXX about here. 568 */ 569 570 /* Call the driver initialization routine. */ 571 lid = hid; /* Pass the driver ID. */ 572 err = cap->cc_newsession(cap->cc_arg, &lid, cri); 573 if (err == 0) { 574 (*sid) = hid; 575 (*sid) <<= 32; 576 (*sid) |= (lid & 0xffffffff); 577 (cap->cc_sessions)++; 578 } else { 579 DPRINTF("crypto_drivers[%d].cc_newsession() failed. error=%d\n", 580 hid, err); 581 } 582 crypto_driver_unlock(cap); 583 goto done; 584 /*break;*/ 585 } 586 587 crypto_driver_unlock(cap); 588 } 589 done: 590 mutex_exit(&crypto_drv_mtx); 591 return err; 592 } 593 594 /* 595 * Delete an existing session (or a reserved session on an unregistered 596 * driver). 597 */ 598 int 599 crypto_freesession(u_int64_t sid) 600 { 601 struct cryptocap *cap; 602 int err = 0; 603 604 /* Determine two IDs. */ 605 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(sid)); 606 if (cap == NULL) 607 return ENOENT; 608 609 if (cap->cc_sessions) 610 (cap->cc_sessions)--; 611 612 /* Call the driver cleanup routine, if available. */ 613 if (cap->cc_freesession) 614 err = cap->cc_freesession(cap->cc_arg, sid); 615 else 616 err = 0; 617 618 /* 619 * If this was the last session of a driver marked as invalid, 620 * make the entry available for reuse. 621 */ 622 if ((cap->cc_flags & CRYPTOCAP_F_CLEANUP) && cap->cc_sessions == 0) 623 crypto_driver_clear(cap); 624 625 crypto_driver_unlock(cap); 626 return err; 627 } 628 629 static bool 630 crypto_checkdriver_initialized(const struct cryptocap *cap) 631 { 632 633 return cap->cc_process != NULL || 634 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0 || 635 cap->cc_sessions != 0; 636 } 637 638 /* 639 * Return an unused driver id. Used by drivers prior to registering 640 * support for the algorithms they handle. 641 */ 642 int32_t 643 crypto_get_driverid(u_int32_t flags) 644 { 645 struct cryptocap *newdrv; 646 struct cryptocap *cap = NULL; 647 int i; 648 649 (void)crypto_init(); /* XXX oh, this is foul! */ 650 651 mutex_enter(&crypto_drv_mtx); 652 for (i = 0; i < crypto_drivers_num; i++) { 653 cap = crypto_checkdriver_uninit(i); 654 if (cap == NULL || crypto_checkdriver_initialized(cap)) 655 continue; 656 break; 657 } 658 659 /* Out of entries, allocate some more. */ 660 if (cap == NULL) { 661 /* Be careful about wrap-around. */ 662 if (2 * crypto_drivers_num <= crypto_drivers_num) { 663 mutex_exit(&crypto_drv_mtx); 664 printf("crypto: driver count wraparound!\n"); 665 return -1; 666 } 667 668 newdrv = malloc(2 * crypto_drivers_num * 669 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 670 if (newdrv == NULL) { 671 mutex_exit(&crypto_drv_mtx); 672 printf("crypto: no space to expand driver table!\n"); 673 return -1; 674 } 675 676 memcpy(newdrv, crypto_drivers, 677 crypto_drivers_num * sizeof(struct cryptocap)); 678 679 crypto_drivers_num *= 2; 680 681 free(crypto_drivers, M_CRYPTO_DATA); 682 crypto_drivers = newdrv; 683 684 cap = crypto_checkdriver_uninit(i); 685 KASSERT(cap != NULL); 686 } 687 688 /* NB: state is zero'd on free */ 689 cap->cc_sessions = 1; /* Mark */ 690 cap->cc_flags = flags; 691 mutex_init(&cap->cc_lock, MUTEX_DEFAULT, IPL_NET); 692 693 if (bootverbose) 694 printf("crypto: assign driver %u, flags %u\n", i, flags); 695 696 mutex_exit(&crypto_drv_mtx); 697 698 return i; 699 } 700 701 static struct cryptocap * 702 crypto_checkdriver_lock(u_int32_t hid) 703 { 704 struct cryptocap *cap; 705 706 KASSERT(crypto_drivers != NULL); 707 708 if (hid >= crypto_drivers_num) 709 return NULL; 710 711 cap = &crypto_drivers[hid]; 712 mutex_enter(&cap->cc_lock); 713 return cap; 714 } 715 716 /* 717 * Use crypto_checkdriver_uninit() instead of crypto_checkdriver() below two 718 * situations 719 * - crypto_drivers[] may not be allocated 720 * - crypto_drivers[hid] may not be initialized 721 */ 722 static struct cryptocap * 723 crypto_checkdriver_uninit(u_int32_t hid) 724 { 725 726 KASSERT(mutex_owned(&crypto_drv_mtx)); 727 728 if (crypto_drivers == NULL) 729 return NULL; 730 731 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 732 } 733 734 /* 735 * Use crypto_checkdriver_uninit() instead of crypto_checkdriver() below two 736 * situations 737 * - crypto_drivers[] may not be allocated 738 * - crypto_drivers[hid] may not be initialized 739 */ 740 static struct cryptocap * 741 crypto_checkdriver(u_int32_t hid) 742 { 743 744 KASSERT(mutex_owned(&crypto_drv_mtx)); 745 746 if (crypto_drivers == NULL || hid >= crypto_drivers_num) 747 return NULL; 748 749 struct cryptocap *cap = &crypto_drivers[hid]; 750 return crypto_checkdriver_initialized(cap) ? cap : NULL; 751 } 752 753 static inline void 754 crypto_driver_lock(struct cryptocap *cap) 755 { 756 757 KASSERT(cap != NULL); 758 759 mutex_enter(&cap->cc_lock); 760 } 761 762 static inline void 763 crypto_driver_unlock(struct cryptocap *cap) 764 { 765 766 KASSERT(cap != NULL); 767 768 mutex_exit(&cap->cc_lock); 769 } 770 771 static void 772 crypto_driver_clear(struct cryptocap *cap) 773 { 774 775 if (cap == NULL) 776 return; 777 778 KASSERT(mutex_owned(&cap->cc_lock)); 779 780 cap->cc_sessions = 0; 781 memset(&cap->cc_max_op_len, 0, sizeof(cap->cc_max_op_len)); 782 memset(&cap->cc_alg, 0, sizeof(cap->cc_alg)); 783 memset(&cap->cc_kalg, 0, sizeof(cap->cc_kalg)); 784 cap->cc_flags = 0; 785 cap->cc_qblocked = 0; 786 cap->cc_kqblocked = 0; 787 788 cap->cc_arg = NULL; 789 cap->cc_newsession = NULL; 790 cap->cc_process = NULL; 791 cap->cc_freesession = NULL; 792 cap->cc_kprocess = NULL; 793 } 794 795 /* 796 * Register support for a key-related algorithm. This routine 797 * is called once for each algorithm supported a driver. 798 */ 799 int 800 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 801 int (*kprocess)(void *, struct cryptkop *, int), 802 void *karg) 803 { 804 struct cryptocap *cap; 805 int err; 806 807 mutex_enter(&crypto_drv_mtx); 808 809 cap = crypto_checkdriver_lock(driverid); 810 if (cap != NULL && 811 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 812 /* 813 * XXX Do some performance testing to determine placing. 814 * XXX We probably need an auxiliary data structure that 815 * XXX describes relative performances. 816 */ 817 818 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 819 if (bootverbose) { 820 printf("crypto: driver %u registers key alg %u " 821 " flags %u\n", 822 driverid, 823 kalg, 824 flags 825 ); 826 } 827 828 if (cap->cc_kprocess == NULL) { 829 cap->cc_karg = karg; 830 cap->cc_kprocess = kprocess; 831 } 832 err = 0; 833 } else 834 err = EINVAL; 835 836 mutex_exit(&crypto_drv_mtx); 837 return err; 838 } 839 840 /* 841 * Register support for a non-key-related algorithm. This routine 842 * is called once for each such algorithm supported by a driver. 843 */ 844 int 845 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 846 u_int32_t flags, 847 int (*newses)(void *, u_int32_t*, struct cryptoini*), 848 int (*freeses)(void *, u_int64_t), 849 int (*process)(void *, struct cryptop *, int), 850 void *arg) 851 { 852 struct cryptocap *cap; 853 int err; 854 855 cap = crypto_checkdriver_lock(driverid); 856 if (cap == NULL) 857 return EINVAL; 858 859 /* NB: algorithms are in the range [1..max] */ 860 if (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) { 861 /* 862 * XXX Do some performance testing to determine placing. 863 * XXX We probably need an auxiliary data structure that 864 * XXX describes relative performances. 865 */ 866 867 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 868 cap->cc_max_op_len[alg] = maxoplen; 869 if (bootverbose) { 870 printf("crypto: driver %u registers alg %u " 871 "flags %u maxoplen %u\n", 872 driverid, 873 alg, 874 flags, 875 maxoplen 876 ); 877 } 878 879 if (cap->cc_process == NULL) { 880 cap->cc_arg = arg; 881 cap->cc_newsession = newses; 882 cap->cc_process = process; 883 cap->cc_freesession = freeses; 884 cap->cc_sessions = 0; /* Unmark */ 885 } 886 err = 0; 887 } else 888 err = EINVAL; 889 890 crypto_driver_unlock(cap); 891 892 return err; 893 } 894 895 static int 896 crypto_unregister_locked(struct cryptocap *cap, int alg, bool all) 897 { 898 int i; 899 u_int32_t ses; 900 bool lastalg = true; 901 902 KASSERT(cap != NULL); 903 KASSERT(mutex_owned(&cap->cc_lock)); 904 905 if (alg < CRYPTO_ALGORITHM_MIN || CRYPTO_ALGORITHM_MAX < alg) 906 return EINVAL; 907 908 if (!all && cap->cc_alg[alg] == 0) 909 return EINVAL; 910 911 cap->cc_alg[alg] = 0; 912 cap->cc_max_op_len[alg] = 0; 913 914 if (all) { 915 if (alg != CRYPTO_ALGORITHM_MAX) 916 lastalg = false; 917 } else { 918 /* Was this the last algorithm ? */ 919 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) 920 if (cap->cc_alg[i] != 0) { 921 lastalg = false; 922 break; 923 } 924 } 925 if (lastalg) { 926 ses = cap->cc_sessions; 927 crypto_driver_clear(cap); 928 if (ses != 0) { 929 /* 930 * If there are pending sessions, just mark as invalid. 931 */ 932 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 933 cap->cc_sessions = ses; 934 } 935 } 936 937 return 0; 938 } 939 940 /* 941 * Unregister a crypto driver. If there are pending sessions using it, 942 * leave enough information around so that subsequent calls using those 943 * sessions will correctly detect the driver has been unregistered and 944 * reroute requests. 945 */ 946 int 947 crypto_unregister(u_int32_t driverid, int alg) 948 { 949 int err; 950 struct cryptocap *cap; 951 952 cap = crypto_checkdriver_lock(driverid); 953 err = crypto_unregister_locked(cap, alg, false); 954 crypto_driver_unlock(cap); 955 956 return err; 957 } 958 959 /* 960 * Unregister all algorithms associated with a crypto driver. 961 * If there are pending sessions using it, leave enough information 962 * around so that subsequent calls using those sessions will 963 * correctly detect the driver has been unregistered and reroute 964 * requests. 965 */ 966 int 967 crypto_unregister_all(u_int32_t driverid) 968 { 969 int err, i; 970 struct cryptocap *cap; 971 972 cap = crypto_checkdriver_lock(driverid); 973 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 974 err = crypto_unregister_locked(cap, i, true); 975 if (err) 976 break; 977 } 978 crypto_driver_unlock(cap); 979 980 return err; 981 } 982 983 /* 984 * Clear blockage on a driver. The what parameter indicates whether 985 * the driver is now ready for cryptop's and/or cryptokop's. 986 */ 987 int 988 crypto_unblock(u_int32_t driverid, int what) 989 { 990 struct cryptocap *cap; 991 int needwakeup = 0; 992 993 cap = crypto_checkdriver_lock(driverid); 994 if (cap == NULL) 995 return EINVAL; 996 997 if (what & CRYPTO_SYMQ) { 998 needwakeup |= cap->cc_qblocked; 999 cap->cc_qblocked = 0; 1000 } 1001 if (what & CRYPTO_ASYMQ) { 1002 needwakeup |= cap->cc_kqblocked; 1003 cap->cc_kqblocked = 0; 1004 } 1005 crypto_driver_unlock(cap); 1006 if (needwakeup) 1007 setsoftcrypto(softintr_cookie); 1008 1009 return 0; 1010 } 1011 1012 /* 1013 * Dispatch a crypto request to a driver or queue 1014 * it, to be processed by the kernel thread. 1015 */ 1016 int 1017 crypto_dispatch(struct cryptop *crp) 1018 { 1019 int result; 1020 struct cryptocap *cap; 1021 1022 KASSERT(crp != NULL); 1023 1024 DPRINTF("crp %p, alg %d\n", crp, crp->crp_desc->crd_alg); 1025 1026 cryptostats.cs_ops++; 1027 1028 #ifdef CRYPTO_TIMING 1029 if (crypto_timing) 1030 nanouptime(&crp->crp_tstamp); 1031 #endif 1032 1033 if ((crp->crp_flags & CRYPTO_F_BATCH) != 0) { 1034 int wasempty; 1035 /* 1036 * Caller marked the request as ``ok to delay''; 1037 * queue it for the swi thread. This is desirable 1038 * when the operation is low priority and/or suitable 1039 * for batching. 1040 * 1041 * don't care list order in batch job. 1042 */ 1043 mutex_enter(&crypto_q_mtx); 1044 wasempty = TAILQ_EMPTY(&crp_q); 1045 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 1046 mutex_exit(&crypto_q_mtx); 1047 if (wasempty) 1048 setsoftcrypto(softintr_cookie); 1049 1050 return 0; 1051 } 1052 1053 mutex_enter(&crypto_q_mtx); 1054 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(crp->crp_sid)); 1055 /* 1056 * TODO: 1057 * If we can ensure the driver has been valid until the driver is 1058 * done crypto_unregister(), this migrate operation is not required. 1059 */ 1060 if (cap == NULL) { 1061 /* 1062 * The driver must be detached, so this request will migrate 1063 * to other drivers in cryptointr() later. 1064 */ 1065 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 1066 mutex_exit(&crypto_q_mtx); 1067 return 0; 1068 } 1069 1070 if (cap->cc_qblocked != 0) { 1071 crypto_driver_unlock(cap); 1072 /* 1073 * The driver is blocked, just queue the op until 1074 * it unblocks and the swi thread gets kicked. 1075 */ 1076 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 1077 mutex_exit(&crypto_q_mtx); 1078 return 0; 1079 } 1080 1081 /* 1082 * Caller marked the request to be processed 1083 * immediately; dispatch it directly to the 1084 * driver unless the driver is currently blocked. 1085 */ 1086 crypto_driver_unlock(cap); 1087 result = crypto_invoke(crp, 0); 1088 if (result == ERESTART) { 1089 /* 1090 * The driver ran out of resources, mark the 1091 * driver ``blocked'' for cryptop's and put 1092 * the op on the queue. 1093 */ 1094 crypto_driver_lock(cap); 1095 cap->cc_qblocked = 1; 1096 crypto_driver_unlock(cap); 1097 TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); 1098 cryptostats.cs_blocks++; 1099 1100 /* 1101 * The crp is enqueued to crp_q, that is, 1102 * no error occurs. So, this function should 1103 * not return error. 1104 */ 1105 result = 0; 1106 } 1107 1108 mutex_exit(&crypto_q_mtx); 1109 return result; 1110 } 1111 1112 /* 1113 * Add an asymetric crypto request to a queue, 1114 * to be processed by the kernel thread. 1115 */ 1116 int 1117 crypto_kdispatch(struct cryptkop *krp) 1118 { 1119 struct cryptocap *cap; 1120 int result; 1121 1122 KASSERT(krp != NULL); 1123 1124 cryptostats.cs_kops++; 1125 1126 mutex_enter(&crypto_q_mtx); 1127 cap = crypto_checkdriver_lock(krp->krp_hid); 1128 /* 1129 * TODO: 1130 * If we can ensure the driver has been valid until the driver is 1131 * done crypto_unregister(), this migrate operation is not required. 1132 */ 1133 if (cap == NULL) { 1134 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 1135 mutex_exit(&crypto_q_mtx); 1136 return 0; 1137 } 1138 1139 if (cap->cc_kqblocked != 0) { 1140 crypto_driver_unlock(cap); 1141 /* 1142 * The driver is blocked, just queue the op until 1143 * it unblocks and the swi thread gets kicked. 1144 */ 1145 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 1146 mutex_exit(&crypto_q_mtx); 1147 return 0; 1148 } 1149 1150 crypto_driver_unlock(cap); 1151 result = crypto_kinvoke(krp, 0); 1152 if (result == ERESTART) { 1153 /* 1154 * The driver ran out of resources, mark the 1155 * driver ``blocked'' for cryptop's and put 1156 * the op on the queue. 1157 */ 1158 crypto_driver_lock(cap); 1159 cap->cc_kqblocked = 1; 1160 crypto_driver_unlock(cap); 1161 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1162 cryptostats.cs_kblocks++; 1163 mutex_exit(&crypto_q_mtx); 1164 1165 /* 1166 * The krp is enqueued to crp_kq, that is, 1167 * no error occurs. So, this function should 1168 * not return error. 1169 */ 1170 result = 0; 1171 } 1172 1173 return result; 1174 } 1175 1176 /* 1177 * Dispatch an assymetric crypto request to the appropriate crypto devices. 1178 */ 1179 static int 1180 crypto_kinvoke(struct cryptkop *krp, int hint) 1181 { 1182 struct cryptocap *cap = NULL; 1183 u_int32_t hid; 1184 int error; 1185 1186 KASSERT(krp != NULL); 1187 1188 /* Sanity checks. */ 1189 if (krp->krp_callback == NULL) { 1190 cv_destroy(&krp->krp_cv); 1191 crypto_kfreereq(krp); 1192 return EINVAL; 1193 } 1194 1195 mutex_enter(&crypto_drv_mtx); 1196 for (hid = 0; hid < crypto_drivers_num; hid++) { 1197 cap = crypto_checkdriver(hid); 1198 if (cap == NULL) 1199 continue; 1200 crypto_driver_lock(cap); 1201 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1202 crypto_devallowsoft == 0) { 1203 crypto_driver_unlock(cap); 1204 continue; 1205 } 1206 if (cap->cc_kprocess == NULL) { 1207 crypto_driver_unlock(cap); 1208 continue; 1209 } 1210 if ((cap->cc_kalg[krp->krp_op] & 1211 CRYPTO_ALG_FLAG_SUPPORTED) == 0) { 1212 crypto_driver_unlock(cap); 1213 continue; 1214 } 1215 break; 1216 } 1217 mutex_exit(&crypto_drv_mtx); 1218 if (cap != NULL) { 1219 int (*process)(void *, struct cryptkop *, int); 1220 void *arg; 1221 1222 process = cap->cc_kprocess; 1223 arg = cap->cc_karg; 1224 krp->krp_hid = hid; 1225 crypto_driver_unlock(cap); 1226 error = (*process)(arg, krp, hint); 1227 } else { 1228 error = ENODEV; 1229 } 1230 1231 if (error) { 1232 krp->krp_status = error; 1233 crypto_kdone(krp); 1234 } 1235 return 0; 1236 } 1237 1238 #ifdef CRYPTO_TIMING 1239 static void 1240 crypto_tstat(struct cryptotstat *ts, struct timespec *tv) 1241 { 1242 struct timespec now, t; 1243 1244 nanouptime(&now); 1245 t.tv_sec = now.tv_sec - tv->tv_sec; 1246 t.tv_nsec = now.tv_nsec - tv->tv_nsec; 1247 if (t.tv_nsec < 0) { 1248 t.tv_sec--; 1249 t.tv_nsec += 1000000000; 1250 } 1251 timespecadd(&ts->acc, &t, &t); 1252 if (timespeccmp(&t, &ts->min, <)) 1253 ts->min = t; 1254 if (timespeccmp(&t, &ts->max, >)) 1255 ts->max = t; 1256 ts->count++; 1257 1258 *tv = now; 1259 } 1260 #endif 1261 1262 /* 1263 * Dispatch a crypto request to the appropriate crypto devices. 1264 */ 1265 static int 1266 crypto_invoke(struct cryptop *crp, int hint) 1267 { 1268 struct cryptocap *cap; 1269 1270 KASSERT(crp != NULL); 1271 1272 #ifdef CRYPTO_TIMING 1273 if (crypto_timing) 1274 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 1275 #endif 1276 /* Sanity checks. */ 1277 if (crp->crp_callback == NULL) { 1278 return EINVAL; 1279 } 1280 if (crp->crp_desc == NULL) { 1281 crp->crp_etype = EINVAL; 1282 crypto_done(crp); 1283 return 0; 1284 } 1285 1286 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(crp->crp_sid)); 1287 if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 1288 int (*process)(void *, struct cryptop *, int); 1289 void *arg; 1290 1291 process = cap->cc_process; 1292 arg = cap->cc_arg; 1293 1294 /* 1295 * Invoke the driver to process the request. 1296 */ 1297 DPRINTF("calling process for %p\n", crp); 1298 crypto_driver_unlock(cap); 1299 return (*process)(arg, crp, hint); 1300 } else { 1301 struct cryptodesc *crd; 1302 u_int64_t nid = 0; 1303 1304 if (cap != NULL) 1305 crypto_driver_unlock(cap); 1306 1307 /* 1308 * Driver has unregistered; migrate the session and return 1309 * an error to the caller so they'll resubmit the op. 1310 */ 1311 crypto_freesession(crp->crp_sid); 1312 1313 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1314 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1315 1316 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 1317 crp->crp_sid = nid; 1318 1319 crp->crp_etype = EAGAIN; 1320 1321 crypto_done(crp); 1322 return 0; 1323 } 1324 } 1325 1326 /* 1327 * Release a set of crypto descriptors. 1328 */ 1329 void 1330 crypto_freereq(struct cryptop *crp) 1331 { 1332 struct cryptodesc *crd; 1333 1334 if (crp == NULL) 1335 return; 1336 DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(crp->crp_sid), crp); 1337 1338 /* sanity check */ 1339 if (crp->crp_flags & CRYPTO_F_ONRETQ) { 1340 panic("crypto_freereq() freeing crp on RETQ\n"); 1341 } 1342 1343 while ((crd = crp->crp_desc) != NULL) { 1344 crp->crp_desc = crd->crd_next; 1345 pool_put(&cryptodesc_pool, crd); 1346 } 1347 pool_put(&cryptop_pool, crp); 1348 } 1349 1350 /* 1351 * Acquire a set of crypto descriptors. 1352 */ 1353 struct cryptop * 1354 crypto_getreq(int num) 1355 { 1356 struct cryptodesc *crd; 1357 struct cryptop *crp; 1358 1359 /* 1360 * When crp_ret_q is full, we restrict here to avoid crp_ret_q overflow 1361 * by error callback. 1362 */ 1363 if (CRYPTO_Q_IS_FULL(crp_ret_q)) { 1364 CRYPTO_Q_INC_DROPS(crp_ret_q); 1365 return NULL; 1366 } 1367 1368 crp = pool_get(&cryptop_pool, 0); 1369 if (crp == NULL) { 1370 return NULL; 1371 } 1372 memset(crp, 0, sizeof(struct cryptop)); 1373 1374 while (num--) { 1375 crd = pool_get(&cryptodesc_pool, 0); 1376 if (crd == NULL) { 1377 crypto_freereq(crp); 1378 return NULL; 1379 } 1380 1381 memset(crd, 0, sizeof(struct cryptodesc)); 1382 crd->crd_next = crp->crp_desc; 1383 crp->crp_desc = crd; 1384 } 1385 1386 return crp; 1387 } 1388 1389 /* 1390 * Release a set of asymmetric crypto descriptors. 1391 * Currently, support one descriptor only. 1392 */ 1393 void 1394 crypto_kfreereq(struct cryptkop *krp) 1395 { 1396 1397 if (krp == NULL) 1398 return; 1399 1400 DPRINTF("krp %p\n", krp); 1401 1402 /* sanity check */ 1403 if (krp->krp_flags & CRYPTO_F_ONRETQ) { 1404 panic("crypto_kfreereq() freeing krp on RETQ\n"); 1405 } 1406 1407 pool_put(&cryptkop_pool, krp); 1408 } 1409 1410 /* 1411 * Acquire a set of asymmetric crypto descriptors. 1412 * Currently, support one descriptor only. 1413 */ 1414 struct cryptkop * 1415 crypto_kgetreq(int num __unused, int prflags) 1416 { 1417 struct cryptkop *krp; 1418 1419 /* 1420 * When crp_ret_kq is full, we restrict here to avoid crp_ret_kq 1421 * overflow by error callback. 1422 */ 1423 if (CRYPTO_Q_IS_FULL(crp_ret_kq)) { 1424 CRYPTO_Q_INC_DROPS(crp_ret_kq); 1425 return NULL; 1426 } 1427 1428 krp = pool_get(&cryptkop_pool, prflags); 1429 if (krp == NULL) { 1430 return NULL; 1431 } 1432 memset(krp, 0, sizeof(struct cryptkop)); 1433 1434 return krp; 1435 } 1436 1437 /* 1438 * Invoke the callback on behalf of the driver. 1439 */ 1440 void 1441 crypto_done(struct cryptop *crp) 1442 { 1443 int wasempty; 1444 1445 KASSERT(crp != NULL); 1446 1447 if (crp->crp_etype != 0) 1448 cryptostats.cs_errs++; 1449 #ifdef CRYPTO_TIMING 1450 if (crypto_timing) 1451 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 1452 #endif 1453 DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(crp->crp_sid), crp); 1454 1455 /* 1456 * Normal case; queue the callback for the thread. 1457 * 1458 * The return queue is manipulated by the swi thread 1459 * and, potentially, by crypto device drivers calling 1460 * back to mark operations completed. Thus we need 1461 * to mask both while manipulating the return queue. 1462 */ 1463 if (crp->crp_flags & CRYPTO_F_CBIMM) { 1464 /* 1465 * Do the callback directly. This is ok when the 1466 * callback routine does very little (e.g. the 1467 * /dev/crypto callback method just does a wakeup). 1468 */ 1469 mutex_spin_enter(&crypto_ret_q_mtx); 1470 crp->crp_flags |= CRYPTO_F_DONE; 1471 mutex_spin_exit(&crypto_ret_q_mtx); 1472 1473 #ifdef CRYPTO_TIMING 1474 if (crypto_timing) { 1475 /* 1476 * NB: We must copy the timestamp before 1477 * doing the callback as the cryptop is 1478 * likely to be reclaimed. 1479 */ 1480 struct timespec t = crp->crp_tstamp; 1481 crypto_tstat(&cryptostats.cs_cb, &t); 1482 crp->crp_callback(crp); 1483 crypto_tstat(&cryptostats.cs_finis, &t); 1484 } else 1485 #endif 1486 crp->crp_callback(crp); 1487 } else { 1488 mutex_spin_enter(&crypto_ret_q_mtx); 1489 crp->crp_flags |= CRYPTO_F_DONE; 1490 #if 0 1491 if (crp->crp_flags & CRYPTO_F_USER) { 1492 /* 1493 * TODO: 1494 * If crp->crp_flags & CRYPTO_F_USER and the used 1495 * encryption driver does all the processing in 1496 * the same context, we can skip enqueueing crp_ret_q 1497 * and cv_signal(&cryptoret_cv). 1498 */ 1499 DPRINTF("lid[%u]: crp %p CRYPTO_F_USER\n", 1500 CRYPTO_SESID2LID(crp->crp_sid), crp); 1501 } else 1502 #endif 1503 { 1504 wasempty = TAILQ_EMPTY(&crp_ret_q); 1505 DPRINTF("lid[%u]: queueing %p\n", 1506 CRYPTO_SESID2LID(crp->crp_sid), crp); 1507 crp->crp_flags |= CRYPTO_F_ONRETQ; 1508 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 1509 CRYPTO_Q_INC(crp_ret_q); 1510 if (wasempty) { 1511 DPRINTF("lid[%u]: waking cryptoret, " 1512 "crp %p hit empty queue\n.", 1513 CRYPTO_SESID2LID(crp->crp_sid), crp); 1514 cv_signal(&cryptoret_cv); 1515 } 1516 } 1517 mutex_spin_exit(&crypto_ret_q_mtx); 1518 } 1519 } 1520 1521 /* 1522 * Invoke the callback on behalf of the driver. 1523 */ 1524 void 1525 crypto_kdone(struct cryptkop *krp) 1526 { 1527 int wasempty; 1528 1529 KASSERT(krp != NULL); 1530 1531 if (krp->krp_status != 0) 1532 cryptostats.cs_kerrs++; 1533 1534 krp->krp_flags |= CRYPTO_F_DONE; 1535 1536 /* 1537 * The return queue is manipulated by the swi thread 1538 * and, potentially, by crypto device drivers calling 1539 * back to mark operations completed. Thus we need 1540 * to mask both while manipulating the return queue. 1541 */ 1542 if (krp->krp_flags & CRYPTO_F_CBIMM) { 1543 krp->krp_callback(krp); 1544 } else { 1545 mutex_spin_enter(&crypto_ret_q_mtx); 1546 wasempty = TAILQ_EMPTY(&crp_ret_kq); 1547 krp->krp_flags |= CRYPTO_F_ONRETQ; 1548 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1549 CRYPTO_Q_INC(crp_ret_kq); 1550 if (wasempty) 1551 cv_signal(&cryptoret_cv); 1552 mutex_spin_exit(&crypto_ret_q_mtx); 1553 } 1554 } 1555 1556 int 1557 crypto_getfeat(int *featp) 1558 { 1559 1560 if (crypto_userasymcrypto == 0) { 1561 *featp = 0; 1562 return 0; 1563 } 1564 1565 mutex_enter(&crypto_drv_mtx); 1566 1567 int feat = 0; 1568 for (int hid = 0; hid < crypto_drivers_num; hid++) { 1569 struct cryptocap *cap; 1570 cap = crypto_checkdriver(hid); 1571 if (cap == NULL) 1572 continue; 1573 1574 crypto_driver_lock(cap); 1575 1576 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1577 crypto_devallowsoft == 0) 1578 goto unlock; 1579 1580 if (cap->cc_kprocess == NULL) 1581 goto unlock; 1582 1583 for (int kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1584 if ((cap->cc_kalg[kalg] & 1585 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1586 feat |= 1 << kalg; 1587 1588 unlock: crypto_driver_unlock(cap); 1589 } 1590 1591 mutex_exit(&crypto_drv_mtx); 1592 *featp = feat; 1593 return (0); 1594 } 1595 1596 /* 1597 * Software interrupt thread to dispatch crypto requests. 1598 */ 1599 static void 1600 cryptointr(void) 1601 { 1602 struct cryptop *crp, *submit, *cnext; 1603 struct cryptkop *krp, *knext; 1604 struct cryptocap *cap; 1605 int result, hint; 1606 1607 cryptostats.cs_intrs++; 1608 mutex_enter(&crypto_q_mtx); 1609 do { 1610 /* 1611 * Find the first element in the queue that can be 1612 * processed and look-ahead to see if multiple ops 1613 * are ready for the same driver. 1614 */ 1615 submit = NULL; 1616 hint = 0; 1617 TAILQ_FOREACH_SAFE(crp, &crp_q, crp_next, cnext) { 1618 u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid); 1619 cap = crypto_checkdriver_lock(hid); 1620 if (cap == NULL || cap->cc_process == NULL) { 1621 if (cap != NULL) 1622 crypto_driver_unlock(cap); 1623 /* Op needs to be migrated, process it. */ 1624 submit = crp; 1625 break; 1626 } 1627 1628 /* 1629 * skip blocked crp regardless of CRYPTO_F_BATCH 1630 */ 1631 if (cap->cc_qblocked != 0) { 1632 crypto_driver_unlock(cap); 1633 continue; 1634 } 1635 crypto_driver_unlock(cap); 1636 1637 /* 1638 * skip batch crp until the end of crp_q 1639 */ 1640 if ((crp->crp_flags & CRYPTO_F_BATCH) != 0) { 1641 if (submit == NULL) { 1642 submit = crp; 1643 } else { 1644 if (CRYPTO_SESID2HID(submit->crp_sid) 1645 == hid) 1646 hint = CRYPTO_HINT_MORE; 1647 } 1648 1649 continue; 1650 } 1651 1652 /* 1653 * found first crp which is neither blocked nor batch. 1654 */ 1655 submit = crp; 1656 /* 1657 * batch crp can be processed much later, so clear hint. 1658 */ 1659 hint = 0; 1660 break; 1661 } 1662 if (submit != NULL) { 1663 TAILQ_REMOVE(&crp_q, submit, crp_next); 1664 result = crypto_invoke(submit, hint); 1665 /* we must take here as the TAILQ op or kinvoke 1666 may need this mutex below. sigh. */ 1667 if (result == ERESTART) { 1668 /* 1669 * The driver ran out of resources, mark the 1670 * driver ``blocked'' for cryptop's and put 1671 * the request back in the queue. It would 1672 * best to put the request back where we got 1673 * it but that's hard so for now we put it 1674 * at the front. This should be ok; putting 1675 * it at the end does not work. 1676 */ 1677 /* validate sid again */ 1678 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(submit->crp_sid)); 1679 if (cap == NULL) { 1680 /* migrate again, sigh... */ 1681 TAILQ_INSERT_TAIL(&crp_q, submit, crp_next); 1682 } else { 1683 cap->cc_qblocked = 1; 1684 crypto_driver_unlock(cap); 1685 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1686 cryptostats.cs_blocks++; 1687 } 1688 } 1689 } 1690 1691 /* As above, but for key ops */ 1692 TAILQ_FOREACH_SAFE(krp, &crp_kq, krp_next, knext) { 1693 cap = crypto_checkdriver_lock(krp->krp_hid); 1694 if (cap == NULL || cap->cc_kprocess == NULL) { 1695 if (cap != NULL) 1696 crypto_driver_unlock(cap); 1697 /* Op needs to be migrated, process it. */ 1698 break; 1699 } 1700 if (!cap->cc_kqblocked) { 1701 crypto_driver_unlock(cap); 1702 break; 1703 } 1704 crypto_driver_unlock(cap); 1705 } 1706 if (krp != NULL) { 1707 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1708 result = crypto_kinvoke(krp, 0); 1709 /* the next iteration will want the mutex. :-/ */ 1710 if (result == ERESTART) { 1711 /* 1712 * The driver ran out of resources, mark the 1713 * driver ``blocked'' for cryptkop's and put 1714 * the request back in the queue. It would 1715 * best to put the request back where we got 1716 * it but that's hard so for now we put it 1717 * at the front. This should be ok; putting 1718 * it at the end does not work. 1719 */ 1720 /* validate sid again */ 1721 cap = crypto_checkdriver_lock(krp->krp_hid); 1722 if (cap == NULL) { 1723 /* migrate again, sigh... */ 1724 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 1725 } else { 1726 cap->cc_kqblocked = 1; 1727 crypto_driver_unlock(cap); 1728 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1729 cryptostats.cs_kblocks++; 1730 } 1731 } 1732 } 1733 } while (submit != NULL || krp != NULL); 1734 mutex_exit(&crypto_q_mtx); 1735 } 1736 1737 /* 1738 * Kernel thread to do callbacks. 1739 */ 1740 static void 1741 cryptoret(void) 1742 { 1743 struct cryptop *crp; 1744 struct cryptkop *krp; 1745 1746 mutex_spin_enter(&crypto_ret_q_mtx); 1747 for (;;) { 1748 crp = TAILQ_FIRST(&crp_ret_q); 1749 if (crp != NULL) { 1750 TAILQ_REMOVE(&crp_ret_q, crp, crp_next); 1751 CRYPTO_Q_DEC(crp_ret_q); 1752 crp->crp_flags &= ~CRYPTO_F_ONRETQ; 1753 } 1754 krp = TAILQ_FIRST(&crp_ret_kq); 1755 if (krp != NULL) { 1756 TAILQ_REMOVE(&crp_ret_kq, krp, krp_next); 1757 CRYPTO_Q_DEC(crp_ret_kq); 1758 krp->krp_flags &= ~CRYPTO_F_ONRETQ; 1759 } 1760 1761 /* drop before calling any callbacks. */ 1762 if (crp == NULL && krp == NULL) { 1763 1764 /* Check for the exit condition. */ 1765 if (crypto_exit_flag != 0) { 1766 1767 /* Time to die. */ 1768 crypto_exit_flag = 0; 1769 cv_broadcast(&cryptoret_cv); 1770 mutex_spin_exit(&crypto_ret_q_mtx); 1771 kthread_exit(0); 1772 } 1773 1774 cryptostats.cs_rets++; 1775 cv_wait(&cryptoret_cv, &crypto_ret_q_mtx); 1776 continue; 1777 } 1778 1779 mutex_spin_exit(&crypto_ret_q_mtx); 1780 1781 if (crp != NULL) { 1782 #ifdef CRYPTO_TIMING 1783 if (crypto_timing) { 1784 /* 1785 * NB: We must copy the timestamp before 1786 * doing the callback as the cryptop is 1787 * likely to be reclaimed. 1788 */ 1789 struct timespec t = crp->crp_tstamp; 1790 crypto_tstat(&cryptostats.cs_cb, &t); 1791 crp->crp_callback(crp); 1792 crypto_tstat(&cryptostats.cs_finis, &t); 1793 } else 1794 #endif 1795 { 1796 crp->crp_callback(crp); 1797 } 1798 } 1799 if (krp != NULL) 1800 krp->krp_callback(krp); 1801 1802 mutex_spin_enter(&crypto_ret_q_mtx); 1803 } 1804 } 1805 1806 /* NetBSD module interface */ 1807 1808 MODULE(MODULE_CLASS_MISC, opencrypto, NULL); 1809 1810 static int 1811 opencrypto_modcmd(modcmd_t cmd, void *opaque) 1812 { 1813 int error = 0; 1814 1815 switch (cmd) { 1816 case MODULE_CMD_INIT: 1817 #ifdef _MODULE 1818 error = crypto_init(); 1819 #endif 1820 break; 1821 case MODULE_CMD_FINI: 1822 #ifdef _MODULE 1823 error = crypto_destroy(true); 1824 #endif 1825 break; 1826 default: 1827 error = ENOTTY; 1828 } 1829 return error; 1830 } 1831
Indexes created Mon Oct 20 08:09:54 GMT 2025