crypto.c revision 1.109
1 /* $NetBSD: crypto.c,v 1.109 2019/10/01 18:00:09 chs Exp $ */ 2 /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.5 2003/02/26 00:14:05 sam Exp $ */ 3 /* $OpenBSD: crypto.c,v 1.41 2002/07/17 23:52:38 art Exp $ */ 4 5 /*- 6 * Copyright (c) 2008 The NetBSD Foundation, Inc. 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by Coyote Point Systems, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * The author of this code is Angelos D. Keromytis (angelos (at) cis.upenn.edu) 36 * 37 * This code was written by Angelos D. Keromytis in Athens, Greece, in 38 * February 2000. Network Security Technologies Inc. (NSTI) kindly 39 * supported the development of this code. 40 * 41 * Copyright (c) 2000, 2001 Angelos D. Keromytis 42 * 43 * Permission to use, copy, and modify this software with or without fee 44 * is hereby granted, provided that this entire notice is included in 45 * all source code copies of any software which is or includes a copy or 46 * modification of this software. 47 * 48 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 49 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 50 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 51 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 52 * PURPOSE. 53 */ 54 55 #include <sys/cdefs.h> 56 __KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.109 2019/10/01 18:00:09 chs Exp $"); 57 58 #include <sys/param.h> 59 #include <sys/reboot.h> 60 #include <sys/systm.h> 61 #include <sys/proc.h> 62 #include <sys/pool.h> 63 #include <sys/kthread.h> 64 #include <sys/once.h> 65 #include <sys/sysctl.h> 66 #include <sys/intr.h> 67 #include <sys/errno.h> 68 #include <sys/module.h> 69 #include <sys/xcall.h> 70 #include <sys/device.h> 71 #include <sys/cpu.h> 72 #include <sys/percpu.h> 73 #include <sys/kmem.h> 74 75 #if defined(_KERNEL_OPT) 76 #include "opt_ocf.h" 77 #endif 78 79 #include <opencrypto/cryptodev.h> 80 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 81 82 /* 83 * Crypto drivers register themselves by allocating a slot in the 84 * crypto_drivers table with crypto_get_driverid() and then registering 85 * each algorithm they support with crypto_register() and crypto_kregister(). 86 */ 87 /* Don't directly access crypto_drivers[i], use crypto_checkdriver(i). */ 88 static struct { 89 kmutex_t mtx; 90 int num; 91 struct cryptocap *list; 92 } crypto_drv __cacheline_aligned; 93 #define crypto_drv_mtx (crypto_drv.mtx) 94 #define crypto_drivers_num (crypto_drv.num) 95 #define crypto_drivers (crypto_drv.list) 96 97 static void *crypto_q_si; 98 static void *crypto_ret_si; 99 100 /* 101 * There are two queues for crypto requests; one for symmetric (e.g. 102 * cipher) operations and one for asymmetric (e.g. MOD) operations. 103 * See below for how synchronization is handled. 104 */ 105 TAILQ_HEAD(crypto_crp_q, cryptop); 106 TAILQ_HEAD(crypto_crp_kq, cryptkop); 107 struct crypto_crp_qs { 108 struct crypto_crp_q *crp_q; 109 struct crypto_crp_kq *crp_kq; 110 }; 111 static percpu_t *crypto_crp_qs_percpu; 112 113 static inline struct crypto_crp_qs * 114 crypto_get_crp_qs(int *s) 115 { 116 117 KASSERT(s != NULL); 118 119 *s = splsoftnet(); 120 return percpu_getref(crypto_crp_qs_percpu); 121 } 122 123 static inline void 124 crypto_put_crp_qs(int *s) 125 { 126 127 KASSERT(s != NULL); 128 129 percpu_putref(crypto_crp_qs_percpu); 130 splx(*s); 131 } 132 133 static void 134 crypto_crp_q_is_busy_pc(void *p, void *arg, struct cpu_info *ci __unused) 135 { 136 struct crypto_crp_qs *qs_pc = p; 137 bool *isempty = arg; 138 139 if (!TAILQ_EMPTY(qs_pc->crp_q) || !TAILQ_EMPTY(qs_pc->crp_kq)) 140 *isempty = true; 141 } 142 143 static void 144 crypto_crp_qs_init_pc(void *p, void *arg __unused, struct cpu_info *ci __unused) 145 { 146 struct crypto_crp_qs *qs = p; 147 148 qs->crp_q = kmem_alloc(sizeof(struct crypto_crp_q), KM_SLEEP); 149 qs->crp_kq = kmem_alloc(sizeof(struct crypto_crp_kq), KM_SLEEP); 150 151 TAILQ_INIT(qs->crp_q); 152 TAILQ_INIT(qs->crp_kq); 153 } 154 155 /* 156 * There are two queues for processing completed crypto requests; one 157 * for the symmetric and one for the asymmetric ops. We only need one 158 * but have two to avoid type futzing (cryptop vs. cryptkop). See below 159 * for how synchronization is handled. 160 */ 161 TAILQ_HEAD(crypto_crp_ret_q, cryptop); 162 TAILQ_HEAD(crypto_crp_ret_kq, cryptkop); 163 struct crypto_crp_ret_qs { 164 kmutex_t crp_ret_q_mtx; 165 bool crp_ret_q_exit_flag; 166 167 struct crypto_crp_ret_q crp_ret_q; 168 int crp_ret_q_len; 169 int crp_ret_q_maxlen; /* queue length limit. <=0 means unlimited. */ 170 int crp_ret_q_drops; 171 172 struct crypto_crp_ret_kq crp_ret_kq; 173 int crp_ret_kq_len; 174 int crp_ret_kq_maxlen; /* queue length limit. <=0 means unlimited. */ 175 int crp_ret_kq_drops; 176 }; 177 struct crypto_crp_ret_qs **crypto_crp_ret_qs_list; 178 179 180 static inline struct crypto_crp_ret_qs * 181 crypto_get_crp_ret_qs(struct cpu_info *ci) 182 { 183 u_int cpuid; 184 struct crypto_crp_ret_qs *qs; 185 186 KASSERT(ci != NULL); 187 188 cpuid = cpu_index(ci); 189 qs = crypto_crp_ret_qs_list[cpuid]; 190 mutex_enter(&qs->crp_ret_q_mtx); 191 return qs; 192 } 193 194 static inline void 195 crypto_put_crp_ret_qs(struct cpu_info *ci) 196 { 197 u_int cpuid; 198 struct crypto_crp_ret_qs *qs; 199 200 KASSERT(ci != NULL); 201 202 cpuid = cpu_index(ci); 203 qs = crypto_crp_ret_qs_list[cpuid]; 204 mutex_exit(&qs->crp_ret_q_mtx); 205 } 206 207 #ifndef CRYPTO_RET_Q_MAXLEN 208 #define CRYPTO_RET_Q_MAXLEN 0 209 #endif 210 #ifndef CRYPTO_RET_KQ_MAXLEN 211 #define CRYPTO_RET_KQ_MAXLEN 0 212 #endif 213 214 static int 215 sysctl_opencrypto_q_len(SYSCTLFN_ARGS) 216 { 217 int error, len = 0; 218 struct sysctlnode node = *rnode; 219 220 for (int i = 0; i < ncpu; i++) { 221 struct crypto_crp_ret_qs *qs; 222 struct cpu_info *ci = cpu_lookup(i); 223 224 qs = crypto_get_crp_ret_qs(ci); 225 len += qs->crp_ret_q_len; 226 crypto_put_crp_ret_qs(ci); 227 } 228 229 node.sysctl_data = &len; 230 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 231 if (error || newp == NULL) 232 return error; 233 234 return 0; 235 } 236 237 static int 238 sysctl_opencrypto_q_drops(SYSCTLFN_ARGS) 239 { 240 int error, drops = 0; 241 struct sysctlnode node = *rnode; 242 243 for (int i = 0; i < ncpu; i++) { 244 struct crypto_crp_ret_qs *qs; 245 struct cpu_info *ci = cpu_lookup(i); 246 247 qs = crypto_get_crp_ret_qs(ci); 248 drops += qs->crp_ret_q_drops; 249 crypto_put_crp_ret_qs(ci); 250 } 251 252 node.sysctl_data = &drops; 253 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 254 if (error || newp == NULL) 255 return error; 256 257 return 0; 258 } 259 260 static int 261 sysctl_opencrypto_q_maxlen(SYSCTLFN_ARGS) 262 { 263 int error, maxlen; 264 struct crypto_crp_ret_qs *qs; 265 struct sysctlnode node = *rnode; 266 267 /* each crp_ret_kq_maxlen is the same. */ 268 qs = crypto_get_crp_ret_qs(curcpu()); 269 maxlen = qs->crp_ret_q_maxlen; 270 crypto_put_crp_ret_qs(curcpu()); 271 272 node.sysctl_data = &maxlen; 273 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 274 if (error || newp == NULL) 275 return error; 276 277 for (int i = 0; i < ncpu; i++) { 278 struct cpu_info *ci = cpu_lookup(i); 279 280 qs = crypto_get_crp_ret_qs(ci); 281 qs->crp_ret_q_maxlen = maxlen; 282 crypto_put_crp_ret_qs(ci); 283 } 284 285 return 0; 286 } 287 288 static int 289 sysctl_opencrypto_kq_len(SYSCTLFN_ARGS) 290 { 291 int error, len = 0; 292 struct sysctlnode node = *rnode; 293 294 for (int i = 0; i < ncpu; i++) { 295 struct crypto_crp_ret_qs *qs; 296 struct cpu_info *ci = cpu_lookup(i); 297 298 qs = crypto_get_crp_ret_qs(ci); 299 len += qs->crp_ret_kq_len; 300 crypto_put_crp_ret_qs(ci); 301 } 302 303 node.sysctl_data = &len; 304 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 305 if (error || newp == NULL) 306 return error; 307 308 return 0; 309 } 310 311 static int 312 sysctl_opencrypto_kq_drops(SYSCTLFN_ARGS) 313 { 314 int error, drops = 0; 315 struct sysctlnode node = *rnode; 316 317 for (int i = 0; i < ncpu; i++) { 318 struct crypto_crp_ret_qs *qs; 319 struct cpu_info *ci = cpu_lookup(i); 320 321 qs = crypto_get_crp_ret_qs(ci); 322 drops += qs->crp_ret_kq_drops; 323 crypto_put_crp_ret_qs(ci); 324 } 325 326 node.sysctl_data = &drops; 327 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 328 if (error || newp == NULL) 329 return error; 330 331 return 0; 332 } 333 334 static int 335 sysctl_opencrypto_kq_maxlen(SYSCTLFN_ARGS) 336 { 337 int error, maxlen; 338 struct crypto_crp_ret_qs *qs; 339 struct sysctlnode node = *rnode; 340 341 /* each crp_ret_kq_maxlen is the same. */ 342 qs = crypto_get_crp_ret_qs(curcpu()); 343 maxlen = qs->crp_ret_kq_maxlen; 344 crypto_put_crp_ret_qs(curcpu()); 345 346 node.sysctl_data = &maxlen; 347 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 348 if (error || newp == NULL) 349 return error; 350 351 for (int i = 0; i < ncpu; i++) { 352 struct cpu_info *ci = cpu_lookup(i); 353 354 qs = crypto_get_crp_ret_qs(ci); 355 qs->crp_ret_kq_maxlen = maxlen; 356 crypto_put_crp_ret_qs(ci); 357 } 358 359 return 0; 360 } 361 362 /* 363 * Crypto op and descriptor data structures are allocated 364 * from separate private zones(FreeBSD)/pools(netBSD/OpenBSD) . 365 */ 366 static pool_cache_t cryptop_cache; 367 static pool_cache_t cryptodesc_cache; 368 static pool_cache_t cryptkop_cache; 369 370 int crypto_usercrypto = 1; /* userland may open /dev/crypto */ 371 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 372 /* 373 * cryptodevallowsoft is (intended to be) sysctl'able, controlling 374 * access to hardware versus software transforms as below: 375 * 376 * crypto_devallowsoft < 0: Force userlevel requests to use software 377 * transforms, always 378 * crypto_devallowsoft = 0: Use hardware if present, grant userlevel 379 * requests for non-accelerated transforms 380 * (handling the latter in software) 381 * crypto_devallowsoft > 0: Allow user requests only for transforms which 382 * are hardware-accelerated. 383 */ 384 int crypto_devallowsoft = 1; /* only use hardware crypto */ 385 386 static void 387 sysctl_opencrypto_setup(struct sysctllog **clog) 388 { 389 const struct sysctlnode *ocnode; 390 const struct sysctlnode *retqnode, *retkqnode; 391 392 sysctl_createv(clog, 0, NULL, NULL, 393 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 394 CTLTYPE_INT, "usercrypto", 395 SYSCTL_DESCR("Enable/disable user-mode access to " 396 "crypto support"), 397 NULL, 0, &crypto_usercrypto, 0, 398 CTL_KERN, CTL_CREATE, CTL_EOL); 399 sysctl_createv(clog, 0, NULL, NULL, 400 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 401 CTLTYPE_INT, "userasymcrypto", 402 SYSCTL_DESCR("Enable/disable user-mode access to " 403 "asymmetric crypto support"), 404 NULL, 0, &crypto_userasymcrypto, 0, 405 CTL_KERN, CTL_CREATE, CTL_EOL); 406 sysctl_createv(clog, 0, NULL, NULL, 407 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 408 CTLTYPE_INT, "cryptodevallowsoft", 409 SYSCTL_DESCR("Enable/disable use of software " 410 "asymmetric crypto support"), 411 NULL, 0, &crypto_devallowsoft, 0, 412 CTL_KERN, CTL_CREATE, CTL_EOL); 413 414 sysctl_createv(clog, 0, NULL, &ocnode, 415 CTLFLAG_PERMANENT, 416 CTLTYPE_NODE, "opencrypto", 417 SYSCTL_DESCR("opencrypto related entries"), 418 NULL, 0, NULL, 0, 419 CTL_CREATE, CTL_EOL); 420 421 sysctl_createv(clog, 0, &ocnode, &retqnode, 422 CTLFLAG_PERMANENT, 423 CTLTYPE_NODE, "crypto_ret_q", 424 SYSCTL_DESCR("crypto_ret_q related entries"), 425 NULL, 0, NULL, 0, 426 CTL_CREATE, CTL_EOL); 427 sysctl_createv(clog, 0, &retqnode, NULL, 428 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 429 CTLTYPE_INT, "len", 430 SYSCTL_DESCR("Current queue length"), 431 sysctl_opencrypto_q_len, 0, 432 NULL, 0, 433 CTL_CREATE, CTL_EOL); 434 sysctl_createv(clog, 0, &retqnode, NULL, 435 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 436 CTLTYPE_INT, "drops", 437 SYSCTL_DESCR("Crypto requests dropped due to full ret queue"), 438 sysctl_opencrypto_q_drops, 0, 439 NULL, 0, 440 CTL_CREATE, CTL_EOL); 441 sysctl_createv(clog, 0, &retqnode, NULL, 442 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 443 CTLTYPE_INT, "maxlen", 444 SYSCTL_DESCR("Maximum allowed queue length"), 445 sysctl_opencrypto_q_maxlen, 0, 446 NULL, 0, 447 CTL_CREATE, CTL_EOL); 448 449 450 sysctl_createv(clog, 0, &ocnode, &retkqnode, 451 CTLFLAG_PERMANENT, 452 CTLTYPE_NODE, "crypto_ret_kq", 453 SYSCTL_DESCR("crypto_ret_kq related entries"), 454 NULL, 0, NULL, 0, 455 CTL_CREATE, CTL_EOL); 456 sysctl_createv(clog, 0, &retkqnode, NULL, 457 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 458 CTLTYPE_INT, "len", 459 SYSCTL_DESCR("Current queue length"), 460 sysctl_opencrypto_kq_len, 0, 461 NULL, 0, 462 CTL_CREATE, CTL_EOL); 463 sysctl_createv(clog, 0, &retkqnode, NULL, 464 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 465 CTLTYPE_INT, "drops", 466 SYSCTL_DESCR("Crypto requests dropped due to full ret queue"), 467 sysctl_opencrypto_kq_drops, 0, 468 NULL, 0, 469 CTL_CREATE, CTL_EOL); 470 sysctl_createv(clog, 0, &retkqnode, NULL, 471 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 472 CTLTYPE_INT, "maxlen", 473 SYSCTL_DESCR("Maximum allowed queue length"), 474 sysctl_opencrypto_kq_maxlen, 0, 475 NULL, 0, 476 CTL_CREATE, CTL_EOL); 477 } 478 479 /* 480 * Synchronization: read carefully, this is non-trivial. 481 * 482 * Crypto requests are submitted via crypto_dispatch. Typically 483 * these come in from network protocols at spl0 (output path) or 484 * spl[,soft]net (input path). 485 * 486 * Requests are typically passed on the driver directly, but they 487 * may also be queued for processing by a software interrupt thread, 488 * cryptointr, that runs at splsoftcrypto. This thread dispatches 489 * the requests to crypto drivers (h/w or s/w) who call crypto_done 490 * when a request is complete. Hardware crypto drivers are assumed 491 * to register their IRQ's as network devices so their interrupt handlers 492 * and subsequent "done callbacks" happen at spl[imp,net]. 493 * 494 * Completed crypto ops are queued for a separate kernel thread that 495 * handles the callbacks at spl0. This decoupling insures the crypto 496 * driver interrupt service routine is not delayed while the callback 497 * takes place and that callbacks are delivered after a context switch 498 * (as opposed to a software interrupt that clients must block). 499 * 500 * This scheme is not intended for SMP machines. 501 */ 502 static void cryptointr(void *); /* swi thread to dispatch ops */ 503 static void cryptoret_softint(void *); /* kernel thread for callbacks*/ 504 static int crypto_destroy(bool); 505 static int crypto_invoke(struct cryptop *crp, int hint); 506 static int crypto_kinvoke(struct cryptkop *krp, int hint); 507 508 static struct cryptocap *crypto_checkdriver_lock(u_int32_t); 509 static struct cryptocap *crypto_checkdriver_uninit(u_int32_t); 510 static struct cryptocap *crypto_checkdriver(u_int32_t); 511 static void crypto_driver_lock(struct cryptocap *); 512 static void crypto_driver_unlock(struct cryptocap *); 513 static void crypto_driver_clear(struct cryptocap *); 514 515 static int crypto_init_finalize(device_t); 516 517 static struct cryptostats cryptostats; 518 #ifdef CRYPTO_TIMING 519 static int crypto_timing = 0; 520 #endif 521 522 static struct sysctllog *sysctl_opencrypto_clog; 523 524 static void 525 crypto_crp_ret_qs_init(void) 526 { 527 int i; 528 529 crypto_crp_ret_qs_list = kmem_alloc(sizeof(struct crypto_crp_ret_qs *) * ncpu, 530 KM_SLEEP); 531 532 for (i = 0; i < ncpu; i++) { 533 struct crypto_crp_ret_qs *qs; 534 535 qs = kmem_alloc(sizeof(struct crypto_crp_ret_qs), KM_SLEEP); 536 mutex_init(&qs->crp_ret_q_mtx, MUTEX_DEFAULT, IPL_NET); 537 qs->crp_ret_q_exit_flag = false; 538 539 TAILQ_INIT(&qs->crp_ret_q); 540 qs->crp_ret_q_len = 0; 541 qs->crp_ret_q_maxlen = CRYPTO_RET_Q_MAXLEN; 542 qs->crp_ret_q_drops = 0; 543 544 TAILQ_INIT(&qs->crp_ret_kq); 545 qs->crp_ret_kq_len = 0; 546 qs->crp_ret_kq_maxlen = CRYPTO_RET_KQ_MAXLEN; 547 qs->crp_ret_kq_drops = 0; 548 549 crypto_crp_ret_qs_list[i] = qs; 550 } 551 } 552 553 static int 554 crypto_init0(void) 555 { 556 557 mutex_init(&crypto_drv_mtx, MUTEX_DEFAULT, IPL_NONE); 558 cryptop_cache = pool_cache_init(sizeof(struct cryptop), 559 coherency_unit, 0, 0, "cryptop", NULL, IPL_NET, NULL, NULL, NULL); 560 cryptodesc_cache = pool_cache_init(sizeof(struct cryptodesc), 561 coherency_unit, 0, 0, "cryptdesc", NULL, IPL_NET, NULL, NULL, NULL); 562 cryptkop_cache = pool_cache_init(sizeof(struct cryptkop), 563 coherency_unit, 0, 0, "cryptkop", NULL, IPL_NET, NULL, NULL, NULL); 564 565 crypto_crp_qs_percpu = percpu_alloc(sizeof(struct crypto_crp_qs)); 566 percpu_foreach(crypto_crp_qs_percpu, crypto_crp_qs_init_pc, NULL); 567 568 crypto_crp_ret_qs_init(); 569 570 crypto_drivers = kmem_zalloc(CRYPTO_DRIVERS_INITIAL * 571 sizeof(struct cryptocap), KM_SLEEP); 572 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 573 574 crypto_q_si = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE, cryptointr, NULL); 575 if (crypto_q_si == NULL) { 576 printf("crypto_init: cannot establish request queue handler\n"); 577 return crypto_destroy(false); 578 } 579 580 /* 581 * Some encryption devices (such as mvcesa) are attached before 582 * ipi_sysinit(). That causes an assertion in ipi_register() as 583 * crypto_ret_si softint uses SOFTINT_RCPU. 584 */ 585 if (config_finalize_register(NULL, crypto_init_finalize) != 0) { 586 printf("crypto_init: cannot register crypto_init_finalize\n"); 587 return crypto_destroy(false); 588 } 589 590 sysctl_opencrypto_setup(&sysctl_opencrypto_clog); 591 592 return 0; 593 } 594 595 static int 596 crypto_init_finalize(device_t self __unused) 597 { 598 599 crypto_ret_si = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE|SOFTINT_RCPU, 600 &cryptoret_softint, NULL); 601 KASSERT(crypto_ret_si != NULL); 602 603 return 0; 604 } 605 606 int 607 crypto_init(void) 608 { 609 static ONCE_DECL(crypto_init_once); 610 611 return RUN_ONCE(&crypto_init_once, crypto_init0); 612 } 613 614 static int 615 crypto_destroy(bool exit_kthread) 616 { 617 int i; 618 619 if (exit_kthread) { 620 struct cryptocap *cap = NULL; 621 uint64_t where; 622 bool is_busy = false; 623 624 /* if we have any in-progress requests, don't unload */ 625 percpu_foreach(crypto_crp_qs_percpu, crypto_crp_q_is_busy_pc, 626 &is_busy); 627 if (is_busy) 628 return EBUSY; 629 /* FIXME: 630 * prohibit enqueue to crp_q and crp_kq after here. 631 */ 632 633 mutex_enter(&crypto_drv_mtx); 634 for (i = 0; i < crypto_drivers_num; i++) { 635 cap = crypto_checkdriver(i); 636 if (cap == NULL) 637 continue; 638 if (cap->cc_sessions != 0) { 639 mutex_exit(&crypto_drv_mtx); 640 return EBUSY; 641 } 642 } 643 mutex_exit(&crypto_drv_mtx); 644 /* FIXME: 645 * prohibit touch crypto_drivers[] and each element after here. 646 */ 647 648 /* 649 * Ensure cryptoret_softint() is never scheduled and then wait 650 * for last softint_execute(). 651 */ 652 for (i = 0; i < ncpu; i++) { 653 struct crypto_crp_ret_qs *qs; 654 struct cpu_info *ci = cpu_lookup(i); 655 656 qs = crypto_get_crp_ret_qs(ci); 657 qs->crp_ret_q_exit_flag = true; 658 crypto_put_crp_ret_qs(ci); 659 } 660 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL); 661 xc_wait(where); 662 } 663 664 if (sysctl_opencrypto_clog != NULL) 665 sysctl_teardown(&sysctl_opencrypto_clog); 666 667 if (crypto_ret_si != NULL) 668 softint_disestablish(crypto_ret_si); 669 670 if (crypto_q_si != NULL) 671 softint_disestablish(crypto_q_si); 672 673 mutex_enter(&crypto_drv_mtx); 674 if (crypto_drivers != NULL) 675 kmem_free(crypto_drivers, 676 crypto_drivers_num * sizeof(struct cryptocap)); 677 mutex_exit(&crypto_drv_mtx); 678 679 percpu_free(crypto_crp_qs_percpu, sizeof(struct crypto_crp_qs)); 680 681 pool_cache_destroy(cryptop_cache); 682 pool_cache_destroy(cryptodesc_cache); 683 pool_cache_destroy(cryptkop_cache); 684 685 mutex_destroy(&crypto_drv_mtx); 686 687 return 0; 688 } 689 690 static bool 691 crypto_driver_suitable(struct cryptocap *cap, struct cryptoini *cri) 692 { 693 struct cryptoini *cr; 694 695 for (cr = cri; cr; cr = cr->cri_next) 696 if (cap->cc_alg[cr->cri_alg] == 0) { 697 DPRINTF("alg %d not supported\n", cr->cri_alg); 698 return false; 699 } 700 701 return true; 702 } 703 704 #define CRYPTO_ACCEPT_HARDWARE 0x1 705 #define CRYPTO_ACCEPT_SOFTWARE 0x2 706 /* 707 * The algorithm we use here is pretty stupid; just use the 708 * first driver that supports all the algorithms we need. 709 * If there are multiple drivers we choose the driver with 710 * the fewest active sessions. We prefer hardware-backed 711 * drivers to software ones. 712 * 713 * XXX We need more smarts here (in real life too, but that's 714 * XXX another story altogether). 715 */ 716 static struct cryptocap * 717 crypto_select_driver_lock(struct cryptoini *cri, int hard) 718 { 719 u_int32_t hid; 720 int accept; 721 struct cryptocap *cap, *best; 722 int error = 0; 723 724 best = NULL; 725 /* 726 * hard == 0 can use both hardware and software drivers. 727 * We use hardware drivers prior to software drivers, so search 728 * hardware drivers at first time. 729 */ 730 if (hard >= 0) 731 accept = CRYPTO_ACCEPT_HARDWARE; 732 else 733 accept = CRYPTO_ACCEPT_SOFTWARE; 734 again: 735 for (hid = 0; hid < crypto_drivers_num; hid++) { 736 cap = crypto_checkdriver(hid); 737 if (cap == NULL) 738 continue; 739 740 crypto_driver_lock(cap); 741 742 /* 743 * If it's not initialized or has remaining sessions 744 * referencing it, skip. 745 */ 746 if (cap->cc_newsession == NULL || 747 (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) { 748 crypto_driver_unlock(cap); 749 continue; 750 } 751 752 /* Hardware required -- ignore software drivers. */ 753 if ((accept & CRYPTO_ACCEPT_SOFTWARE) == 0 754 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE)) { 755 crypto_driver_unlock(cap); 756 continue; 757 } 758 /* Software required -- ignore hardware drivers. */ 759 if ((accept & CRYPTO_ACCEPT_HARDWARE) == 0 760 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) { 761 crypto_driver_unlock(cap); 762 continue; 763 } 764 765 /* See if all the algorithms are supported. */ 766 if (crypto_driver_suitable(cap, cri)) { 767 if (best == NULL) { 768 /* keep holding crypto_driver_lock(cap) */ 769 best = cap; 770 continue; 771 } else if (cap->cc_sessions < best->cc_sessions) { 772 crypto_driver_unlock(best); 773 /* keep holding crypto_driver_lock(cap) */ 774 best = cap; 775 continue; 776 } 777 } 778 779 crypto_driver_unlock(cap); 780 } 781 if (best == NULL && hard == 0 782 && (accept & CRYPTO_ACCEPT_SOFTWARE) == 0) { 783 accept = CRYPTO_ACCEPT_SOFTWARE; 784 goto again; 785 } 786 787 if (best == NULL && hard == 0 && error == 0) { 788 mutex_exit(&crypto_drv_mtx); 789 error = module_autoload("swcrypto", MODULE_CLASS_DRIVER); 790 mutex_enter(&crypto_drv_mtx); 791 if (error == 0) { 792 error = EINVAL; 793 goto again; 794 } 795 } 796 797 return best; 798 } 799 800 /* 801 * Create a new session. 802 */ 803 int 804 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 805 { 806 struct cryptocap *cap; 807 int err = EINVAL; 808 809 mutex_enter(&crypto_drv_mtx); 810 811 cap = crypto_select_driver_lock(cri, hard); 812 if (cap != NULL) { 813 u_int32_t hid, lid; 814 815 hid = cap - crypto_drivers; 816 /* 817 * Can't do everything in one session. 818 * 819 * XXX Fix this. We need to inject a "virtual" session layer right 820 * XXX about here. 821 */ 822 823 /* Call the driver initialization routine. */ 824 lid = hid; /* Pass the driver ID. */ 825 crypto_driver_unlock(cap); 826 err = cap->cc_newsession(cap->cc_arg, &lid, cri); 827 crypto_driver_lock(cap); 828 if (err == 0) { 829 (*sid) = hid; 830 (*sid) <<= 32; 831 (*sid) |= (lid & 0xffffffff); 832 (cap->cc_sessions)++; 833 } else { 834 DPRINTF("crypto_drivers[%d].cc_newsession() failed. error=%d\n", 835 hid, err); 836 } 837 crypto_driver_unlock(cap); 838 } 839 840 mutex_exit(&crypto_drv_mtx); 841 842 return err; 843 } 844 845 /* 846 * Delete an existing session (or a reserved session on an unregistered 847 * driver). 848 */ 849 int 850 crypto_freesession(u_int64_t sid) 851 { 852 struct cryptocap *cap; 853 int err = 0; 854 855 /* Determine two IDs. */ 856 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(sid)); 857 if (cap == NULL) 858 return ENOENT; 859 860 if (cap->cc_sessions) 861 (cap->cc_sessions)--; 862 863 /* Call the driver cleanup routine, if available. */ 864 if (cap->cc_freesession) 865 err = cap->cc_freesession(cap->cc_arg, sid); 866 else 867 err = 0; 868 869 /* 870 * If this was the last session of a driver marked as invalid, 871 * make the entry available for reuse. 872 */ 873 if ((cap->cc_flags & CRYPTOCAP_F_CLEANUP) && cap->cc_sessions == 0) 874 crypto_driver_clear(cap); 875 876 crypto_driver_unlock(cap); 877 return err; 878 } 879 880 static bool 881 crypto_checkdriver_initialized(const struct cryptocap *cap) 882 { 883 884 return cap->cc_process != NULL || 885 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0 || 886 cap->cc_sessions != 0; 887 } 888 889 /* 890 * Return an unused driver id. Used by drivers prior to registering 891 * support for the algorithms they handle. 892 */ 893 int32_t 894 crypto_get_driverid(u_int32_t flags) 895 { 896 struct cryptocap *newdrv; 897 struct cryptocap *cap = NULL; 898 int i; 899 900 (void)crypto_init(); /* XXX oh, this is foul! */ 901 902 mutex_enter(&crypto_drv_mtx); 903 for (i = 0; i < crypto_drivers_num; i++) { 904 cap = crypto_checkdriver_uninit(i); 905 if (cap == NULL || crypto_checkdriver_initialized(cap)) 906 continue; 907 break; 908 } 909 910 /* Out of entries, allocate some more. */ 911 if (cap == NULL) { 912 /* Be careful about wrap-around. */ 913 if (2 * crypto_drivers_num <= crypto_drivers_num) { 914 mutex_exit(&crypto_drv_mtx); 915 printf("crypto: driver count wraparound!\n"); 916 return -1; 917 } 918 919 newdrv = kmem_zalloc(2 * crypto_drivers_num * 920 sizeof(struct cryptocap), KM_SLEEP); 921 memcpy(newdrv, crypto_drivers, 922 crypto_drivers_num * sizeof(struct cryptocap)); 923 kmem_free(crypto_drivers, 924 crypto_drivers_num * sizeof(struct cryptocap)); 925 926 crypto_drivers_num *= 2; 927 crypto_drivers = newdrv; 928 929 cap = crypto_checkdriver_uninit(i); 930 KASSERT(cap != NULL); 931 } 932 933 /* NB: state is zero'd on free */ 934 cap->cc_sessions = 1; /* Mark */ 935 cap->cc_flags = flags; 936 mutex_init(&cap->cc_lock, MUTEX_DEFAULT, IPL_NET); 937 938 if (bootverbose) 939 printf("crypto: assign driver %u, flags %u\n", i, flags); 940 941 mutex_exit(&crypto_drv_mtx); 942 943 return i; 944 } 945 946 static struct cryptocap * 947 crypto_checkdriver_lock(u_int32_t hid) 948 { 949 struct cryptocap *cap; 950 951 KASSERT(crypto_drivers != NULL); 952 953 if (hid >= crypto_drivers_num) 954 return NULL; 955 956 cap = &crypto_drivers[hid]; 957 mutex_enter(&cap->cc_lock); 958 return cap; 959 } 960 961 /* 962 * Use crypto_checkdriver_uninit() instead of crypto_checkdriver() below two 963 * situations 964 * - crypto_drivers[] may not be allocated 965 * - crypto_drivers[hid] may not be initialized 966 */ 967 static struct cryptocap * 968 crypto_checkdriver_uninit(u_int32_t hid) 969 { 970 971 KASSERT(mutex_owned(&crypto_drv_mtx)); 972 973 if (crypto_drivers == NULL) 974 return NULL; 975 976 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 977 } 978 979 /* 980 * Use crypto_checkdriver_uninit() instead of crypto_checkdriver() below two 981 * situations 982 * - crypto_drivers[] may not be allocated 983 * - crypto_drivers[hid] may not be initialized 984 */ 985 static struct cryptocap * 986 crypto_checkdriver(u_int32_t hid) 987 { 988 989 KASSERT(mutex_owned(&crypto_drv_mtx)); 990 991 if (crypto_drivers == NULL || hid >= crypto_drivers_num) 992 return NULL; 993 994 struct cryptocap *cap = &crypto_drivers[hid]; 995 return crypto_checkdriver_initialized(cap) ? cap : NULL; 996 } 997 998 static inline void 999 crypto_driver_lock(struct cryptocap *cap) 1000 { 1001 1002 KASSERT(cap != NULL); 1003 1004 mutex_enter(&cap->cc_lock); 1005 } 1006 1007 static inline void 1008 crypto_driver_unlock(struct cryptocap *cap) 1009 { 1010 1011 KASSERT(cap != NULL); 1012 1013 mutex_exit(&cap->cc_lock); 1014 } 1015 1016 static void 1017 crypto_driver_clear(struct cryptocap *cap) 1018 { 1019 1020 if (cap == NULL) 1021 return; 1022 1023 KASSERT(mutex_owned(&cap->cc_lock)); 1024 1025 cap->cc_sessions = 0; 1026 memset(&cap->cc_max_op_len, 0, sizeof(cap->cc_max_op_len)); 1027 memset(&cap->cc_alg, 0, sizeof(cap->cc_alg)); 1028 memset(&cap->cc_kalg, 0, sizeof(cap->cc_kalg)); 1029 cap->cc_flags = 0; 1030 cap->cc_qblocked = 0; 1031 cap->cc_kqblocked = 0; 1032 1033 cap->cc_arg = NULL; 1034 cap->cc_newsession = NULL; 1035 cap->cc_process = NULL; 1036 cap->cc_freesession = NULL; 1037 cap->cc_kprocess = NULL; 1038 } 1039 1040 /* 1041 * Register support for a key-related algorithm. This routine 1042 * is called once for each algorithm supported a driver. 1043 */ 1044 int 1045 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 1046 int (*kprocess)(void *, struct cryptkop *, int), 1047 void *karg) 1048 { 1049 struct cryptocap *cap; 1050 int err; 1051 1052 mutex_enter(&crypto_drv_mtx); 1053 1054 cap = crypto_checkdriver_lock(driverid); 1055 if (cap != NULL && 1056 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 1057 /* 1058 * XXX Do some performance testing to determine placing. 1059 * XXX We probably need an auxiliary data structure that 1060 * XXX describes relative performances. 1061 */ 1062 1063 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 1064 if (bootverbose) { 1065 printf("crypto: driver %u registers key alg %u " 1066 " flags %u\n", 1067 driverid, 1068 kalg, 1069 flags 1070 ); 1071 } 1072 1073 if (cap->cc_kprocess == NULL) { 1074 cap->cc_karg = karg; 1075 cap->cc_kprocess = kprocess; 1076 } 1077 err = 0; 1078 } else 1079 err = EINVAL; 1080 1081 mutex_exit(&crypto_drv_mtx); 1082 return err; 1083 } 1084 1085 /* 1086 * Register support for a non-key-related algorithm. This routine 1087 * is called once for each such algorithm supported by a driver. 1088 */ 1089 int 1090 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 1091 u_int32_t flags, 1092 int (*newses)(void *, u_int32_t*, struct cryptoini*), 1093 int (*freeses)(void *, u_int64_t), 1094 int (*process)(void *, struct cryptop *, int), 1095 void *arg) 1096 { 1097 struct cryptocap *cap; 1098 int err; 1099 1100 cap = crypto_checkdriver_lock(driverid); 1101 if (cap == NULL) 1102 return EINVAL; 1103 1104 /* NB: algorithms are in the range [1..max] */ 1105 if (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) { 1106 /* 1107 * XXX Do some performance testing to determine placing. 1108 * XXX We probably need an auxiliary data structure that 1109 * XXX describes relative performances. 1110 */ 1111 1112 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 1113 cap->cc_max_op_len[alg] = maxoplen; 1114 if (bootverbose) { 1115 printf("crypto: driver %u registers alg %u " 1116 "flags %u maxoplen %u\n", 1117 driverid, 1118 alg, 1119 flags, 1120 maxoplen 1121 ); 1122 } 1123 1124 if (cap->cc_process == NULL) { 1125 cap->cc_arg = arg; 1126 cap->cc_newsession = newses; 1127 cap->cc_process = process; 1128 cap->cc_freesession = freeses; 1129 cap->cc_sessions = 0; /* Unmark */ 1130 } 1131 err = 0; 1132 } else 1133 err = EINVAL; 1134 1135 crypto_driver_unlock(cap); 1136 1137 return err; 1138 } 1139 1140 static int 1141 crypto_unregister_locked(struct cryptocap *cap, int alg, bool all) 1142 { 1143 int i; 1144 u_int32_t ses; 1145 bool lastalg = true; 1146 1147 KASSERT(cap != NULL); 1148 KASSERT(mutex_owned(&cap->cc_lock)); 1149 1150 if (alg < CRYPTO_ALGORITHM_MIN || CRYPTO_ALGORITHM_MAX < alg) 1151 return EINVAL; 1152 1153 if (!all && cap->cc_alg[alg] == 0) 1154 return EINVAL; 1155 1156 cap->cc_alg[alg] = 0; 1157 cap->cc_max_op_len[alg] = 0; 1158 1159 if (all) { 1160 if (alg != CRYPTO_ALGORITHM_MAX) 1161 lastalg = false; 1162 } else { 1163 /* Was this the last algorithm ? */ 1164 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) 1165 if (cap->cc_alg[i] != 0) { 1166 lastalg = false; 1167 break; 1168 } 1169 } 1170 if (lastalg) { 1171 ses = cap->cc_sessions; 1172 crypto_driver_clear(cap); 1173 if (ses != 0) { 1174 /* 1175 * If there are pending sessions, just mark as invalid. 1176 */ 1177 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 1178 cap->cc_sessions = ses; 1179 } 1180 } 1181 1182 return 0; 1183 } 1184 1185 /* 1186 * Unregister a crypto driver. If there are pending sessions using it, 1187 * leave enough information around so that subsequent calls using those 1188 * sessions will correctly detect the driver has been unregistered and 1189 * reroute requests. 1190 */ 1191 int 1192 crypto_unregister(u_int32_t driverid, int alg) 1193 { 1194 int err; 1195 struct cryptocap *cap; 1196 1197 cap = crypto_checkdriver_lock(driverid); 1198 err = crypto_unregister_locked(cap, alg, false); 1199 crypto_driver_unlock(cap); 1200 1201 return err; 1202 } 1203 1204 /* 1205 * Unregister all algorithms associated with a crypto driver. 1206 * If there are pending sessions using it, leave enough information 1207 * around so that subsequent calls using those sessions will 1208 * correctly detect the driver has been unregistered and reroute 1209 * requests. 1210 */ 1211 int 1212 crypto_unregister_all(u_int32_t driverid) 1213 { 1214 int err, i; 1215 struct cryptocap *cap; 1216 1217 cap = crypto_checkdriver_lock(driverid); 1218 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 1219 err = crypto_unregister_locked(cap, i, true); 1220 if (err) 1221 break; 1222 } 1223 crypto_driver_unlock(cap); 1224 1225 return err; 1226 } 1227 1228 /* 1229 * Clear blockage on a driver. The what parameter indicates whether 1230 * the driver is now ready for cryptop's and/or cryptokop's. 1231 */ 1232 int 1233 crypto_unblock(u_int32_t driverid, int what) 1234 { 1235 struct cryptocap *cap; 1236 int needwakeup = 0; 1237 1238 cap = crypto_checkdriver_lock(driverid); 1239 if (cap == NULL) 1240 return EINVAL; 1241 1242 if (what & CRYPTO_SYMQ) { 1243 needwakeup |= cap->cc_qblocked; 1244 cap->cc_qblocked = 0; 1245 } 1246 if (what & CRYPTO_ASYMQ) { 1247 needwakeup |= cap->cc_kqblocked; 1248 cap->cc_kqblocked = 0; 1249 } 1250 crypto_driver_unlock(cap); 1251 if (needwakeup) { 1252 kpreempt_disable(); 1253 softint_schedule(crypto_q_si); 1254 kpreempt_enable(); 1255 } 1256 1257 return 0; 1258 } 1259 1260 /* 1261 * Dispatch a crypto request to a driver or queue 1262 * it, to be processed by the kernel thread. 1263 */ 1264 int 1265 crypto_dispatch(struct cryptop *crp) 1266 { 1267 int result, s; 1268 struct cryptocap *cap; 1269 struct crypto_crp_qs *crp_qs; 1270 struct crypto_crp_q *crp_q; 1271 1272 KASSERT(crp != NULL); 1273 1274 DPRINTF("crp %p, alg %d\n", crp, crp->crp_desc->crd_alg); 1275 1276 cryptostats.cs_ops++; 1277 1278 #ifdef CRYPTO_TIMING 1279 if (crypto_timing) 1280 nanouptime(&crp->crp_tstamp); 1281 #endif 1282 1283 if ((crp->crp_flags & CRYPTO_F_BATCH) != 0) { 1284 int wasempty; 1285 /* 1286 * Caller marked the request as ``ok to delay''; 1287 * queue it for the swi thread. This is desirable 1288 * when the operation is low priority and/or suitable 1289 * for batching. 1290 * 1291 * don't care list order in batch job. 1292 */ 1293 crp_qs = crypto_get_crp_qs(&s); 1294 crp_q = crp_qs->crp_q; 1295 wasempty = TAILQ_EMPTY(crp_q); 1296 TAILQ_INSERT_TAIL(crp_q, crp, crp_next); 1297 crypto_put_crp_qs(&s); 1298 crp_q = NULL; 1299 if (wasempty) { 1300 kpreempt_disable(); 1301 softint_schedule(crypto_q_si); 1302 kpreempt_enable(); 1303 } 1304 1305 return 0; 1306 } 1307 1308 crp_qs = crypto_get_crp_qs(&s); 1309 crp_q = crp_qs->crp_q; 1310 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(crp->crp_sid)); 1311 /* 1312 * TODO: 1313 * If we can ensure the driver has been valid until the driver is 1314 * done crypto_unregister(), this migrate operation is not required. 1315 */ 1316 if (cap == NULL) { 1317 /* 1318 * The driver must be detached, so this request will migrate 1319 * to other drivers in cryptointr() later. 1320 */ 1321 TAILQ_INSERT_TAIL(crp_q, crp, crp_next); 1322 result = 0; 1323 goto out; 1324 } 1325 1326 if (cap->cc_qblocked != 0) { 1327 crypto_driver_unlock(cap); 1328 /* 1329 * The driver is blocked, just queue the op until 1330 * it unblocks and the swi thread gets kicked. 1331 */ 1332 TAILQ_INSERT_TAIL(crp_q, crp, crp_next); 1333 result = 0; 1334 goto out; 1335 } 1336 1337 /* 1338 * Caller marked the request to be processed 1339 * immediately; dispatch it directly to the 1340 * driver unless the driver is currently blocked. 1341 */ 1342 crypto_driver_unlock(cap); 1343 result = crypto_invoke(crp, 0); 1344 if (result == ERESTART) { 1345 /* 1346 * The driver ran out of resources, mark the 1347 * driver ``blocked'' for cryptop's and put 1348 * the op on the queue. 1349 */ 1350 crypto_driver_lock(cap); 1351 cap->cc_qblocked = 1; 1352 crypto_driver_unlock(cap); 1353 TAILQ_INSERT_HEAD(crp_q, crp, crp_next); 1354 cryptostats.cs_blocks++; 1355 1356 /* 1357 * The crp is enqueued to crp_q, that is, 1358 * no error occurs. So, this function should 1359 * not return error. 1360 */ 1361 result = 0; 1362 } 1363 1364 out: 1365 crypto_put_crp_qs(&s); 1366 return result; 1367 } 1368 1369 /* 1370 * Add an asymetric crypto request to a queue, 1371 * to be processed by the kernel thread. 1372 */ 1373 int 1374 crypto_kdispatch(struct cryptkop *krp) 1375 { 1376 int result, s; 1377 struct cryptocap *cap; 1378 struct crypto_crp_qs *crp_qs; 1379 struct crypto_crp_kq *crp_kq; 1380 1381 KASSERT(krp != NULL); 1382 1383 cryptostats.cs_kops++; 1384 1385 crp_qs = crypto_get_crp_qs(&s); 1386 crp_kq = crp_qs->crp_kq; 1387 cap = crypto_checkdriver_lock(krp->krp_hid); 1388 /* 1389 * TODO: 1390 * If we can ensure the driver has been valid until the driver is 1391 * done crypto_unregister(), this migrate operation is not required. 1392 */ 1393 if (cap == NULL) { 1394 TAILQ_INSERT_TAIL(crp_kq, krp, krp_next); 1395 result = 0; 1396 goto out; 1397 } 1398 1399 if (cap->cc_kqblocked != 0) { 1400 crypto_driver_unlock(cap); 1401 /* 1402 * The driver is blocked, just queue the op until 1403 * it unblocks and the swi thread gets kicked. 1404 */ 1405 TAILQ_INSERT_TAIL(crp_kq, krp, krp_next); 1406 result = 0; 1407 goto out; 1408 } 1409 1410 crypto_driver_unlock(cap); 1411 result = crypto_kinvoke(krp, 0); 1412 if (result == ERESTART) { 1413 /* 1414 * The driver ran out of resources, mark the 1415 * driver ``blocked'' for cryptop's and put 1416 * the op on the queue. 1417 */ 1418 crypto_driver_lock(cap); 1419 cap->cc_kqblocked = 1; 1420 crypto_driver_unlock(cap); 1421 TAILQ_INSERT_HEAD(crp_kq, krp, krp_next); 1422 cryptostats.cs_kblocks++; 1423 1424 /* 1425 * The krp is enqueued to crp_kq, that is, 1426 * no error occurs. So, this function should 1427 * not return error. 1428 */ 1429 result = 0; 1430 } 1431 1432 out: 1433 crypto_put_crp_qs(&s); 1434 return result; 1435 } 1436 1437 /* 1438 * Dispatch an assymetric crypto request to the appropriate crypto devices. 1439 */ 1440 static int 1441 crypto_kinvoke(struct cryptkop *krp, int hint) 1442 { 1443 struct cryptocap *cap = NULL; 1444 u_int32_t hid; 1445 int error; 1446 1447 KASSERT(krp != NULL); 1448 1449 /* Sanity checks. */ 1450 if (krp->krp_callback == NULL) { 1451 cv_destroy(&krp->krp_cv); 1452 crypto_kfreereq(krp); 1453 return EINVAL; 1454 } 1455 1456 mutex_enter(&crypto_drv_mtx); 1457 for (hid = 0; hid < crypto_drivers_num; hid++) { 1458 cap = crypto_checkdriver(hid); 1459 if (cap == NULL) 1460 continue; 1461 crypto_driver_lock(cap); 1462 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1463 crypto_devallowsoft == 0) { 1464 crypto_driver_unlock(cap); 1465 continue; 1466 } 1467 if (cap->cc_kprocess == NULL) { 1468 crypto_driver_unlock(cap); 1469 continue; 1470 } 1471 if ((cap->cc_kalg[krp->krp_op] & 1472 CRYPTO_ALG_FLAG_SUPPORTED) == 0) { 1473 crypto_driver_unlock(cap); 1474 continue; 1475 } 1476 break; 1477 } 1478 mutex_exit(&crypto_drv_mtx); 1479 if (cap != NULL) { 1480 int (*process)(void *, struct cryptkop *, int); 1481 void *arg; 1482 1483 process = cap->cc_kprocess; 1484 arg = cap->cc_karg; 1485 krp->krp_hid = hid; 1486 krp->reqcpu = curcpu(); 1487 crypto_driver_unlock(cap); 1488 error = (*process)(arg, krp, hint); 1489 } else { 1490 error = ENODEV; 1491 } 1492 1493 if (error) { 1494 krp->krp_status = error; 1495 crypto_kdone(krp); 1496 } 1497 return 0; 1498 } 1499 1500 #ifdef CRYPTO_TIMING 1501 static void 1502 crypto_tstat(struct cryptotstat *ts, struct timespec *tv) 1503 { 1504 struct timespec now, t; 1505 1506 nanouptime(&now); 1507 t.tv_sec = now.tv_sec - tv->tv_sec; 1508 t.tv_nsec = now.tv_nsec - tv->tv_nsec; 1509 if (t.tv_nsec < 0) { 1510 t.tv_sec--; 1511 t.tv_nsec += 1000000000; 1512 } 1513 timespecadd(&ts->acc, &t, &t); 1514 if (timespeccmp(&t, &ts->min, <)) 1515 ts->min = t; 1516 if (timespeccmp(&t, &ts->max, >)) 1517 ts->max = t; 1518 ts->count++; 1519 1520 *tv = now; 1521 } 1522 #endif 1523 1524 /* 1525 * Dispatch a crypto request to the appropriate crypto devices. 1526 */ 1527 static int 1528 crypto_invoke(struct cryptop *crp, int hint) 1529 { 1530 struct cryptocap *cap; 1531 1532 KASSERT(crp != NULL); 1533 1534 #ifdef CRYPTO_TIMING 1535 if (crypto_timing) 1536 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 1537 #endif 1538 /* Sanity checks. */ 1539 if (crp->crp_callback == NULL) { 1540 return EINVAL; 1541 } 1542 if (crp->crp_desc == NULL) { 1543 crp->crp_etype = EINVAL; 1544 crypto_done(crp); 1545 return 0; 1546 } 1547 1548 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(crp->crp_sid)); 1549 if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 1550 int (*process)(void *, struct cryptop *, int); 1551 void *arg; 1552 1553 process = cap->cc_process; 1554 arg = cap->cc_arg; 1555 crp->reqcpu = curcpu(); 1556 1557 /* 1558 * Invoke the driver to process the request. 1559 */ 1560 DPRINTF("calling process for %p\n", crp); 1561 crypto_driver_unlock(cap); 1562 return (*process)(arg, crp, hint); 1563 } else { 1564 struct cryptodesc *crd; 1565 u_int64_t nid = 0; 1566 1567 if (cap != NULL) 1568 crypto_driver_unlock(cap); 1569 1570 /* 1571 * Driver has unregistered; migrate the session and return 1572 * an error to the caller so they'll resubmit the op. 1573 */ 1574 crypto_freesession(crp->crp_sid); 1575 1576 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1577 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1578 1579 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 1580 crp->crp_sid = nid; 1581 1582 crp->crp_etype = EAGAIN; 1583 1584 crypto_done(crp); 1585 return 0; 1586 } 1587 } 1588 1589 /* 1590 * Release a set of crypto descriptors. 1591 */ 1592 void 1593 crypto_freereq(struct cryptop *crp) 1594 { 1595 struct cryptodesc *crd; 1596 1597 if (crp == NULL) 1598 return; 1599 DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(crp->crp_sid), crp); 1600 1601 /* sanity check */ 1602 if (crp->crp_flags & CRYPTO_F_ONRETQ) { 1603 panic("crypto_freereq() freeing crp on RETQ\n"); 1604 } 1605 1606 while ((crd = crp->crp_desc) != NULL) { 1607 crp->crp_desc = crd->crd_next; 1608 pool_cache_put(cryptodesc_cache, crd); 1609 } 1610 pool_cache_put(cryptop_cache, crp); 1611 } 1612 1613 /* 1614 * Acquire a set of crypto descriptors. 1615 */ 1616 struct cryptop * 1617 crypto_getreq(int num) 1618 { 1619 struct cryptodesc *crd; 1620 struct cryptop *crp; 1621 struct crypto_crp_ret_qs *qs; 1622 1623 /* 1624 * When crp_ret_q is full, we restrict here to avoid crp_ret_q overflow 1625 * by error callback. 1626 */ 1627 qs = crypto_get_crp_ret_qs(curcpu()); 1628 if (qs->crp_ret_q_maxlen > 0 1629 && qs->crp_ret_q_len > qs->crp_ret_q_maxlen) { 1630 qs->crp_ret_q_drops++; 1631 crypto_put_crp_ret_qs(curcpu()); 1632 return NULL; 1633 } 1634 crypto_put_crp_ret_qs(curcpu()); 1635 1636 crp = pool_cache_get(cryptop_cache, PR_NOWAIT); 1637 if (crp == NULL) { 1638 return NULL; 1639 } 1640 memset(crp, 0, sizeof(struct cryptop)); 1641 1642 while (num--) { 1643 crd = pool_cache_get(cryptodesc_cache, PR_NOWAIT); 1644 if (crd == NULL) { 1645 crypto_freereq(crp); 1646 return NULL; 1647 } 1648 1649 memset(crd, 0, sizeof(struct cryptodesc)); 1650 crd->crd_next = crp->crp_desc; 1651 crp->crp_desc = crd; 1652 } 1653 1654 return crp; 1655 } 1656 1657 /* 1658 * Release a set of asymmetric crypto descriptors. 1659 * Currently, support one descriptor only. 1660 */ 1661 void 1662 crypto_kfreereq(struct cryptkop *krp) 1663 { 1664 1665 if (krp == NULL) 1666 return; 1667 1668 DPRINTF("krp %p\n", krp); 1669 1670 /* sanity check */ 1671 if (krp->krp_flags & CRYPTO_F_ONRETQ) { 1672 panic("crypto_kfreereq() freeing krp on RETQ\n"); 1673 } 1674 1675 pool_cache_put(cryptkop_cache, krp); 1676 } 1677 1678 /* 1679 * Acquire a set of asymmetric crypto descriptors. 1680 * Currently, support one descriptor only. 1681 */ 1682 struct cryptkop * 1683 crypto_kgetreq(int num __unused, int prflags) 1684 { 1685 struct cryptkop *krp; 1686 struct crypto_crp_ret_qs *qs; 1687 1688 /* 1689 * When crp_ret_kq is full, we restrict here to avoid crp_ret_kq 1690 * overflow by error callback. 1691 */ 1692 qs = crypto_get_crp_ret_qs(curcpu()); 1693 if (qs->crp_ret_kq_maxlen > 0 1694 && qs->crp_ret_kq_len > qs->crp_ret_kq_maxlen) { 1695 qs->crp_ret_kq_drops++; 1696 crypto_put_crp_ret_qs(curcpu()); 1697 return NULL; 1698 } 1699 crypto_put_crp_ret_qs(curcpu()); 1700 1701 krp = pool_cache_get(cryptkop_cache, prflags); 1702 if (krp == NULL) { 1703 return NULL; 1704 } 1705 memset(krp, 0, sizeof(struct cryptkop)); 1706 1707 return krp; 1708 } 1709 1710 /* 1711 * Invoke the callback on behalf of the driver. 1712 */ 1713 void 1714 crypto_done(struct cryptop *crp) 1715 { 1716 1717 KASSERT(crp != NULL); 1718 1719 if (crp->crp_etype != 0) 1720 cryptostats.cs_errs++; 1721 #ifdef CRYPTO_TIMING 1722 if (crypto_timing) 1723 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 1724 #endif 1725 DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(crp->crp_sid), crp); 1726 1727 /* 1728 * Normal case; queue the callback for the thread. 1729 * 1730 * The return queue is manipulated by the swi thread 1731 * and, potentially, by crypto device drivers calling 1732 * back to mark operations completed. Thus we need 1733 * to mask both while manipulating the return queue. 1734 */ 1735 if (crp->crp_flags & CRYPTO_F_CBIMM) { 1736 /* 1737 * Do the callback directly. This is ok when the 1738 * callback routine does very little (e.g. the 1739 * /dev/crypto callback method just does a wakeup). 1740 */ 1741 crp->crp_flags |= CRYPTO_F_DONE; 1742 1743 #ifdef CRYPTO_TIMING 1744 if (crypto_timing) { 1745 /* 1746 * NB: We must copy the timestamp before 1747 * doing the callback as the cryptop is 1748 * likely to be reclaimed. 1749 */ 1750 struct timespec t = crp->crp_tstamp; 1751 crypto_tstat(&cryptostats.cs_cb, &t); 1752 crp->crp_callback(crp); 1753 crypto_tstat(&cryptostats.cs_finis, &t); 1754 } else 1755 #endif 1756 crp->crp_callback(crp); 1757 } else { 1758 crp->crp_flags |= CRYPTO_F_DONE; 1759 #if 0 1760 if (crp->crp_flags & CRYPTO_F_USER) { 1761 /* 1762 * TODO: 1763 * If crp->crp_flags & CRYPTO_F_USER and the used 1764 * encryption driver does all the processing in 1765 * the same context, we can skip enqueueing crp_ret_q 1766 * and softint_schedule(crypto_ret_si). 1767 */ 1768 DPRINTF("lid[%u]: crp %p CRYPTO_F_USER\n", 1769 CRYPTO_SESID2LID(crp->crp_sid), crp); 1770 } else 1771 #endif 1772 { 1773 int wasempty; 1774 struct crypto_crp_ret_qs *qs; 1775 struct crypto_crp_ret_q *crp_ret_q; 1776 1777 qs = crypto_get_crp_ret_qs(crp->reqcpu); 1778 crp_ret_q = &qs->crp_ret_q; 1779 wasempty = TAILQ_EMPTY(crp_ret_q); 1780 DPRINTF("lid[%u]: queueing %p\n", 1781 CRYPTO_SESID2LID(crp->crp_sid), crp); 1782 crp->crp_flags |= CRYPTO_F_ONRETQ; 1783 TAILQ_INSERT_TAIL(crp_ret_q, crp, crp_next); 1784 qs->crp_ret_q_len++; 1785 if (wasempty && !qs->crp_ret_q_exit_flag) { 1786 DPRINTF("lid[%u]: waking cryptoret," 1787 "crp %p hit empty queue\n.", 1788 CRYPTO_SESID2LID(crp->crp_sid), crp); 1789 softint_schedule_cpu(crypto_ret_si, crp->reqcpu); 1790 } 1791 crypto_put_crp_ret_qs(crp->reqcpu); 1792 } 1793 } 1794 } 1795 1796 /* 1797 * Invoke the callback on behalf of the driver. 1798 */ 1799 void 1800 crypto_kdone(struct cryptkop *krp) 1801 { 1802 1803 KASSERT(krp != NULL); 1804 1805 if (krp->krp_status != 0) 1806 cryptostats.cs_kerrs++; 1807 1808 krp->krp_flags |= CRYPTO_F_DONE; 1809 1810 /* 1811 * The return queue is manipulated by the swi thread 1812 * and, potentially, by crypto device drivers calling 1813 * back to mark operations completed. Thus we need 1814 * to mask both while manipulating the return queue. 1815 */ 1816 if (krp->krp_flags & CRYPTO_F_CBIMM) { 1817 krp->krp_callback(krp); 1818 } else { 1819 int wasempty; 1820 struct crypto_crp_ret_qs *qs; 1821 struct crypto_crp_ret_kq *crp_ret_kq; 1822 1823 qs = crypto_get_crp_ret_qs(krp->reqcpu); 1824 crp_ret_kq = &qs->crp_ret_kq; 1825 1826 wasempty = TAILQ_EMPTY(crp_ret_kq); 1827 krp->krp_flags |= CRYPTO_F_ONRETQ; 1828 TAILQ_INSERT_TAIL(crp_ret_kq, krp, krp_next); 1829 qs->crp_ret_kq_len++; 1830 if (wasempty && !qs->crp_ret_q_exit_flag) 1831 softint_schedule_cpu(crypto_ret_si, krp->reqcpu); 1832 crypto_put_crp_ret_qs(krp->reqcpu); 1833 } 1834 } 1835 1836 int 1837 crypto_getfeat(int *featp) 1838 { 1839 1840 if (crypto_userasymcrypto == 0) { 1841 *featp = 0; 1842 return 0; 1843 } 1844 1845 mutex_enter(&crypto_drv_mtx); 1846 1847 int feat = 0; 1848 for (int hid = 0; hid < crypto_drivers_num; hid++) { 1849 struct cryptocap *cap; 1850 cap = crypto_checkdriver(hid); 1851 if (cap == NULL) 1852 continue; 1853 1854 crypto_driver_lock(cap); 1855 1856 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1857 crypto_devallowsoft == 0) 1858 goto unlock; 1859 1860 if (cap->cc_kprocess == NULL) 1861 goto unlock; 1862 1863 for (int kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1864 if ((cap->cc_kalg[kalg] & 1865 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1866 feat |= 1 << kalg; 1867 1868 unlock: crypto_driver_unlock(cap); 1869 } 1870 1871 mutex_exit(&crypto_drv_mtx); 1872 *featp = feat; 1873 return (0); 1874 } 1875 1876 /* 1877 * Software interrupt thread to dispatch crypto requests. 1878 */ 1879 static void 1880 cryptointr(void *arg __unused) 1881 { 1882 struct cryptop *crp, *submit, *cnext; 1883 struct cryptkop *krp, *knext; 1884 struct cryptocap *cap; 1885 struct crypto_crp_qs *crp_qs; 1886 struct crypto_crp_q *crp_q; 1887 struct crypto_crp_kq *crp_kq; 1888 int result, hint, s; 1889 1890 cryptostats.cs_intrs++; 1891 crp_qs = crypto_get_crp_qs(&s); 1892 crp_q = crp_qs->crp_q; 1893 crp_kq = crp_qs->crp_kq; 1894 do { 1895 /* 1896 * Find the first element in the queue that can be 1897 * processed and look-ahead to see if multiple ops 1898 * are ready for the same driver. 1899 */ 1900 submit = NULL; 1901 hint = 0; 1902 TAILQ_FOREACH_SAFE(crp, crp_q, crp_next, cnext) { 1903 u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid); 1904 cap = crypto_checkdriver_lock(hid); 1905 if (cap == NULL || cap->cc_process == NULL) { 1906 if (cap != NULL) 1907 crypto_driver_unlock(cap); 1908 /* Op needs to be migrated, process it. */ 1909 submit = crp; 1910 break; 1911 } 1912 1913 /* 1914 * skip blocked crp regardless of CRYPTO_F_BATCH 1915 */ 1916 if (cap->cc_qblocked != 0) { 1917 crypto_driver_unlock(cap); 1918 continue; 1919 } 1920 crypto_driver_unlock(cap); 1921 1922 /* 1923 * skip batch crp until the end of crp_q 1924 */ 1925 if ((crp->crp_flags & CRYPTO_F_BATCH) != 0) { 1926 if (submit == NULL) { 1927 submit = crp; 1928 } else { 1929 if (CRYPTO_SESID2HID(submit->crp_sid) 1930 == hid) 1931 hint = CRYPTO_HINT_MORE; 1932 } 1933 1934 continue; 1935 } 1936 1937 /* 1938 * found first crp which is neither blocked nor batch. 1939 */ 1940 submit = crp; 1941 /* 1942 * batch crp can be processed much later, so clear hint. 1943 */ 1944 hint = 0; 1945 break; 1946 } 1947 if (submit != NULL) { 1948 TAILQ_REMOVE(crp_q, submit, crp_next); 1949 result = crypto_invoke(submit, hint); 1950 /* we must take here as the TAILQ op or kinvoke 1951 may need this mutex below. sigh. */ 1952 if (result == ERESTART) { 1953 /* 1954 * The driver ran out of resources, mark the 1955 * driver ``blocked'' for cryptop's and put 1956 * the request back in the queue. It would 1957 * best to put the request back where we got 1958 * it but that's hard so for now we put it 1959 * at the front. This should be ok; putting 1960 * it at the end does not work. 1961 */ 1962 /* validate sid again */ 1963 cap = crypto_checkdriver_lock(CRYPTO_SESID2HID(submit->crp_sid)); 1964 if (cap == NULL) { 1965 /* migrate again, sigh... */ 1966 TAILQ_INSERT_TAIL(crp_q, submit, crp_next); 1967 } else { 1968 cap->cc_qblocked = 1; 1969 crypto_driver_unlock(cap); 1970 TAILQ_INSERT_HEAD(crp_q, submit, crp_next); 1971 cryptostats.cs_blocks++; 1972 } 1973 } 1974 } 1975 1976 /* As above, but for key ops */ 1977 TAILQ_FOREACH_SAFE(krp, crp_kq, krp_next, knext) { 1978 cap = crypto_checkdriver_lock(krp->krp_hid); 1979 if (cap == NULL || cap->cc_kprocess == NULL) { 1980 if (cap != NULL) 1981 crypto_driver_unlock(cap); 1982 /* Op needs to be migrated, process it. */ 1983 break; 1984 } 1985 if (!cap->cc_kqblocked) { 1986 crypto_driver_unlock(cap); 1987 break; 1988 } 1989 crypto_driver_unlock(cap); 1990 } 1991 if (krp != NULL) { 1992 TAILQ_REMOVE(crp_kq, krp, krp_next); 1993 result = crypto_kinvoke(krp, 0); 1994 /* the next iteration will want the mutex. :-/ */ 1995 if (result == ERESTART) { 1996 /* 1997 * The driver ran out of resources, mark the 1998 * driver ``blocked'' for cryptkop's and put 1999 * the request back in the queue. It would 2000 * best to put the request back where we got 2001 * it but that's hard so for now we put it 2002 * at the front. This should be ok; putting 2003 * it at the end does not work. 2004 */ 2005 /* validate sid again */ 2006 cap = crypto_checkdriver_lock(krp->krp_hid); 2007 if (cap == NULL) { 2008 /* migrate again, sigh... */ 2009 TAILQ_INSERT_TAIL(crp_kq, krp, krp_next); 2010 } else { 2011 cap->cc_kqblocked = 1; 2012 crypto_driver_unlock(cap); 2013 TAILQ_INSERT_HEAD(crp_kq, krp, krp_next); 2014 cryptostats.cs_kblocks++; 2015 } 2016 } 2017 } 2018 } while (submit != NULL || krp != NULL); 2019 crypto_put_crp_qs(&s); 2020 } 2021 2022 /* 2023 * softint handler to do callbacks. 2024 */ 2025 static void 2026 cryptoret_softint(void *arg __unused) 2027 { 2028 struct crypto_crp_ret_qs *qs; 2029 struct crypto_crp_ret_q *crp_ret_q; 2030 struct crypto_crp_ret_kq *crp_ret_kq; 2031 2032 qs = crypto_get_crp_ret_qs(curcpu()); 2033 crp_ret_q = &qs->crp_ret_q; 2034 crp_ret_kq = &qs->crp_ret_kq; 2035 for (;;) { 2036 struct cryptop *crp; 2037 struct cryptkop *krp; 2038 2039 crp = TAILQ_FIRST(crp_ret_q); 2040 if (crp != NULL) { 2041 TAILQ_REMOVE(crp_ret_q, crp, crp_next); 2042 qs->crp_ret_q_len--; 2043 crp->crp_flags &= ~CRYPTO_F_ONRETQ; 2044 } 2045 krp = TAILQ_FIRST(crp_ret_kq); 2046 if (krp != NULL) { 2047 TAILQ_REMOVE(crp_ret_kq, krp, krp_next); 2048 qs->crp_ret_q_len--; 2049 krp->krp_flags &= ~CRYPTO_F_ONRETQ; 2050 } 2051 2052 /* drop before calling any callbacks. */ 2053 if (crp == NULL && krp == NULL) 2054 break; 2055 2056 mutex_spin_exit(&qs->crp_ret_q_mtx); 2057 if (crp != NULL) { 2058 #ifdef CRYPTO_TIMING 2059 if (crypto_timing) { 2060 /* 2061 * NB: We must copy the timestamp before 2062 * doing the callback as the cryptop is 2063 * likely to be reclaimed. 2064 */ 2065 struct timespec t = crp->crp_tstamp; 2066 crypto_tstat(&cryptostats.cs_cb, &t); 2067 crp->crp_callback(crp); 2068 crypto_tstat(&cryptostats.cs_finis, &t); 2069 } else 2070 #endif 2071 { 2072 crp->crp_callback(crp); 2073 } 2074 } 2075 if (krp != NULL) 2076 krp->krp_callback(krp); 2077 2078 mutex_spin_enter(&qs->crp_ret_q_mtx); 2079 } 2080 crypto_put_crp_ret_qs(curcpu()); 2081 } 2082 2083 /* NetBSD module interface */ 2084 2085 MODULE(MODULE_CLASS_MISC, opencrypto, NULL); 2086 2087 static int 2088 opencrypto_modcmd(modcmd_t cmd, void *opaque) 2089 { 2090 int error = 0; 2091 2092 switch (cmd) { 2093 case MODULE_CMD_INIT: 2094 #ifdef _MODULE 2095 error = crypto_init(); 2096 #endif 2097 break; 2098 case MODULE_CMD_FINI: 2099 #ifdef _MODULE 2100 error = crypto_destroy(true); 2101 #endif 2102 break; 2103 default: 2104 error = ENOTTY; 2105 } 2106 return error; 2107 } 2108
Indexes created Sat Sep 27 21:09:56 GMT 2025