crypto.c revision 1.74 1 1.74 knakahar /* $NetBSD: crypto.c,v 1.74 2017/05/24 09:57:36 knakahara Exp $ */
2 1.1 jonathan /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.5 2003/02/26 00:14:05 sam Exp $ */
3 1.1 jonathan /* $OpenBSD: crypto.c,v 1.41 2002/07/17 23:52:38 art Exp $ */
4 1.1 jonathan
5 1.27 tls /*-
6 1.27 tls * Copyright (c) 2008 The NetBSD Foundation, Inc.
7 1.27 tls * All rights reserved.
8 1.27 tls *
9 1.27 tls * This code is derived from software contributed to The NetBSD Foundation
10 1.27 tls * by Coyote Point Systems, Inc.
11 1.27 tls *
12 1.27 tls * Redistribution and use in source and binary forms, with or without
13 1.27 tls * modification, are permitted provided that the following conditions
14 1.27 tls * are met:
15 1.27 tls * 1. Redistributions of source code must retain the above copyright
16 1.27 tls * notice, this list of conditions and the following disclaimer.
17 1.27 tls * 2. Redistributions in binary form must reproduce the above copyright
18 1.27 tls * notice, this list of conditions and the following disclaimer in the
19 1.27 tls * documentation and/or other materials provided with the distribution.
20 1.27 tls *
21 1.27 tls * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 1.27 tls * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 1.27 tls * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 1.27 tls * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 1.27 tls * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 1.27 tls * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 1.27 tls * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 1.27 tls * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 1.27 tls * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 1.27 tls * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 1.27 tls * POSSIBILITY OF SUCH DAMAGE.
32 1.27 tls */
33 1.27 tls
34 1.1 jonathan /*
35 1.1 jonathan * The author of this code is Angelos D. Keromytis (angelos (at) cis.upenn.edu)
36 1.1 jonathan *
37 1.1 jonathan * This code was written by Angelos D. Keromytis in Athens, Greece, in
38 1.1 jonathan * February 2000. Network Security Technologies Inc. (NSTI) kindly
39 1.1 jonathan * supported the development of this code.
40 1.1 jonathan *
41 1.1 jonathan * Copyright (c) 2000, 2001 Angelos D. Keromytis
42 1.1 jonathan *
43 1.1 jonathan * Permission to use, copy, and modify this software with or without fee
44 1.1 jonathan * is hereby granted, provided that this entire notice is included in
45 1.1 jonathan * all source code copies of any software which is or includes a copy or
46 1.1 jonathan * modification of this software.
47 1.1 jonathan *
48 1.1 jonathan * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
49 1.1 jonathan * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
50 1.1 jonathan * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
51 1.1 jonathan * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
52 1.1 jonathan * PURPOSE.
53 1.1 jonathan */
54 1.1 jonathan
55 1.1 jonathan #include <sys/cdefs.h>
56 1.74 knakahar __KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.74 2017/05/24 09:57:36 knakahara Exp $");
57 1.1 jonathan
58 1.1 jonathan #include <sys/param.h>
59 1.1 jonathan #include <sys/reboot.h>
60 1.1 jonathan #include <sys/systm.h>
61 1.1 jonathan #include <sys/malloc.h>
62 1.1 jonathan #include <sys/proc.h>
63 1.1 jonathan #include <sys/pool.h>
64 1.1 jonathan #include <sys/kthread.h>
65 1.11 thorpej #include <sys/once.h>
66 1.13 christos #include <sys/sysctl.h>
67 1.21 ad #include <sys/intr.h>
68 1.42 pgoyette #include <sys/errno.h>
69 1.42 pgoyette #include <sys/module.h>
70 1.1 jonathan
71 1.42 pgoyette #if defined(_KERNEL_OPT)
72 1.23 tls #include "opt_ocf.h"
73 1.42 pgoyette #endif
74 1.42 pgoyette
75 1.21 ad #include <opencrypto/cryptodev.h>
76 1.1 jonathan #include <opencrypto/xform.h> /* XXX for M_XDATA */
77 1.1 jonathan
78 1.49 knakahar static kmutex_t crypto_q_mtx;
79 1.49 knakahar static kmutex_t crypto_ret_q_mtx;
80 1.49 knakahar static kcondvar_t cryptoret_cv;
81 1.23 tls
82 1.23 tls /* below are kludges for residual code wrtitten to FreeBSD interfaces */
83 1.1 jonathan #define SWI_CRYPTO 17
84 1.1 jonathan #define register_swi(lvl, fn) \
85 1.38 drochner softint_establish(SOFTINT_NET|SOFTINT_MPSAFE, (void (*)(void *))fn, NULL)
86 1.21 ad #define unregister_swi(lvl, fn) softint_disestablish(softintr_cookie)
87 1.56 knakahar #define setsoftcrypto(x) \
88 1.56 knakahar do{ \
89 1.56 knakahar kpreempt_disable(); \
90 1.56 knakahar softint_schedule(x); \
91 1.56 knakahar kpreempt_enable(); \
92 1.56 knakahar }while(0)
93 1.1 jonathan
94 1.30 darran int crypto_ret_q_check(struct cryptop *);
95 1.30 darran
96 1.1 jonathan /*
97 1.1 jonathan * Crypto drivers register themselves by allocating a slot in the
98 1.1 jonathan * crypto_drivers table with crypto_get_driverid() and then registering
99 1.1 jonathan * each algorithm they support with crypto_register() and crypto_kregister().
100 1.1 jonathan */
101 1.57 knakahar static kmutex_t crypto_drv_mtx;
102 1.11 thorpej static struct cryptocap *crypto_drivers;
103 1.11 thorpej static int crypto_drivers_num;
104 1.37 christos static void *softintr_cookie;
105 1.46 pgoyette static int crypto_exit_flag;
106 1.1 jonathan
107 1.1 jonathan /*
108 1.1 jonathan * There are two queues for crypto requests; one for symmetric (e.g.
109 1.1 jonathan * cipher) operations and one for asymmetric (e.g. MOD) operations.
110 1.1 jonathan * See below for how synchronization is handled.
111 1.1 jonathan */
112 1.11 thorpej static TAILQ_HEAD(,cryptop) crp_q = /* request queues */
113 1.11 thorpej TAILQ_HEAD_INITIALIZER(crp_q);
114 1.11 thorpej static TAILQ_HEAD(,cryptkop) crp_kq =
115 1.11 thorpej TAILQ_HEAD_INITIALIZER(crp_kq);
116 1.1 jonathan
117 1.1 jonathan /*
118 1.1 jonathan * There are two queues for processing completed crypto requests; one
119 1.1 jonathan * for the symmetric and one for the asymmetric ops. We only need one
120 1.1 jonathan * but have two to avoid type futzing (cryptop vs. cryptkop). See below
121 1.1 jonathan * for how synchronization is handled.
122 1.1 jonathan */
123 1.23 tls static TAILQ_HEAD(crprethead, cryptop) crp_ret_q = /* callback queues */
124 1.11 thorpej TAILQ_HEAD_INITIALIZER(crp_ret_q);
125 1.23 tls static TAILQ_HEAD(krprethead, cryptkop) crp_ret_kq =
126 1.11 thorpej TAILQ_HEAD_INITIALIZER(crp_ret_kq);
127 1.1 jonathan
128 1.73 knakahar #define DEFINIT_CRYPTO_Q_LEN(name) \
129 1.73 knakahar static int crypto_##name##_len = 0
130 1.73 knakahar
131 1.73 knakahar #define DEFINIT_CRYPTO_Q_DROPS(name) \
132 1.73 knakahar static int crypto_##name##_drops = 0
133 1.73 knakahar
134 1.73 knakahar #define CRYPTO_Q_MAXLEN 0
135 1.73 knakahar #define DEFINIT_CRYPTO_Q_MAXLEN(name) \
136 1.73 knakahar static int crypto_##name##_maxlen = CRYPTO_Q_MAXLEN
137 1.73 knakahar
138 1.73 knakahar #define CRYPTO_Q_INC(name) \
139 1.73 knakahar do { \
140 1.73 knakahar crypto_##name##_len++; \
141 1.73 knakahar } while(0);
142 1.73 knakahar
143 1.73 knakahar #define CRYPTO_Q_DEC(name) \
144 1.73 knakahar do { \
145 1.73 knakahar crypto_##name##_len--; \
146 1.73 knakahar } while(0);
147 1.73 knakahar
148 1.74 knakahar #define CRYPTO_Q_INC_DROPS(name) \
149 1.74 knakahar do { \
150 1.74 knakahar crypto_##name##_drops++; \
151 1.74 knakahar } while(0);
152 1.74 knakahar
153 1.74 knakahar #define CRYPTO_Q_IS_FULL(name) \
154 1.74 knakahar (crypto_##name##_maxlen > 0 \
155 1.74 knakahar && (crypto_##name##_len > crypto_##name##_maxlen))
156 1.74 knakahar
157 1.73 knakahar /*
158 1.73 knakahar * current queue length.
159 1.73 knakahar */
160 1.73 knakahar DEFINIT_CRYPTO_Q_LEN(crp_ret_q);
161 1.73 knakahar DEFINIT_CRYPTO_Q_LEN(crp_ret_kq);
162 1.73 knakahar
163 1.73 knakahar /*
164 1.73 knakahar * queue dropped count.
165 1.73 knakahar */
166 1.73 knakahar DEFINIT_CRYPTO_Q_DROPS(crp_ret_q);
167 1.73 knakahar DEFINIT_CRYPTO_Q_DROPS(crp_ret_kq);
168 1.73 knakahar
169 1.73 knakahar /*
170 1.73 knakahar * queue length limit.
171 1.73 knakahar * default value is 0. <=0 means unlimited.
172 1.73 knakahar */
173 1.73 knakahar DEFINIT_CRYPTO_Q_MAXLEN(crp_ret_q);
174 1.73 knakahar DEFINIT_CRYPTO_Q_MAXLEN(crp_ret_kq);
175 1.73 knakahar
176 1.73 knakahar /*
177 1.73 knakahar * TODO:
178 1.73 knakahar * make percpu
179 1.73 knakahar */
180 1.73 knakahar static int
181 1.73 knakahar sysctl_opencrypto_q_len(SYSCTLFN_ARGS)
182 1.73 knakahar {
183 1.73 knakahar int error;
184 1.73 knakahar
185 1.73 knakahar error = sysctl_lookup(SYSCTLFN_CALL(rnode));
186 1.73 knakahar if (error || newp == NULL)
187 1.73 knakahar return error;
188 1.73 knakahar
189 1.73 knakahar return 0;
190 1.73 knakahar }
191 1.73 knakahar
192 1.73 knakahar /*
193 1.73 knakahar * TODO:
194 1.73 knakahar * make percpu
195 1.73 knakahar */
196 1.73 knakahar static int
197 1.73 knakahar sysctl_opencrypto_q_drops(SYSCTLFN_ARGS)
198 1.73 knakahar {
199 1.73 knakahar int error;
200 1.73 knakahar
201 1.73 knakahar error = sysctl_lookup(SYSCTLFN_CALL(rnode));
202 1.73 knakahar if (error || newp == NULL)
203 1.73 knakahar return error;
204 1.73 knakahar
205 1.73 knakahar return 0;
206 1.73 knakahar }
207 1.73 knakahar
208 1.73 knakahar /*
209 1.73 knakahar * need to make percpu?
210 1.73 knakahar */
211 1.73 knakahar static int
212 1.73 knakahar sysctl_opencrypto_q_maxlen(SYSCTLFN_ARGS)
213 1.73 knakahar {
214 1.73 knakahar int error;
215 1.73 knakahar
216 1.73 knakahar error = sysctl_lookup(SYSCTLFN_CALL(rnode));
217 1.73 knakahar if (error || newp == NULL)
218 1.73 knakahar return error;
219 1.73 knakahar
220 1.73 knakahar return 0;
221 1.73 knakahar }
222 1.73 knakahar
223 1.1 jonathan /*
224 1.1 jonathan * Crypto op and desciptor data structures are allocated
225 1.1 jonathan * from separate private zones(FreeBSD)/pools(netBSD/OpenBSD) .
226 1.1 jonathan */
227 1.1 jonathan struct pool cryptop_pool;
228 1.1 jonathan struct pool cryptodesc_pool;
229 1.23 tls struct pool cryptkop_pool;
230 1.1 jonathan
231 1.1 jonathan int crypto_usercrypto = 1; /* userland may open /dev/crypto */
232 1.1 jonathan int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */
233 1.10 perry /*
234 1.6 jonathan * cryptodevallowsoft is (intended to be) sysctl'able, controlling
235 1.6 jonathan * access to hardware versus software transforms as below:
236 1.6 jonathan *
237 1.6 jonathan * crypto_devallowsoft < 0: Force userlevel requests to use software
238 1.6 jonathan * transforms, always
239 1.6 jonathan * crypto_devallowsoft = 0: Use hardware if present, grant userlevel
240 1.6 jonathan * requests for non-accelerated transforms
241 1.6 jonathan * (handling the latter in software)
242 1.6 jonathan * crypto_devallowsoft > 0: Allow user requests only for transforms which
243 1.6 jonathan * are hardware-accelerated.
244 1.6 jonathan */
245 1.9 jonathan int crypto_devallowsoft = 1; /* only use hardware crypto */
246 1.6 jonathan
247 1.72 knakahar static void
248 1.72 knakahar sysctl_opencrypto_setup(struct sysctllog **clog)
249 1.13 christos {
250 1.73 knakahar const struct sysctlnode *ocnode;
251 1.73 knakahar const struct sysctlnode *retqnode, *retkqnode;
252 1.45 pooka
253 1.13 christos sysctl_createv(clog, 0, NULL, NULL,
254 1.13 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
255 1.13 christos CTLTYPE_INT, "usercrypto",
256 1.13 christos SYSCTL_DESCR("Enable/disable user-mode access to "
257 1.13 christos "crypto support"),
258 1.13 christos NULL, 0, &crypto_usercrypto, 0,
259 1.13 christos CTL_KERN, CTL_CREATE, CTL_EOL);
260 1.13 christos sysctl_createv(clog, 0, NULL, NULL,
261 1.13 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
262 1.13 christos CTLTYPE_INT, "userasymcrypto",
263 1.13 christos SYSCTL_DESCR("Enable/disable user-mode access to "
264 1.13 christos "asymmetric crypto support"),
265 1.13 christos NULL, 0, &crypto_userasymcrypto, 0,
266 1.13 christos CTL_KERN, CTL_CREATE, CTL_EOL);
267 1.13 christos sysctl_createv(clog, 0, NULL, NULL,
268 1.13 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
269 1.13 christos CTLTYPE_INT, "cryptodevallowsoft",
270 1.13 christos SYSCTL_DESCR("Enable/disable use of software "
271 1.13 christos "asymmetric crypto support"),
272 1.13 christos NULL, 0, &crypto_devallowsoft, 0,
273 1.13 christos CTL_KERN, CTL_CREATE, CTL_EOL);
274 1.73 knakahar
275 1.73 knakahar sysctl_createv(clog, 0, NULL, &ocnode,
276 1.73 knakahar CTLFLAG_PERMANENT,
277 1.73 knakahar CTLTYPE_NODE, "opencrypto",
278 1.73 knakahar SYSCTL_DESCR("opencrypto related entries"),
279 1.73 knakahar NULL, 0, NULL, 0,
280 1.73 knakahar CTL_CREATE, CTL_EOL);
281 1.73 knakahar
282 1.73 knakahar sysctl_createv(clog, 0, &ocnode, &retqnode,
283 1.73 knakahar CTLFLAG_PERMANENT,
284 1.73 knakahar CTLTYPE_NODE, "crypto_ret_q",
285 1.73 knakahar SYSCTL_DESCR("crypto_ret_q related entries"),
286 1.73 knakahar NULL, 0, NULL, 0,
287 1.73 knakahar CTL_CREATE, CTL_EOL);
288 1.73 knakahar sysctl_createv(clog, 0, &retqnode, NULL,
289 1.73 knakahar CTLFLAG_PERMANENT|CTLFLAG_READONLY,
290 1.73 knakahar CTLTYPE_INT, "len",
291 1.73 knakahar SYSCTL_DESCR("Current queue length"),
292 1.73 knakahar sysctl_opencrypto_q_len, 0,
293 1.73 knakahar (void *)&crypto_crp_ret_q_len, 0,
294 1.73 knakahar CTL_CREATE, CTL_EOL);
295 1.73 knakahar sysctl_createv(clog, 0, &retqnode, NULL,
296 1.73 knakahar CTLFLAG_PERMANENT|CTLFLAG_READONLY,
297 1.73 knakahar CTLTYPE_INT, "drops",
298 1.73 knakahar SYSCTL_DESCR("Crypto requests dropped due to full ret queue"),
299 1.73 knakahar sysctl_opencrypto_q_drops, 0,
300 1.73 knakahar (void *)&crypto_crp_ret_q_drops, 0,
301 1.73 knakahar CTL_CREATE, CTL_EOL);
302 1.73 knakahar sysctl_createv(clog, 0, &retqnode, NULL,
303 1.73 knakahar CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
304 1.73 knakahar CTLTYPE_INT, "maxlen",
305 1.73 knakahar SYSCTL_DESCR("Maximum allowed queue length"),
306 1.73 knakahar sysctl_opencrypto_q_maxlen, 0,
307 1.73 knakahar (void *)&crypto_crp_ret_q_maxlen, 0,
308 1.73 knakahar CTL_CREATE, CTL_EOL);
309 1.73 knakahar
310 1.73 knakahar sysctl_createv(clog, 0, &ocnode, &retkqnode,
311 1.73 knakahar CTLFLAG_PERMANENT,
312 1.73 knakahar CTLTYPE_NODE, "crypto_ret_kq",
313 1.73 knakahar SYSCTL_DESCR("crypto_ret_kq related entries"),
314 1.73 knakahar NULL, 0, NULL, 0,
315 1.73 knakahar CTL_CREATE, CTL_EOL);
316 1.73 knakahar sysctl_createv(clog, 0, &retkqnode, NULL,
317 1.73 knakahar CTLFLAG_PERMANENT|CTLFLAG_READONLY,
318 1.73 knakahar CTLTYPE_INT, "len",
319 1.73 knakahar SYSCTL_DESCR("Current queue length"),
320 1.73 knakahar sysctl_opencrypto_q_len, 0,
321 1.73 knakahar (void *)&crypto_crp_ret_kq_len, 0,
322 1.73 knakahar CTL_CREATE, CTL_EOL);
323 1.73 knakahar sysctl_createv(clog, 0, &retkqnode, NULL,
324 1.73 knakahar CTLFLAG_PERMANENT|CTLFLAG_READONLY,
325 1.73 knakahar CTLTYPE_INT, "drops",
326 1.73 knakahar SYSCTL_DESCR("Crypto requests dropped due to full ret queue"),
327 1.73 knakahar sysctl_opencrypto_q_drops, 0,
328 1.73 knakahar (void *)&crypto_crp_ret_kq_drops, 0,
329 1.73 knakahar CTL_CREATE, CTL_EOL);
330 1.73 knakahar sysctl_createv(clog, 0, &retkqnode, NULL,
331 1.73 knakahar CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
332 1.73 knakahar CTLTYPE_INT, "maxlen",
333 1.73 knakahar SYSCTL_DESCR("Maximum allowed queue length"),
334 1.73 knakahar sysctl_opencrypto_q_maxlen, 0,
335 1.73 knakahar (void *)&crypto_crp_ret_kq_maxlen, 0,
336 1.73 knakahar CTL_CREATE, CTL_EOL);
337 1.13 christos }
338 1.1 jonathan
339 1.1 jonathan MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
340 1.1 jonathan
341 1.1 jonathan /*
342 1.1 jonathan * Synchronization: read carefully, this is non-trivial.
343 1.1 jonathan *
344 1.1 jonathan * Crypto requests are submitted via crypto_dispatch. Typically
345 1.1 jonathan * these come in from network protocols at spl0 (output path) or
346 1.1 jonathan * spl[,soft]net (input path).
347 1.1 jonathan *
348 1.1 jonathan * Requests are typically passed on the driver directly, but they
349 1.1 jonathan * may also be queued for processing by a software interrupt thread,
350 1.10 perry * cryptointr, that runs at splsoftcrypto. This thread dispatches
351 1.1 jonathan * the requests to crypto drivers (h/w or s/w) who call crypto_done
352 1.1 jonathan * when a request is complete. Hardware crypto drivers are assumed
353 1.1 jonathan * to register their IRQ's as network devices so their interrupt handlers
354 1.1 jonathan * and subsequent "done callbacks" happen at spl[imp,net].
355 1.1 jonathan *
356 1.1 jonathan * Completed crypto ops are queued for a separate kernel thread that
357 1.1 jonathan * handles the callbacks at spl0. This decoupling insures the crypto
358 1.1 jonathan * driver interrupt service routine is not delayed while the callback
359 1.1 jonathan * takes place and that callbacks are delivered after a context switch
360 1.1 jonathan * (as opposed to a software interrupt that clients must block).
361 1.1 jonathan *
362 1.1 jonathan * This scheme is not intended for SMP machines.
363 1.10 perry */
364 1.1 jonathan static void cryptointr(void); /* swi thread to dispatch ops */
365 1.1 jonathan static void cryptoret(void); /* kernel thread for callbacks*/
366 1.20 ad static struct lwp *cryptothread;
367 1.46 pgoyette static int crypto_destroy(bool);
368 1.1 jonathan static int crypto_invoke(struct cryptop *crp, int hint);
369 1.1 jonathan static int crypto_kinvoke(struct cryptkop *krp, int hint);
370 1.1 jonathan
371 1.1 jonathan static struct cryptostats cryptostats;
372 1.23 tls #ifdef CRYPTO_TIMING
373 1.1 jonathan static int crypto_timing = 0;
374 1.23 tls #endif
375 1.1 jonathan
376 1.47 christos static struct sysctllog *sysctl_opencrypto_clog;
377 1.44 pgoyette
378 1.12 yamt static int
379 1.11 thorpej crypto_init0(void)
380 1.1 jonathan {
381 1.1 jonathan int error;
382 1.1 jonathan
383 1.57 knakahar mutex_init(&crypto_drv_mtx, MUTEX_DEFAULT, IPL_NONE);
384 1.40 drochner mutex_init(&crypto_q_mtx, MUTEX_DEFAULT, IPL_NET);
385 1.40 drochner mutex_init(&crypto_ret_q_mtx, MUTEX_DEFAULT, IPL_NET);
386 1.36 pgoyette cv_init(&cryptoret_cv, "crypto_w");
387 1.23 tls pool_init(&cryptop_pool, sizeof(struct cryptop), 0, 0,
388 1.48 msaitoh 0, "cryptop", NULL, IPL_NET);
389 1.23 tls pool_init(&cryptodesc_pool, sizeof(struct cryptodesc), 0, 0,
390 1.23 tls 0, "cryptodesc", NULL, IPL_NET);
391 1.23 tls pool_init(&cryptkop_pool, sizeof(struct cryptkop), 0, 0,
392 1.23 tls 0, "cryptkop", NULL, IPL_NET);
393 1.1 jonathan
394 1.11 thorpej crypto_drivers = malloc(CRYPTO_DRIVERS_INITIAL *
395 1.1 jonathan sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
396 1.1 jonathan if (crypto_drivers == NULL) {
397 1.1 jonathan printf("crypto_init: cannot malloc driver table\n");
398 1.46 pgoyette return ENOMEM;
399 1.1 jonathan }
400 1.11 thorpej crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
401 1.1 jonathan
402 1.1 jonathan softintr_cookie = register_swi(SWI_CRYPTO, cryptointr);
403 1.25 tls error = kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL,
404 1.37 christos (void (*)(void *))cryptoret, NULL, &cryptothread, "cryptoret");
405 1.1 jonathan if (error) {
406 1.1 jonathan printf("crypto_init: cannot start cryptoret thread; error %d",
407 1.1 jonathan error);
408 1.46 pgoyette return crypto_destroy(false);
409 1.1 jonathan }
410 1.20 ad
411 1.44 pgoyette sysctl_opencrypto_setup(&sysctl_opencrypto_clog);
412 1.72 knakahar
413 1.12 yamt return 0;
414 1.11 thorpej }
415 1.11 thorpej
416 1.46 pgoyette int
417 1.11 thorpej crypto_init(void)
418 1.11 thorpej {
419 1.18 daniel static ONCE_DECL(crypto_init_once);
420 1.11 thorpej
421 1.46 pgoyette return RUN_ONCE(&crypto_init_once, crypto_init0);
422 1.1 jonathan }
423 1.1 jonathan
424 1.46 pgoyette static int
425 1.46 pgoyette crypto_destroy(bool exit_kthread)
426 1.1 jonathan {
427 1.46 pgoyette int i;
428 1.46 pgoyette
429 1.46 pgoyette if (exit_kthread) {
430 1.46 pgoyette mutex_spin_enter(&crypto_ret_q_mtx);
431 1.46 pgoyette
432 1.46 pgoyette /* if we have any in-progress requests, don't unload */
433 1.51 knakahar if (!TAILQ_EMPTY(&crp_q) || !TAILQ_EMPTY(&crp_kq)) {
434 1.51 knakahar mutex_spin_exit(&crypto_ret_q_mtx);
435 1.46 pgoyette return EBUSY;
436 1.51 knakahar }
437 1.46 pgoyette
438 1.46 pgoyette for (i = 0; i < crypto_drivers_num; i++)
439 1.46 pgoyette if (crypto_drivers[i].cc_sessions != 0)
440 1.46 pgoyette break;
441 1.51 knakahar if (i < crypto_drivers_num) {
442 1.51 knakahar mutex_spin_exit(&crypto_ret_q_mtx);
443 1.46 pgoyette return EBUSY;
444 1.51 knakahar }
445 1.46 pgoyette
446 1.46 pgoyette /* kick the cryptoret thread and wait for it to exit */
447 1.46 pgoyette crypto_exit_flag = 1;
448 1.46 pgoyette cv_signal(&cryptoret_cv);
449 1.46 pgoyette
450 1.46 pgoyette while (crypto_exit_flag != 0)
451 1.46 pgoyette cv_wait(&cryptoret_cv, &crypto_ret_q_mtx);
452 1.46 pgoyette mutex_spin_exit(&crypto_ret_q_mtx);
453 1.46 pgoyette }
454 1.46 pgoyette
455 1.46 pgoyette if (sysctl_opencrypto_clog != NULL)
456 1.46 pgoyette sysctl_teardown(&sysctl_opencrypto_clog);
457 1.46 pgoyette
458 1.46 pgoyette unregister_swi(SWI_CRYPTO, cryptointr);
459 1.46 pgoyette
460 1.57 knakahar mutex_enter(&crypto_drv_mtx);
461 1.1 jonathan if (crypto_drivers != NULL)
462 1.1 jonathan free(crypto_drivers, M_CRYPTO_DATA);
463 1.57 knakahar mutex_exit(&crypto_drv_mtx);
464 1.46 pgoyette
465 1.46 pgoyette pool_destroy(&cryptop_pool);
466 1.46 pgoyette pool_destroy(&cryptodesc_pool);
467 1.46 pgoyette pool_destroy(&cryptkop_pool);
468 1.46 pgoyette
469 1.46 pgoyette cv_destroy(&cryptoret_cv);
470 1.46 pgoyette
471 1.46 pgoyette mutex_destroy(&crypto_ret_q_mtx);
472 1.46 pgoyette mutex_destroy(&crypto_q_mtx);
473 1.57 knakahar mutex_destroy(&crypto_drv_mtx);
474 1.46 pgoyette
475 1.46 pgoyette return 0;
476 1.1 jonathan }
477 1.1 jonathan
478 1.1 jonathan /*
479 1.57 knakahar * Create a new session.
480 1.1 jonathan */
481 1.1 jonathan int
482 1.1 jonathan crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
483 1.1 jonathan {
484 1.1 jonathan struct cryptoini *cr;
485 1.1 jonathan u_int32_t hid, lid;
486 1.1 jonathan int err = EINVAL;
487 1.1 jonathan
488 1.57 knakahar mutex_enter(&crypto_drv_mtx);
489 1.1 jonathan
490 1.1 jonathan if (crypto_drivers == NULL)
491 1.1 jonathan goto done;
492 1.1 jonathan
493 1.1 jonathan /*
494 1.1 jonathan * The algorithm we use here is pretty stupid; just use the
495 1.1 jonathan * first driver that supports all the algorithms we need.
496 1.1 jonathan *
497 1.1 jonathan * XXX We need more smarts here (in real life too, but that's
498 1.1 jonathan * XXX another story altogether).
499 1.1 jonathan */
500 1.1 jonathan
501 1.1 jonathan for (hid = 0; hid < crypto_drivers_num; hid++) {
502 1.1 jonathan /*
503 1.1 jonathan * If it's not initialized or has remaining sessions
504 1.1 jonathan * referencing it, skip.
505 1.1 jonathan */
506 1.1 jonathan if (crypto_drivers[hid].cc_newsession == NULL ||
507 1.1 jonathan (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP))
508 1.1 jonathan continue;
509 1.1 jonathan
510 1.1 jonathan /* Hardware required -- ignore software drivers. */
511 1.1 jonathan if (hard > 0 &&
512 1.1 jonathan (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE))
513 1.1 jonathan continue;
514 1.1 jonathan /* Software required -- ignore hardware drivers. */
515 1.1 jonathan if (hard < 0 &&
516 1.1 jonathan (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0)
517 1.1 jonathan continue;
518 1.1 jonathan
519 1.1 jonathan /* See if all the algorithms are supported. */
520 1.1 jonathan for (cr = cri; cr; cr = cr->cri_next)
521 1.33 darran if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0) {
522 1.64 knakahar DPRINTF("alg %d not supported\n", cr->cri_alg);
523 1.1 jonathan break;
524 1.33 darran }
525 1.1 jonathan
526 1.1 jonathan if (cr == NULL) {
527 1.1 jonathan /* Ok, all algorithms are supported. */
528 1.1 jonathan
529 1.1 jonathan /*
530 1.1 jonathan * Can't do everything in one session.
531 1.1 jonathan *
532 1.1 jonathan * XXX Fix this. We need to inject a "virtual" session layer right
533 1.1 jonathan * XXX about here.
534 1.1 jonathan */
535 1.1 jonathan
536 1.1 jonathan /* Call the driver initialization routine. */
537 1.1 jonathan lid = hid; /* Pass the driver ID. */
538 1.1 jonathan err = crypto_drivers[hid].cc_newsession(
539 1.1 jonathan crypto_drivers[hid].cc_arg, &lid, cri);
540 1.1 jonathan if (err == 0) {
541 1.1 jonathan (*sid) = hid;
542 1.1 jonathan (*sid) <<= 32;
543 1.1 jonathan (*sid) |= (lid & 0xffffffff);
544 1.1 jonathan crypto_drivers[hid].cc_sessions++;
545 1.52 knakahar } else {
546 1.64 knakahar DPRINTF("crypto_drivers[%d].cc_newsession() failed. error=%d\n",
547 1.64 knakahar hid, err);
548 1.1 jonathan }
549 1.1 jonathan goto done;
550 1.1 jonathan /*break;*/
551 1.1 jonathan }
552 1.1 jonathan }
553 1.1 jonathan done:
554 1.57 knakahar mutex_exit(&crypto_drv_mtx);
555 1.1 jonathan return err;
556 1.1 jonathan }
557 1.1 jonathan
558 1.1 jonathan /*
559 1.1 jonathan * Delete an existing session (or a reserved session on an unregistered
560 1.57 knakahar * driver).
561 1.1 jonathan */
562 1.1 jonathan int
563 1.1 jonathan crypto_freesession(u_int64_t sid)
564 1.1 jonathan {
565 1.1 jonathan u_int32_t hid;
566 1.1 jonathan int err = 0;
567 1.1 jonathan
568 1.57 knakahar mutex_enter(&crypto_drv_mtx);
569 1.1 jonathan
570 1.1 jonathan if (crypto_drivers == NULL) {
571 1.1 jonathan err = EINVAL;
572 1.1 jonathan goto done;
573 1.1 jonathan }
574 1.1 jonathan
575 1.1 jonathan /* Determine two IDs. */
576 1.35 jakllsch hid = CRYPTO_SESID2HID(sid);
577 1.1 jonathan
578 1.1 jonathan if (hid >= crypto_drivers_num) {
579 1.1 jonathan err = ENOENT;
580 1.1 jonathan goto done;
581 1.1 jonathan }
582 1.1 jonathan
583 1.1 jonathan if (crypto_drivers[hid].cc_sessions)
584 1.1 jonathan crypto_drivers[hid].cc_sessions--;
585 1.1 jonathan
586 1.1 jonathan /* Call the driver cleanup routine, if available. */
587 1.23 tls if (crypto_drivers[hid].cc_freesession) {
588 1.1 jonathan err = crypto_drivers[hid].cc_freesession(
589 1.1 jonathan crypto_drivers[hid].cc_arg, sid);
590 1.23 tls }
591 1.1 jonathan else
592 1.1 jonathan err = 0;
593 1.1 jonathan
594 1.1 jonathan /*
595 1.1 jonathan * If this was the last session of a driver marked as invalid,
596 1.1 jonathan * make the entry available for reuse.
597 1.1 jonathan */
598 1.1 jonathan if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) &&
599 1.1 jonathan crypto_drivers[hid].cc_sessions == 0)
600 1.31 cegger memset(&crypto_drivers[hid], 0, sizeof(struct cryptocap));
601 1.1 jonathan
602 1.1 jonathan done:
603 1.57 knakahar mutex_exit(&crypto_drv_mtx);
604 1.1 jonathan return err;
605 1.1 jonathan }
606 1.1 jonathan
607 1.1 jonathan /*
608 1.1 jonathan * Return an unused driver id. Used by drivers prior to registering
609 1.1 jonathan * support for the algorithms they handle.
610 1.1 jonathan */
611 1.1 jonathan int32_t
612 1.1 jonathan crypto_get_driverid(u_int32_t flags)
613 1.1 jonathan {
614 1.1 jonathan struct cryptocap *newdrv;
615 1.23 tls int i;
616 1.1 jonathan
617 1.46 pgoyette (void)crypto_init(); /* XXX oh, this is foul! */
618 1.11 thorpej
619 1.57 knakahar mutex_enter(&crypto_drv_mtx);
620 1.1 jonathan for (i = 0; i < crypto_drivers_num; i++)
621 1.1 jonathan if (crypto_drivers[i].cc_process == NULL &&
622 1.1 jonathan (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
623 1.1 jonathan crypto_drivers[i].cc_sessions == 0)
624 1.1 jonathan break;
625 1.1 jonathan
626 1.1 jonathan /* Out of entries, allocate some more. */
627 1.1 jonathan if (i == crypto_drivers_num) {
628 1.1 jonathan /* Be careful about wrap-around. */
629 1.1 jonathan if (2 * crypto_drivers_num <= crypto_drivers_num) {
630 1.57 knakahar mutex_exit(&crypto_drv_mtx);
631 1.1 jonathan printf("crypto: driver count wraparound!\n");
632 1.1 jonathan return -1;
633 1.1 jonathan }
634 1.1 jonathan
635 1.1 jonathan newdrv = malloc(2 * crypto_drivers_num *
636 1.1 jonathan sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
637 1.1 jonathan if (newdrv == NULL) {
638 1.57 knakahar mutex_exit(&crypto_drv_mtx);
639 1.1 jonathan printf("crypto: no space to expand driver table!\n");
640 1.1 jonathan return -1;
641 1.1 jonathan }
642 1.1 jonathan
643 1.34 tsutsui memcpy(newdrv, crypto_drivers,
644 1.1 jonathan crypto_drivers_num * sizeof(struct cryptocap));
645 1.1 jonathan
646 1.1 jonathan crypto_drivers_num *= 2;
647 1.1 jonathan
648 1.1 jonathan free(crypto_drivers, M_CRYPTO_DATA);
649 1.1 jonathan crypto_drivers = newdrv;
650 1.1 jonathan }
651 1.1 jonathan
652 1.1 jonathan /* NB: state is zero'd on free */
653 1.1 jonathan crypto_drivers[i].cc_sessions = 1; /* Mark */
654 1.1 jonathan crypto_drivers[i].cc_flags = flags;
655 1.1 jonathan
656 1.1 jonathan if (bootverbose)
657 1.1 jonathan printf("crypto: assign driver %u, flags %u\n", i, flags);
658 1.1 jonathan
659 1.57 knakahar mutex_exit(&crypto_drv_mtx);
660 1.1 jonathan
661 1.1 jonathan return i;
662 1.1 jonathan }
663 1.1 jonathan
664 1.1 jonathan static struct cryptocap *
665 1.1 jonathan crypto_checkdriver(u_int32_t hid)
666 1.1 jonathan {
667 1.1 jonathan if (crypto_drivers == NULL)
668 1.1 jonathan return NULL;
669 1.1 jonathan return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
670 1.1 jonathan }
671 1.1 jonathan
672 1.1 jonathan /*
673 1.1 jonathan * Register support for a key-related algorithm. This routine
674 1.1 jonathan * is called once for each algorithm supported a driver.
675 1.1 jonathan */
676 1.1 jonathan int
677 1.1 jonathan crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags,
678 1.37 christos int (*kprocess)(void *, struct cryptkop *, int),
679 1.1 jonathan void *karg)
680 1.1 jonathan {
681 1.1 jonathan struct cryptocap *cap;
682 1.1 jonathan int err;
683 1.1 jonathan
684 1.57 knakahar mutex_enter(&crypto_drv_mtx);
685 1.1 jonathan
686 1.1 jonathan cap = crypto_checkdriver(driverid);
687 1.1 jonathan if (cap != NULL &&
688 1.1 jonathan (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
689 1.1 jonathan /*
690 1.1 jonathan * XXX Do some performance testing to determine placing.
691 1.1 jonathan * XXX We probably need an auxiliary data structure that
692 1.1 jonathan * XXX describes relative performances.
693 1.1 jonathan */
694 1.1 jonathan
695 1.1 jonathan cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
696 1.23 tls if (bootverbose) {
697 1.23 tls printf("crypto: driver %u registers key alg %u "
698 1.23 tls " flags %u\n",
699 1.23 tls driverid,
700 1.23 tls kalg,
701 1.23 tls flags
702 1.1 jonathan );
703 1.23 tls }
704 1.1 jonathan
705 1.1 jonathan if (cap->cc_kprocess == NULL) {
706 1.1 jonathan cap->cc_karg = karg;
707 1.1 jonathan cap->cc_kprocess = kprocess;
708 1.1 jonathan }
709 1.1 jonathan err = 0;
710 1.1 jonathan } else
711 1.1 jonathan err = EINVAL;
712 1.1 jonathan
713 1.57 knakahar mutex_exit(&crypto_drv_mtx);
714 1.1 jonathan return err;
715 1.1 jonathan }
716 1.1 jonathan
717 1.1 jonathan /*
718 1.1 jonathan * Register support for a non-key-related algorithm. This routine
719 1.1 jonathan * is called once for each such algorithm supported by a driver.
720 1.1 jonathan */
721 1.1 jonathan int
722 1.1 jonathan crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
723 1.1 jonathan u_int32_t flags,
724 1.37 christos int (*newses)(void *, u_int32_t*, struct cryptoini*),
725 1.37 christos int (*freeses)(void *, u_int64_t),
726 1.37 christos int (*process)(void *, struct cryptop *, int),
727 1.1 jonathan void *arg)
728 1.1 jonathan {
729 1.1 jonathan struct cryptocap *cap;
730 1.23 tls int err;
731 1.1 jonathan
732 1.57 knakahar mutex_enter(&crypto_drv_mtx);
733 1.1 jonathan
734 1.1 jonathan cap = crypto_checkdriver(driverid);
735 1.1 jonathan /* NB: algorithms are in the range [1..max] */
736 1.1 jonathan if (cap != NULL &&
737 1.1 jonathan (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
738 1.1 jonathan /*
739 1.1 jonathan * XXX Do some performance testing to determine placing.
740 1.1 jonathan * XXX We probably need an auxiliary data structure that
741 1.1 jonathan * XXX describes relative performances.
742 1.1 jonathan */
743 1.1 jonathan
744 1.1 jonathan cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
745 1.1 jonathan cap->cc_max_op_len[alg] = maxoplen;
746 1.23 tls if (bootverbose) {
747 1.23 tls printf("crypto: driver %u registers alg %u "
748 1.23 tls "flags %u maxoplen %u\n",
749 1.23 tls driverid,
750 1.23 tls alg,
751 1.23 tls flags,
752 1.23 tls maxoplen
753 1.1 jonathan );
754 1.23 tls }
755 1.1 jonathan
756 1.1 jonathan if (cap->cc_process == NULL) {
757 1.1 jonathan cap->cc_arg = arg;
758 1.1 jonathan cap->cc_newsession = newses;
759 1.1 jonathan cap->cc_process = process;
760 1.1 jonathan cap->cc_freesession = freeses;
761 1.1 jonathan cap->cc_sessions = 0; /* Unmark */
762 1.1 jonathan }
763 1.1 jonathan err = 0;
764 1.1 jonathan } else
765 1.1 jonathan err = EINVAL;
766 1.1 jonathan
767 1.57 knakahar mutex_exit(&crypto_drv_mtx);
768 1.1 jonathan return err;
769 1.1 jonathan }
770 1.1 jonathan
771 1.61 knakahar static int
772 1.62 knakahar crypto_unregister_locked(u_int32_t driverid, int alg, bool all)
773 1.61 knakahar {
774 1.61 knakahar int i;
775 1.61 knakahar u_int32_t ses;
776 1.61 knakahar struct cryptocap *cap;
777 1.61 knakahar bool lastalg = true;
778 1.61 knakahar
779 1.61 knakahar KASSERT(mutex_owned(&crypto_drv_mtx));
780 1.61 knakahar
781 1.61 knakahar if (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)
782 1.61 knakahar return EINVAL;
783 1.61 knakahar
784 1.61 knakahar cap = crypto_checkdriver(driverid);
785 1.62 knakahar if (cap == NULL || (!all && cap->cc_alg[alg] == 0))
786 1.61 knakahar return EINVAL;
787 1.61 knakahar
788 1.61 knakahar cap->cc_alg[alg] = 0;
789 1.61 knakahar cap->cc_max_op_len[alg] = 0;
790 1.61 knakahar
791 1.62 knakahar if (all) {
792 1.62 knakahar if (alg != CRYPTO_ALGORITHM_MAX)
793 1.61 knakahar lastalg = false;
794 1.62 knakahar } else {
795 1.62 knakahar /* Was this the last algorithm ? */
796 1.62 knakahar for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++)
797 1.62 knakahar if (cap->cc_alg[i] != 0) {
798 1.62 knakahar lastalg = false;
799 1.62 knakahar break;
800 1.62 knakahar }
801 1.62 knakahar }
802 1.61 knakahar if (lastalg) {
803 1.61 knakahar ses = cap->cc_sessions;
804 1.61 knakahar memset(cap, 0, sizeof(struct cryptocap));
805 1.61 knakahar if (ses != 0) {
806 1.61 knakahar /*
807 1.61 knakahar * If there are pending sessions, just mark as invalid.
808 1.61 knakahar */
809 1.61 knakahar cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
810 1.61 knakahar cap->cc_sessions = ses;
811 1.61 knakahar }
812 1.61 knakahar }
813 1.61 knakahar
814 1.61 knakahar return 0;
815 1.61 knakahar }
816 1.61 knakahar
817 1.1 jonathan /*
818 1.1 jonathan * Unregister a crypto driver. If there are pending sessions using it,
819 1.1 jonathan * leave enough information around so that subsequent calls using those
820 1.1 jonathan * sessions will correctly detect the driver has been unregistered and
821 1.1 jonathan * reroute requests.
822 1.1 jonathan */
823 1.1 jonathan int
824 1.1 jonathan crypto_unregister(u_int32_t driverid, int alg)
825 1.1 jonathan {
826 1.61 knakahar int err;
827 1.1 jonathan
828 1.57 knakahar mutex_enter(&crypto_drv_mtx);
829 1.62 knakahar err = crypto_unregister_locked(driverid, alg, false);
830 1.61 knakahar mutex_exit(&crypto_drv_mtx);
831 1.1 jonathan
832 1.1 jonathan return err;
833 1.1 jonathan }
834 1.1 jonathan
835 1.1 jonathan /*
836 1.1 jonathan * Unregister all algorithms associated with a crypto driver.
837 1.1 jonathan * If there are pending sessions using it, leave enough information
838 1.1 jonathan * around so that subsequent calls using those sessions will
839 1.1 jonathan * correctly detect the driver has been unregistered and reroute
840 1.1 jonathan * requests.
841 1.1 jonathan */
842 1.1 jonathan int
843 1.1 jonathan crypto_unregister_all(u_int32_t driverid)
844 1.1 jonathan {
845 1.62 knakahar int err, i;
846 1.1 jonathan
847 1.57 knakahar mutex_enter(&crypto_drv_mtx);
848 1.62 knakahar for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
849 1.62 knakahar err = crypto_unregister_locked(driverid, i, true);
850 1.62 knakahar if (err)
851 1.62 knakahar break;
852 1.62 knakahar }
853 1.62 knakahar mutex_exit(&crypto_drv_mtx);
854 1.1 jonathan
855 1.1 jonathan return err;
856 1.1 jonathan }
857 1.1 jonathan
858 1.1 jonathan /*
859 1.1 jonathan * Clear blockage on a driver. The what parameter indicates whether
860 1.1 jonathan * the driver is now ready for cryptop's and/or cryptokop's.
861 1.1 jonathan */
862 1.1 jonathan int
863 1.1 jonathan crypto_unblock(u_int32_t driverid, int what)
864 1.1 jonathan {
865 1.1 jonathan struct cryptocap *cap;
866 1.55 knakahar int needwakeup = 0;
867 1.1 jonathan
868 1.40 drochner mutex_spin_enter(&crypto_q_mtx);
869 1.1 jonathan cap = crypto_checkdriver(driverid);
870 1.55 knakahar if (cap == NULL) {
871 1.40 drochner mutex_spin_exit(&crypto_q_mtx);
872 1.55 knakahar return EINVAL;
873 1.55 knakahar }
874 1.55 knakahar
875 1.55 knakahar if (what & CRYPTO_SYMQ) {
876 1.55 knakahar needwakeup |= cap->cc_qblocked;
877 1.55 knakahar cap->cc_qblocked = 0;
878 1.55 knakahar }
879 1.55 knakahar if (what & CRYPTO_ASYMQ) {
880 1.55 knakahar needwakeup |= cap->cc_kqblocked;
881 1.55 knakahar cap->cc_kqblocked = 0;
882 1.24 tls }
883 1.56 knakahar mutex_spin_exit(&crypto_q_mtx);
884 1.55 knakahar if (needwakeup)
885 1.55 knakahar setsoftcrypto(softintr_cookie);
886 1.1 jonathan
887 1.55 knakahar return 0;
888 1.1 jonathan }
889 1.1 jonathan
890 1.1 jonathan /*
891 1.1 jonathan * Dispatch a crypto request to a driver or queue
892 1.1 jonathan * it, to be processed by the kernel thread.
893 1.1 jonathan */
894 1.1 jonathan int
895 1.1 jonathan crypto_dispatch(struct cryptop *crp)
896 1.1 jonathan {
897 1.59 knakahar u_int32_t hid;
898 1.23 tls int result;
899 1.65 knakahar struct cryptocap *cap;
900 1.1 jonathan
901 1.59 knakahar KASSERT(crp != NULL);
902 1.59 knakahar
903 1.59 knakahar hid = CRYPTO_SESID2HID(crp->crp_sid);
904 1.59 knakahar
905 1.64 knakahar DPRINTF("crp %p, alg %d\n", crp, crp->crp_desc->crd_alg);
906 1.1 jonathan
907 1.1 jonathan cryptostats.cs_ops++;
908 1.1 jonathan
909 1.1 jonathan #ifdef CRYPTO_TIMING
910 1.1 jonathan if (crypto_timing)
911 1.1 jonathan nanouptime(&crp->crp_tstamp);
912 1.1 jonathan #endif
913 1.58 knakahar
914 1.65 knakahar if ((crp->crp_flags & CRYPTO_F_BATCH) != 0) {
915 1.1 jonathan int wasempty = TAILQ_EMPTY(&crp_q);
916 1.1 jonathan /*
917 1.1 jonathan * Caller marked the request as ``ok to delay'';
918 1.1 jonathan * queue it for the swi thread. This is desirable
919 1.1 jonathan * when the operation is low priority and/or suitable
920 1.1 jonathan * for batching.
921 1.1 jonathan */
922 1.65 knakahar mutex_spin_enter(&crypto_q_mtx);
923 1.1 jonathan TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
924 1.56 knakahar mutex_spin_exit(&crypto_q_mtx);
925 1.65 knakahar if (wasempty)
926 1.1 jonathan setsoftcrypto(softintr_cookie);
927 1.65 knakahar
928 1.65 knakahar return 0;
929 1.65 knakahar }
930 1.65 knakahar
931 1.65 knakahar mutex_spin_enter(&crypto_q_mtx);
932 1.65 knakahar
933 1.66 knakahar cap = crypto_checkdriver(hid);
934 1.66 knakahar /*
935 1.66 knakahar * TODO:
936 1.66 knakahar * If we can ensure the driver has been valid until the driver is
937 1.66 knakahar * done crypto_unregister(), this migrate operation is not required.
938 1.66 knakahar */
939 1.66 knakahar if (cap == NULL) {
940 1.66 knakahar /*
941 1.66 knakahar * The driver must be detached, so this request will migrate
942 1.66 knakahar * to other drivers in cryptointr() later.
943 1.66 knakahar */
944 1.66 knakahar TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
945 1.66 knakahar mutex_spin_exit(&crypto_q_mtx);
946 1.66 knakahar
947 1.66 knakahar return 0;
948 1.66 knakahar }
949 1.66 knakahar
950 1.65 knakahar /*
951 1.67 knakahar * TODO:
952 1.67 knakahar * cap->cc_qblocked should be protected by a spin lock other than
953 1.67 knakahar * crypto_q_mtx.
954 1.67 knakahar */
955 1.67 knakahar if (cap->cc_qblocked != 0) {
956 1.67 knakahar /*
957 1.67 knakahar * The driver is blocked, just queue the op until
958 1.67 knakahar * it unblocks and the swi thread gets kicked.
959 1.67 knakahar */
960 1.67 knakahar TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
961 1.67 knakahar mutex_spin_exit(&crypto_q_mtx);
962 1.67 knakahar
963 1.67 knakahar return 0;
964 1.67 knakahar }
965 1.67 knakahar
966 1.67 knakahar /*
967 1.65 knakahar * Caller marked the request to be processed
968 1.65 knakahar * immediately; dispatch it directly to the
969 1.65 knakahar * driver unless the driver is currently blocked.
970 1.65 knakahar */
971 1.67 knakahar mutex_spin_exit(&crypto_q_mtx);
972 1.67 knakahar result = crypto_invoke(crp, 0);
973 1.67 knakahar if (result == ERESTART) {
974 1.67 knakahar /*
975 1.67 knakahar * The driver ran out of resources, mark the
976 1.67 knakahar * driver ``blocked'' for cryptop's and put
977 1.67 knakahar * the op on the queue.
978 1.67 knakahar */
979 1.67 knakahar mutex_spin_enter(&crypto_q_mtx);
980 1.67 knakahar crypto_drivers[hid].cc_qblocked = 1;
981 1.67 knakahar TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
982 1.67 knakahar cryptostats.cs_blocks++;
983 1.65 knakahar mutex_spin_exit(&crypto_q_mtx);
984 1.65 knakahar
985 1.65 knakahar /*
986 1.67 knakahar * The crp is enqueued to crp_q, that is,
987 1.67 knakahar * no error occurs. So, this function should
988 1.67 knakahar * not return error.
989 1.65 knakahar */
990 1.1 jonathan result = 0;
991 1.1 jonathan }
992 1.1 jonathan
993 1.1 jonathan return result;
994 1.1 jonathan }
995 1.1 jonathan
996 1.1 jonathan /*
997 1.1 jonathan * Add an asymetric crypto request to a queue,
998 1.1 jonathan * to be processed by the kernel thread.
999 1.1 jonathan */
1000 1.1 jonathan int
1001 1.1 jonathan crypto_kdispatch(struct cryptkop *krp)
1002 1.1 jonathan {
1003 1.1 jonathan struct cryptocap *cap;
1004 1.23 tls int result;
1005 1.1 jonathan
1006 1.59 knakahar KASSERT(krp != NULL);
1007 1.59 knakahar
1008 1.40 drochner mutex_spin_enter(&crypto_q_mtx);
1009 1.1 jonathan cryptostats.cs_kops++;
1010 1.1 jonathan
1011 1.1 jonathan cap = crypto_checkdriver(krp->krp_hid);
1012 1.68 knakahar /*
1013 1.68 knakahar * TODO:
1014 1.68 knakahar * If we can ensure the driver has been valid until the driver is
1015 1.68 knakahar * done crypto_unregister(), this migrate operation is not required.
1016 1.68 knakahar */
1017 1.68 knakahar if (cap == NULL) {
1018 1.68 knakahar TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
1019 1.40 drochner mutex_spin_exit(&crypto_q_mtx);
1020 1.58 knakahar
1021 1.68 knakahar return 0;
1022 1.68 knakahar }
1023 1.68 knakahar
1024 1.68 knakahar if (cap->cc_kqblocked != 0) {
1025 1.1 jonathan /*
1026 1.1 jonathan * The driver is blocked, just queue the op until
1027 1.1 jonathan * it unblocks and the swi thread gets kicked.
1028 1.1 jonathan */
1029 1.1 jonathan TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
1030 1.68 knakahar mutex_spin_exit(&crypto_q_mtx);
1031 1.68 knakahar
1032 1.68 knakahar return 0;
1033 1.68 knakahar }
1034 1.68 knakahar
1035 1.68 knakahar mutex_spin_exit(&crypto_q_mtx);
1036 1.68 knakahar result = crypto_kinvoke(krp, 0);
1037 1.68 knakahar if (result == ERESTART) {
1038 1.68 knakahar /*
1039 1.68 knakahar * The driver ran out of resources, mark the
1040 1.68 knakahar * driver ``blocked'' for cryptop's and put
1041 1.68 knakahar * the op on the queue.
1042 1.68 knakahar */
1043 1.68 knakahar mutex_spin_enter(&crypto_q_mtx);
1044 1.68 knakahar crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1045 1.68 knakahar TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1046 1.68 knakahar cryptostats.cs_kblocks++;
1047 1.68 knakahar mutex_spin_exit(&crypto_q_mtx);
1048 1.68 knakahar
1049 1.68 knakahar /*
1050 1.68 knakahar * The krp is enqueued to crp_kq, that is,
1051 1.68 knakahar * no error occurs. So, this function should
1052 1.68 knakahar * not return error.
1053 1.68 knakahar */
1054 1.1 jonathan result = 0;
1055 1.1 jonathan }
1056 1.1 jonathan
1057 1.1 jonathan return result;
1058 1.1 jonathan }
1059 1.1 jonathan
1060 1.1 jonathan /*
1061 1.1 jonathan * Dispatch an assymetric crypto request to the appropriate crypto devices.
1062 1.1 jonathan */
1063 1.1 jonathan static int
1064 1.1 jonathan crypto_kinvoke(struct cryptkop *krp, int hint)
1065 1.1 jonathan {
1066 1.1 jonathan u_int32_t hid;
1067 1.1 jonathan int error;
1068 1.1 jonathan
1069 1.59 knakahar KASSERT(krp != NULL);
1070 1.59 knakahar
1071 1.1 jonathan /* Sanity checks. */
1072 1.1 jonathan if (krp->krp_callback == NULL) {
1073 1.30 darran cv_destroy(&krp->krp_cv);
1074 1.23 tls pool_put(&cryptkop_pool, krp);
1075 1.1 jonathan return EINVAL;
1076 1.1 jonathan }
1077 1.1 jonathan
1078 1.57 knakahar mutex_enter(&crypto_drv_mtx);
1079 1.1 jonathan for (hid = 0; hid < crypto_drivers_num; hid++) {
1080 1.1 jonathan if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1081 1.1 jonathan crypto_devallowsoft == 0)
1082 1.1 jonathan continue;
1083 1.1 jonathan if (crypto_drivers[hid].cc_kprocess == NULL)
1084 1.1 jonathan continue;
1085 1.1 jonathan if ((crypto_drivers[hid].cc_kalg[krp->krp_op] &
1086 1.1 jonathan CRYPTO_ALG_FLAG_SUPPORTED) == 0)
1087 1.1 jonathan continue;
1088 1.1 jonathan break;
1089 1.1 jonathan }
1090 1.1 jonathan if (hid < crypto_drivers_num) {
1091 1.37 christos int (*process)(void *, struct cryptkop *, int);
1092 1.37 christos void *arg;
1093 1.37 christos
1094 1.37 christos process = crypto_drivers[hid].cc_kprocess;
1095 1.37 christos arg = crypto_drivers[hid].cc_karg;
1096 1.57 knakahar mutex_exit(&crypto_drv_mtx);
1097 1.1 jonathan krp->krp_hid = hid;
1098 1.37 christos error = (*process)(arg, krp, hint);
1099 1.1 jonathan } else {
1100 1.57 knakahar mutex_exit(&crypto_drv_mtx);
1101 1.1 jonathan error = ENODEV;
1102 1.1 jonathan }
1103 1.1 jonathan
1104 1.1 jonathan if (error) {
1105 1.1 jonathan krp->krp_status = error;
1106 1.1 jonathan crypto_kdone(krp);
1107 1.1 jonathan }
1108 1.1 jonathan return 0;
1109 1.1 jonathan }
1110 1.1 jonathan
1111 1.1 jonathan #ifdef CRYPTO_TIMING
1112 1.1 jonathan static void
1113 1.1 jonathan crypto_tstat(struct cryptotstat *ts, struct timespec *tv)
1114 1.1 jonathan {
1115 1.1 jonathan struct timespec now, t;
1116 1.1 jonathan
1117 1.1 jonathan nanouptime(&now);
1118 1.1 jonathan t.tv_sec = now.tv_sec - tv->tv_sec;
1119 1.1 jonathan t.tv_nsec = now.tv_nsec - tv->tv_nsec;
1120 1.1 jonathan if (t.tv_nsec < 0) {
1121 1.1 jonathan t.tv_sec--;
1122 1.1 jonathan t.tv_nsec += 1000000000;
1123 1.1 jonathan }
1124 1.1 jonathan timespecadd(&ts->acc, &t, &t);
1125 1.1 jonathan if (timespeccmp(&t, &ts->min, <))
1126 1.1 jonathan ts->min = t;
1127 1.1 jonathan if (timespeccmp(&t, &ts->max, >))
1128 1.1 jonathan ts->max = t;
1129 1.1 jonathan ts->count++;
1130 1.1 jonathan
1131 1.1 jonathan *tv = now;
1132 1.1 jonathan }
1133 1.1 jonathan #endif
1134 1.1 jonathan
1135 1.1 jonathan /*
1136 1.1 jonathan * Dispatch a crypto request to the appropriate crypto devices.
1137 1.1 jonathan */
1138 1.1 jonathan static int
1139 1.1 jonathan crypto_invoke(struct cryptop *crp, int hint)
1140 1.1 jonathan {
1141 1.1 jonathan u_int32_t hid;
1142 1.1 jonathan
1143 1.59 knakahar KASSERT(crp != NULL);
1144 1.59 knakahar
1145 1.1 jonathan #ifdef CRYPTO_TIMING
1146 1.1 jonathan if (crypto_timing)
1147 1.1 jonathan crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
1148 1.1 jonathan #endif
1149 1.1 jonathan /* Sanity checks. */
1150 1.1 jonathan if (crp->crp_callback == NULL) {
1151 1.1 jonathan return EINVAL;
1152 1.1 jonathan }
1153 1.1 jonathan if (crp->crp_desc == NULL) {
1154 1.1 jonathan crp->crp_etype = EINVAL;
1155 1.1 jonathan crypto_done(crp);
1156 1.1 jonathan return 0;
1157 1.1 jonathan }
1158 1.1 jonathan
1159 1.35 jakllsch hid = CRYPTO_SESID2HID(crp->crp_sid);
1160 1.37 christos
1161 1.63 knakahar if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
1162 1.37 christos int (*process)(void *, struct cryptop *, int);
1163 1.37 christos void *arg;
1164 1.37 christos
1165 1.1 jonathan process = crypto_drivers[hid].cc_process;
1166 1.37 christos arg = crypto_drivers[hid].cc_arg;
1167 1.37 christos
1168 1.37 christos /*
1169 1.37 christos * Invoke the driver to process the request.
1170 1.37 christos */
1171 1.64 knakahar DPRINTF("calling process for %p\n", crp);
1172 1.37 christos return (*process)(arg, crp, hint);
1173 1.1 jonathan } else {
1174 1.1 jonathan struct cryptodesc *crd;
1175 1.16 mrg u_int64_t nid = 0;
1176 1.1 jonathan
1177 1.1 jonathan /*
1178 1.1 jonathan * Driver has unregistered; migrate the session and return
1179 1.1 jonathan * an error to the caller so they'll resubmit the op.
1180 1.1 jonathan */
1181 1.63 knakahar crypto_freesession(crp->crp_sid);
1182 1.63 knakahar
1183 1.1 jonathan for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
1184 1.1 jonathan crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
1185 1.1 jonathan
1186 1.1 jonathan if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
1187 1.1 jonathan crp->crp_sid = nid;
1188 1.1 jonathan
1189 1.1 jonathan crp->crp_etype = EAGAIN;
1190 1.23 tls
1191 1.1 jonathan crypto_done(crp);
1192 1.1 jonathan return 0;
1193 1.1 jonathan }
1194 1.1 jonathan }
1195 1.1 jonathan
1196 1.1 jonathan /*
1197 1.1 jonathan * Release a set of crypto descriptors.
1198 1.1 jonathan */
1199 1.1 jonathan void
1200 1.1 jonathan crypto_freereq(struct cryptop *crp)
1201 1.1 jonathan {
1202 1.1 jonathan struct cryptodesc *crd;
1203 1.1 jonathan
1204 1.1 jonathan if (crp == NULL)
1205 1.1 jonathan return;
1206 1.64 knakahar DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(crp->crp_sid), crp);
1207 1.1 jonathan
1208 1.30 darran /* sanity check */
1209 1.30 darran if (crp->crp_flags & CRYPTO_F_ONRETQ) {
1210 1.30 darran panic("crypto_freereq() freeing crp on RETQ\n");
1211 1.30 darran }
1212 1.30 darran
1213 1.1 jonathan while ((crd = crp->crp_desc) != NULL) {
1214 1.1 jonathan crp->crp_desc = crd->crd_next;
1215 1.1 jonathan pool_put(&cryptodesc_pool, crd);
1216 1.1 jonathan }
1217 1.1 jonathan pool_put(&cryptop_pool, crp);
1218 1.1 jonathan }
1219 1.1 jonathan
1220 1.1 jonathan /*
1221 1.1 jonathan * Acquire a set of crypto descriptors.
1222 1.1 jonathan */
1223 1.1 jonathan struct cryptop *
1224 1.1 jonathan crypto_getreq(int num)
1225 1.1 jonathan {
1226 1.1 jonathan struct cryptodesc *crd;
1227 1.1 jonathan struct cryptop *crp;
1228 1.1 jonathan
1229 1.74 knakahar /*
1230 1.74 knakahar * When crp_ret_q is full, we restrict here to avoid crp_ret_q overflow
1231 1.74 knakahar * by error callback.
1232 1.74 knakahar */
1233 1.74 knakahar if (CRYPTO_Q_IS_FULL(crp_ret_q)) {
1234 1.74 knakahar CRYPTO_Q_INC_DROPS(crp_ret_q);
1235 1.74 knakahar return NULL;
1236 1.74 knakahar }
1237 1.74 knakahar
1238 1.1 jonathan crp = pool_get(&cryptop_pool, 0);
1239 1.1 jonathan if (crp == NULL) {
1240 1.1 jonathan return NULL;
1241 1.1 jonathan }
1242 1.31 cegger memset(crp, 0, sizeof(struct cryptop));
1243 1.1 jonathan
1244 1.1 jonathan while (num--) {
1245 1.1 jonathan crd = pool_get(&cryptodesc_pool, 0);
1246 1.1 jonathan if (crd == NULL) {
1247 1.1 jonathan crypto_freereq(crp);
1248 1.1 jonathan return NULL;
1249 1.1 jonathan }
1250 1.1 jonathan
1251 1.31 cegger memset(crd, 0, sizeof(struct cryptodesc));
1252 1.1 jonathan crd->crd_next = crp->crp_desc;
1253 1.1 jonathan crp->crp_desc = crd;
1254 1.1 jonathan }
1255 1.1 jonathan
1256 1.1 jonathan return crp;
1257 1.1 jonathan }
1258 1.1 jonathan
1259 1.1 jonathan /*
1260 1.1 jonathan * Invoke the callback on behalf of the driver.
1261 1.1 jonathan */
1262 1.1 jonathan void
1263 1.1 jonathan crypto_done(struct cryptop *crp)
1264 1.1 jonathan {
1265 1.23 tls int wasempty;
1266 1.23 tls
1267 1.59 knakahar KASSERT(crp != NULL);
1268 1.59 knakahar
1269 1.1 jonathan if (crp->crp_etype != 0)
1270 1.1 jonathan cryptostats.cs_errs++;
1271 1.1 jonathan #ifdef CRYPTO_TIMING
1272 1.1 jonathan if (crypto_timing)
1273 1.1 jonathan crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
1274 1.1 jonathan #endif
1275 1.64 knakahar DPRINTF("lid[%u]: crp %p\n", CRYPTO_SESID2LID(crp->crp_sid), crp);
1276 1.27 tls
1277 1.1 jonathan /*
1278 1.23 tls * Normal case; queue the callback for the thread.
1279 1.23 tls *
1280 1.23 tls * The return queue is manipulated by the swi thread
1281 1.23 tls * and, potentially, by crypto device drivers calling
1282 1.23 tls * back to mark operations completed. Thus we need
1283 1.23 tls * to mask both while manipulating the return queue.
1284 1.1 jonathan */
1285 1.27 tls if (crp->crp_flags & CRYPTO_F_CBIMM) {
1286 1.27 tls /*
1287 1.27 tls * Do the callback directly. This is ok when the
1288 1.27 tls * callback routine does very little (e.g. the
1289 1.27 tls * /dev/crypto callback method just does a wakeup).
1290 1.27 tls */
1291 1.40 drochner mutex_spin_enter(&crypto_ret_q_mtx);
1292 1.30 darran crp->crp_flags |= CRYPTO_F_DONE;
1293 1.40 drochner mutex_spin_exit(&crypto_ret_q_mtx);
1294 1.30 darran
1295 1.27 tls #ifdef CRYPTO_TIMING
1296 1.27 tls if (crypto_timing) {
1297 1.27 tls /*
1298 1.27 tls * NB: We must copy the timestamp before
1299 1.27 tls * doing the callback as the cryptop is
1300 1.27 tls * likely to be reclaimed.
1301 1.27 tls */
1302 1.27 tls struct timespec t = crp->crp_tstamp;
1303 1.27 tls crypto_tstat(&cryptostats.cs_cb, &t);
1304 1.27 tls crp->crp_callback(crp);
1305 1.27 tls crypto_tstat(&cryptostats.cs_finis, &t);
1306 1.27 tls } else
1307 1.27 tls #endif
1308 1.27 tls crp->crp_callback(crp);
1309 1.27 tls } else {
1310 1.40 drochner mutex_spin_enter(&crypto_ret_q_mtx);
1311 1.30 darran crp->crp_flags |= CRYPTO_F_DONE;
1312 1.52 knakahar #if 0
1313 1.30 darran if (crp->crp_flags & CRYPTO_F_USER) {
1314 1.52 knakahar /*
1315 1.52 knakahar * TODO:
1316 1.52 knakahar * If crp->crp_flags & CRYPTO_F_USER and the used
1317 1.52 knakahar * encryption driver does all the processing in
1318 1.52 knakahar * the same context, we can skip enqueueing crp_ret_q
1319 1.52 knakahar * and cv_signal(&cryptoret_cv).
1320 1.30 darran */
1321 1.64 knakahar DPRINTF("lid[%u]: crp %p CRYPTO_F_USER\n",
1322 1.64 knakahar CRYPTO_SESID2LID(crp->crp_sid), crp);
1323 1.52 knakahar } else
1324 1.52 knakahar #endif
1325 1.52 knakahar {
1326 1.30 darran wasempty = TAILQ_EMPTY(&crp_ret_q);
1327 1.64 knakahar DPRINTF("lid[%u]: queueing %p\n",
1328 1.64 knakahar CRYPTO_SESID2LID(crp->crp_sid), crp);
1329 1.30 darran crp->crp_flags |= CRYPTO_F_ONRETQ;
1330 1.30 darran TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
1331 1.73 knakahar CRYPTO_Q_INC(crp_ret_q);
1332 1.30 darran if (wasempty) {
1333 1.64 knakahar DPRINTF("lid[%u]: waking cryptoret, "
1334 1.35 jakllsch "crp %p hit empty queue\n.",
1335 1.64 knakahar CRYPTO_SESID2LID(crp->crp_sid), crp);
1336 1.30 darran cv_signal(&cryptoret_cv);
1337 1.30 darran }
1338 1.27 tls }
1339 1.40 drochner mutex_spin_exit(&crypto_ret_q_mtx);
1340 1.1 jonathan }
1341 1.1 jonathan }
1342 1.1 jonathan
1343 1.1 jonathan /*
1344 1.1 jonathan * Invoke the callback on behalf of the driver.
1345 1.1 jonathan */
1346 1.1 jonathan void
1347 1.1 jonathan crypto_kdone(struct cryptkop *krp)
1348 1.1 jonathan {
1349 1.23 tls int wasempty;
1350 1.1 jonathan
1351 1.59 knakahar KASSERT(krp != NULL);
1352 1.59 knakahar
1353 1.1 jonathan if (krp->krp_status != 0)
1354 1.1 jonathan cryptostats.cs_kerrs++;
1355 1.27 tls
1356 1.27 tls krp->krp_flags |= CRYPTO_F_DONE;
1357 1.27 tls
1358 1.1 jonathan /*
1359 1.1 jonathan * The return queue is manipulated by the swi thread
1360 1.1 jonathan * and, potentially, by crypto device drivers calling
1361 1.1 jonathan * back to mark operations completed. Thus we need
1362 1.1 jonathan * to mask both while manipulating the return queue.
1363 1.1 jonathan */
1364 1.27 tls if (krp->krp_flags & CRYPTO_F_CBIMM) {
1365 1.27 tls krp->krp_callback(krp);
1366 1.27 tls } else {
1367 1.40 drochner mutex_spin_enter(&crypto_ret_q_mtx);
1368 1.27 tls wasempty = TAILQ_EMPTY(&crp_ret_kq);
1369 1.27 tls krp->krp_flags |= CRYPTO_F_ONRETQ;
1370 1.27 tls TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
1371 1.73 knakahar CRYPTO_Q_INC(crp_ret_kq);
1372 1.27 tls if (wasempty)
1373 1.27 tls cv_signal(&cryptoret_cv);
1374 1.40 drochner mutex_spin_exit(&crypto_ret_q_mtx);
1375 1.27 tls }
1376 1.1 jonathan }
1377 1.1 jonathan
1378 1.1 jonathan int
1379 1.1 jonathan crypto_getfeat(int *featp)
1380 1.1 jonathan {
1381 1.1 jonathan int hid, kalg, feat = 0;
1382 1.1 jonathan
1383 1.57 knakahar if (crypto_userasymcrypto == 0)
1384 1.57 knakahar return 0;
1385 1.1 jonathan
1386 1.57 knakahar mutex_enter(&crypto_drv_mtx);
1387 1.1 jonathan
1388 1.1 jonathan for (hid = 0; hid < crypto_drivers_num; hid++) {
1389 1.1 jonathan if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1390 1.7 jonathan crypto_devallowsoft == 0) {
1391 1.1 jonathan continue;
1392 1.1 jonathan }
1393 1.1 jonathan if (crypto_drivers[hid].cc_kprocess == NULL)
1394 1.1 jonathan continue;
1395 1.1 jonathan for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1396 1.1 jonathan if ((crypto_drivers[hid].cc_kalg[kalg] &
1397 1.1 jonathan CRYPTO_ALG_FLAG_SUPPORTED) != 0)
1398 1.1 jonathan feat |= 1 << kalg;
1399 1.1 jonathan }
1400 1.57 knakahar
1401 1.57 knakahar mutex_exit(&crypto_drv_mtx);
1402 1.1 jonathan *featp = feat;
1403 1.1 jonathan return (0);
1404 1.1 jonathan }
1405 1.1 jonathan
1406 1.1 jonathan /*
1407 1.1 jonathan * Software interrupt thread to dispatch crypto requests.
1408 1.1 jonathan */
1409 1.1 jonathan static void
1410 1.1 jonathan cryptointr(void)
1411 1.1 jonathan {
1412 1.30 darran struct cryptop *crp, *submit, *cnext;
1413 1.30 darran struct cryptkop *krp, *knext;
1414 1.1 jonathan struct cryptocap *cap;
1415 1.23 tls int result, hint;
1416 1.1 jonathan
1417 1.1 jonathan cryptostats.cs_intrs++;
1418 1.40 drochner mutex_spin_enter(&crypto_q_mtx);
1419 1.1 jonathan do {
1420 1.1 jonathan /*
1421 1.1 jonathan * Find the first element in the queue that can be
1422 1.1 jonathan * processed and look-ahead to see if multiple ops
1423 1.1 jonathan * are ready for the same driver.
1424 1.1 jonathan */
1425 1.1 jonathan submit = NULL;
1426 1.1 jonathan hint = 0;
1427 1.30 darran TAILQ_FOREACH_SAFE(crp, &crp_q, crp_next, cnext) {
1428 1.35 jakllsch u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid);
1429 1.1 jonathan cap = crypto_checkdriver(hid);
1430 1.1 jonathan if (cap == NULL || cap->cc_process == NULL) {
1431 1.1 jonathan /* Op needs to be migrated, process it. */
1432 1.69 knakahar submit = crp;
1433 1.1 jonathan break;
1434 1.1 jonathan }
1435 1.70 knakahar
1436 1.70 knakahar /*
1437 1.70 knakahar * skip blocked crp regardless of CRYPTO_F_BATCH
1438 1.70 knakahar */
1439 1.70 knakahar if (cap->cc_qblocked != 0)
1440 1.70 knakahar continue;
1441 1.70 knakahar
1442 1.71 knakahar /*
1443 1.71 knakahar * skip batch crp until the end of crp_q
1444 1.71 knakahar */
1445 1.71 knakahar if ((crp->crp_flags & CRYPTO_F_BATCH) != 0) {
1446 1.71 knakahar if (submit == NULL) {
1447 1.71 knakahar submit = crp;
1448 1.71 knakahar } else {
1449 1.71 knakahar if (CRYPTO_SESID2HID(submit->crp_sid)
1450 1.71 knakahar == hid)
1451 1.71 knakahar hint = CRYPTO_HINT_MORE;
1452 1.71 knakahar }
1453 1.71 knakahar
1454 1.71 knakahar continue;
1455 1.1 jonathan }
1456 1.71 knakahar
1457 1.71 knakahar /*
1458 1.71 knakahar * found first crp which is neither blocked nor batch.
1459 1.71 knakahar */
1460 1.71 knakahar submit = crp;
1461 1.71 knakahar /*
1462 1.71 knakahar * batch crp can be processed much later, so clear hint.
1463 1.71 knakahar */
1464 1.71 knakahar hint = 0;
1465 1.71 knakahar break;
1466 1.1 jonathan }
1467 1.1 jonathan if (submit != NULL) {
1468 1.1 jonathan TAILQ_REMOVE(&crp_q, submit, crp_next);
1469 1.40 drochner mutex_spin_exit(&crypto_q_mtx);
1470 1.1 jonathan result = crypto_invoke(submit, hint);
1471 1.23 tls /* we must take here as the TAILQ op or kinvoke
1472 1.23 tls may need this mutex below. sigh. */
1473 1.48 msaitoh mutex_spin_enter(&crypto_q_mtx);
1474 1.1 jonathan if (result == ERESTART) {
1475 1.1 jonathan /*
1476 1.1 jonathan * The driver ran out of resources, mark the
1477 1.1 jonathan * driver ``blocked'' for cryptop's and put
1478 1.1 jonathan * the request back in the queue. It would
1479 1.1 jonathan * best to put the request back where we got
1480 1.1 jonathan * it but that's hard so for now we put it
1481 1.1 jonathan * at the front. This should be ok; putting
1482 1.1 jonathan * it at the end does not work.
1483 1.1 jonathan */
1484 1.1 jonathan /* XXX validate sid again? */
1485 1.35 jakllsch crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1486 1.1 jonathan TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1487 1.1 jonathan cryptostats.cs_blocks++;
1488 1.1 jonathan }
1489 1.1 jonathan }
1490 1.1 jonathan
1491 1.1 jonathan /* As above, but for key ops */
1492 1.30 darran TAILQ_FOREACH_SAFE(krp, &crp_kq, krp_next, knext) {
1493 1.1 jonathan cap = crypto_checkdriver(krp->krp_hid);
1494 1.1 jonathan if (cap == NULL || cap->cc_kprocess == NULL) {
1495 1.1 jonathan /* Op needs to be migrated, process it. */
1496 1.1 jonathan break;
1497 1.1 jonathan }
1498 1.1 jonathan if (!cap->cc_kqblocked)
1499 1.1 jonathan break;
1500 1.1 jonathan }
1501 1.1 jonathan if (krp != NULL) {
1502 1.1 jonathan TAILQ_REMOVE(&crp_kq, krp, krp_next);
1503 1.40 drochner mutex_spin_exit(&crypto_q_mtx);
1504 1.1 jonathan result = crypto_kinvoke(krp, 0);
1505 1.23 tls /* the next iteration will want the mutex. :-/ */
1506 1.40 drochner mutex_spin_enter(&crypto_q_mtx);
1507 1.1 jonathan if (result == ERESTART) {
1508 1.1 jonathan /*
1509 1.1 jonathan * The driver ran out of resources, mark the
1510 1.1 jonathan * driver ``blocked'' for cryptkop's and put
1511 1.1 jonathan * the request back in the queue. It would
1512 1.1 jonathan * best to put the request back where we got
1513 1.1 jonathan * it but that's hard so for now we put it
1514 1.1 jonathan * at the front. This should be ok; putting
1515 1.1 jonathan * it at the end does not work.
1516 1.1 jonathan */
1517 1.1 jonathan /* XXX validate sid again? */
1518 1.1 jonathan crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1519 1.1 jonathan TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1520 1.1 jonathan cryptostats.cs_kblocks++;
1521 1.1 jonathan }
1522 1.1 jonathan }
1523 1.1 jonathan } while (submit != NULL || krp != NULL);
1524 1.40 drochner mutex_spin_exit(&crypto_q_mtx);
1525 1.1 jonathan }
1526 1.1 jonathan
1527 1.1 jonathan /*
1528 1.1 jonathan * Kernel thread to do callbacks.
1529 1.1 jonathan */
1530 1.1 jonathan static void
1531 1.1 jonathan cryptoret(void)
1532 1.1 jonathan {
1533 1.1 jonathan struct cryptop *crp;
1534 1.1 jonathan struct cryptkop *krp;
1535 1.1 jonathan
1536 1.40 drochner mutex_spin_enter(&crypto_ret_q_mtx);
1537 1.1 jonathan for (;;) {
1538 1.1 jonathan crp = TAILQ_FIRST(&crp_ret_q);
1539 1.23 tls if (crp != NULL) {
1540 1.1 jonathan TAILQ_REMOVE(&crp_ret_q, crp, crp_next);
1541 1.73 knakahar CRYPTO_Q_DEC(crp_ret_q);
1542 1.23 tls crp->crp_flags &= ~CRYPTO_F_ONRETQ;
1543 1.23 tls }
1544 1.1 jonathan krp = TAILQ_FIRST(&crp_ret_kq);
1545 1.23 tls if (krp != NULL) {
1546 1.1 jonathan TAILQ_REMOVE(&crp_ret_kq, krp, krp_next);
1547 1.73 knakahar CRYPTO_Q_DEC(crp_ret_kq);
1548 1.23 tls krp->krp_flags &= ~CRYPTO_F_ONRETQ;
1549 1.23 tls }
1550 1.1 jonathan
1551 1.23 tls /* drop before calling any callbacks. */
1552 1.26 ad if (crp == NULL && krp == NULL) {
1553 1.46 pgoyette
1554 1.46 pgoyette /* Check for the exit condition. */
1555 1.46 pgoyette if (crypto_exit_flag != 0) {
1556 1.46 pgoyette
1557 1.46 pgoyette /* Time to die. */
1558 1.46 pgoyette crypto_exit_flag = 0;
1559 1.46 pgoyette cv_broadcast(&cryptoret_cv);
1560 1.46 pgoyette mutex_spin_exit(&crypto_ret_q_mtx);
1561 1.46 pgoyette kthread_exit(0);
1562 1.46 pgoyette }
1563 1.46 pgoyette
1564 1.26 ad cryptostats.cs_rets++;
1565 1.40 drochner cv_wait(&cryptoret_cv, &crypto_ret_q_mtx);
1566 1.26 ad continue;
1567 1.26 ad }
1568 1.26 ad
1569 1.40 drochner mutex_spin_exit(&crypto_ret_q_mtx);
1570 1.26 ad
1571 1.26 ad if (crp != NULL) {
1572 1.1 jonathan #ifdef CRYPTO_TIMING
1573 1.26 ad if (crypto_timing) {
1574 1.26 ad /*
1575 1.26 ad * NB: We must copy the timestamp before
1576 1.26 ad * doing the callback as the cryptop is
1577 1.26 ad * likely to be reclaimed.
1578 1.26 ad */
1579 1.26 ad struct timespec t = crp->crp_tstamp;
1580 1.26 ad crypto_tstat(&cryptostats.cs_cb, &t);
1581 1.26 ad crp->crp_callback(crp);
1582 1.26 ad crypto_tstat(&cryptostats.cs_finis, &t);
1583 1.26 ad } else
1584 1.1 jonathan #endif
1585 1.26 ad {
1586 1.26 ad crp->crp_callback(crp);
1587 1.1 jonathan }
1588 1.1 jonathan }
1589 1.26 ad if (krp != NULL)
1590 1.26 ad krp->krp_callback(krp);
1591 1.26 ad
1592 1.40 drochner mutex_spin_enter(&crypto_ret_q_mtx);
1593 1.1 jonathan }
1594 1.1 jonathan }
1595 1.42 pgoyette
1596 1.42 pgoyette /* NetBSD module interface */
1597 1.42 pgoyette
1598 1.42 pgoyette MODULE(MODULE_CLASS_MISC, opencrypto, NULL);
1599 1.42 pgoyette
1600 1.42 pgoyette static int
1601 1.42 pgoyette opencrypto_modcmd(modcmd_t cmd, void *opaque)
1602 1.42 pgoyette {
1603 1.46 pgoyette int error = 0;
1604 1.42 pgoyette
1605 1.42 pgoyette switch (cmd) {
1606 1.42 pgoyette case MODULE_CMD_INIT:
1607 1.43 pgoyette #ifdef _MODULE
1608 1.46 pgoyette error = crypto_init();
1609 1.43 pgoyette #endif
1610 1.46 pgoyette break;
1611 1.42 pgoyette case MODULE_CMD_FINI:
1612 1.43 pgoyette #ifdef _MODULE
1613 1.46 pgoyette error = crypto_destroy(true);
1614 1.43 pgoyette #endif
1615 1.46 pgoyette break;
1616 1.42 pgoyette default:
1617 1.46 pgoyette error = ENOTTY;
1618 1.42 pgoyette }
1619 1.46 pgoyette return error;
1620 1.42 pgoyette }
1621