crypto.c revision 1.10 1 1.10 perry /* $NetBSD: crypto.c,v 1.10 2005/02/26 22:39:52 perry Exp $ */
2 1.1 jonathan /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.5 2003/02/26 00:14:05 sam Exp $ */
3 1.1 jonathan /* $OpenBSD: crypto.c,v 1.41 2002/07/17 23:52:38 art Exp $ */
4 1.1 jonathan
5 1.1 jonathan /*
6 1.1 jonathan * The author of this code is Angelos D. Keromytis (angelos (at) cis.upenn.edu)
7 1.1 jonathan *
8 1.1 jonathan * This code was written by Angelos D. Keromytis in Athens, Greece, in
9 1.1 jonathan * February 2000. Network Security Technologies Inc. (NSTI) kindly
10 1.1 jonathan * supported the development of this code.
11 1.1 jonathan *
12 1.1 jonathan * Copyright (c) 2000, 2001 Angelos D. Keromytis
13 1.1 jonathan *
14 1.1 jonathan * Permission to use, copy, and modify this software with or without fee
15 1.1 jonathan * is hereby granted, provided that this entire notice is included in
16 1.1 jonathan * all source code copies of any software which is or includes a copy or
17 1.1 jonathan * modification of this software.
18 1.1 jonathan *
19 1.1 jonathan * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
20 1.1 jonathan * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
21 1.1 jonathan * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
22 1.1 jonathan * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
23 1.1 jonathan * PURPOSE.
24 1.1 jonathan */
25 1.1 jonathan
26 1.1 jonathan #include <sys/cdefs.h>
27 1.10 perry __KERNEL_RCSID(0, "$NetBSD: crypto.c,v 1.10 2005/02/26 22:39:52 perry Exp $");
28 1.1 jonathan
29 1.1 jonathan /* XXX FIXME: should be defopt'ed */
30 1.1 jonathan #define CRYPTO_TIMING /* enable cryptop timing stuff */
31 1.1 jonathan
32 1.1 jonathan #include <sys/param.h>
33 1.1 jonathan #include <sys/reboot.h>
34 1.1 jonathan #include <sys/systm.h>
35 1.1 jonathan #include <sys/malloc.h>
36 1.1 jonathan #include <sys/proc.h>
37 1.1 jonathan #include <sys/pool.h>
38 1.1 jonathan #include <opencrypto/cryptodev.h>
39 1.1 jonathan #include <opencrypto/cryptosoft.h> /* swcr_init() */
40 1.1 jonathan #include <sys/kthread.h>
41 1.1 jonathan
42 1.1 jonathan #include <opencrypto/xform.h> /* XXX for M_XDATA */
43 1.1 jonathan
44 1.1 jonathan
45 1.1 jonathan #ifdef __NetBSD__
46 1.1 jonathan #define splcrypto splnet
47 1.1 jonathan /* below is kludges to check whats still missing */
48 1.1 jonathan #define SWI_CRYPTO 17
49 1.1 jonathan #define register_swi(lvl, fn) \
50 1.1 jonathan softintr_establish(IPL_SOFTNET, (void (*)(void*))fn, NULL)
51 1.1 jonathan #define unregister_swi(lvl, fn) softintr_disestablish(softintr_cookie)
52 1.1 jonathan #define setsoftcrypto(x) softintr_schedule(x)
53 1.4 lha
54 1.4 lha static void nanouptime(struct timespec *);
55 1.4 lha static void
56 1.4 lha nanouptime(struct timespec *tp)
57 1.4 lha {
58 1.4 lha struct timeval tv;
59 1.4 lha microtime(&tv);
60 1.4 lha TIMEVAL_TO_TIMESPEC(&tv, tp);
61 1.4 lha }
62 1.4 lha
63 1.1 jonathan #endif
64 1.1 jonathan
65 1.1 jonathan #define SESID2HID(sid) (((sid) >> 32) & 0xffffffff)
66 1.1 jonathan
67 1.1 jonathan /*
68 1.1 jonathan * Crypto drivers register themselves by allocating a slot in the
69 1.1 jonathan * crypto_drivers table with crypto_get_driverid() and then registering
70 1.1 jonathan * each algorithm they support with crypto_register() and crypto_kregister().
71 1.1 jonathan */
72 1.1 jonathan static struct cryptocap *crypto_drivers = NULL;
73 1.1 jonathan static int crypto_drivers_num = 0;
74 1.1 jonathan static void* softintr_cookie;
75 1.1 jonathan
76 1.1 jonathan /*
77 1.1 jonathan * There are two queues for crypto requests; one for symmetric (e.g.
78 1.1 jonathan * cipher) operations and one for asymmetric (e.g. MOD) operations.
79 1.1 jonathan * See below for how synchronization is handled.
80 1.1 jonathan */
81 1.1 jonathan static TAILQ_HEAD(,cryptop) crp_q; /* request queues */
82 1.1 jonathan static TAILQ_HEAD(,cryptkop) crp_kq;
83 1.1 jonathan
84 1.1 jonathan /*
85 1.1 jonathan * There are two queues for processing completed crypto requests; one
86 1.1 jonathan * for the symmetric and one for the asymmetric ops. We only need one
87 1.1 jonathan * but have two to avoid type futzing (cryptop vs. cryptkop). See below
88 1.1 jonathan * for how synchronization is handled.
89 1.1 jonathan */
90 1.1 jonathan static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */
91 1.1 jonathan static TAILQ_HEAD(,cryptkop) crp_ret_kq;
92 1.1 jonathan
93 1.1 jonathan /*
94 1.1 jonathan * Crypto op and desciptor data structures are allocated
95 1.1 jonathan * from separate private zones(FreeBSD)/pools(netBSD/OpenBSD) .
96 1.1 jonathan */
97 1.1 jonathan struct pool cryptop_pool;
98 1.1 jonathan struct pool cryptodesc_pool;
99 1.1 jonathan int crypto_pool_initialized = 0;
100 1.1 jonathan
101 1.1 jonathan #ifdef __NetBSD__
102 1.8 jonathan void cryptoattach(int);
103 1.1 jonathan static void deferred_crypto_thread(void *arg);
104 1.1 jonathan #endif
105 1.1 jonathan
106 1.1 jonathan int crypto_usercrypto = 1; /* userland may open /dev/crypto */
107 1.1 jonathan int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */
108 1.10 perry /*
109 1.6 jonathan * cryptodevallowsoft is (intended to be) sysctl'able, controlling
110 1.6 jonathan * access to hardware versus software transforms as below:
111 1.6 jonathan *
112 1.6 jonathan * crypto_devallowsoft < 0: Force userlevel requests to use software
113 1.6 jonathan * transforms, always
114 1.6 jonathan * crypto_devallowsoft = 0: Use hardware if present, grant userlevel
115 1.6 jonathan * requests for non-accelerated transforms
116 1.6 jonathan * (handling the latter in software)
117 1.6 jonathan * crypto_devallowsoft > 0: Allow user requests only for transforms which
118 1.6 jonathan * are hardware-accelerated.
119 1.6 jonathan */
120 1.9 jonathan int crypto_devallowsoft = 1; /* only use hardware crypto */
121 1.6 jonathan
122 1.1 jonathan #ifdef __FreeBSD__
123 1.1 jonathan SYSCTL_INT(_kern, OID_AUTO, usercrypto, CTLFLAG_RW,
124 1.1 jonathan &crypto_usercrypto, 0,
125 1.1 jonathan "Enable/disable user-mode access to crypto support");
126 1.1 jonathan SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
127 1.1 jonathan &crypto_userasymcrypto, 0,
128 1.1 jonathan "Enable/disable user-mode access to asymmetric crypto support");
129 1.1 jonathan SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
130 1.1 jonathan &crypto_devallowsoft, 0,
131 1.1 jonathan "Enable/disable use of software asym crypto support");
132 1.1 jonathan #endif
133 1.1 jonathan
134 1.1 jonathan MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
135 1.1 jonathan
136 1.1 jonathan /*
137 1.1 jonathan * Synchronization: read carefully, this is non-trivial.
138 1.1 jonathan *
139 1.1 jonathan * Crypto requests are submitted via crypto_dispatch. Typically
140 1.1 jonathan * these come in from network protocols at spl0 (output path) or
141 1.1 jonathan * spl[,soft]net (input path).
142 1.1 jonathan *
143 1.1 jonathan * Requests are typically passed on the driver directly, but they
144 1.1 jonathan * may also be queued for processing by a software interrupt thread,
145 1.10 perry * cryptointr, that runs at splsoftcrypto. This thread dispatches
146 1.1 jonathan * the requests to crypto drivers (h/w or s/w) who call crypto_done
147 1.1 jonathan * when a request is complete. Hardware crypto drivers are assumed
148 1.1 jonathan * to register their IRQ's as network devices so their interrupt handlers
149 1.1 jonathan * and subsequent "done callbacks" happen at spl[imp,net].
150 1.1 jonathan *
151 1.1 jonathan * Completed crypto ops are queued for a separate kernel thread that
152 1.1 jonathan * handles the callbacks at spl0. This decoupling insures the crypto
153 1.1 jonathan * driver interrupt service routine is not delayed while the callback
154 1.1 jonathan * takes place and that callbacks are delivered after a context switch
155 1.1 jonathan * (as opposed to a software interrupt that clients must block).
156 1.1 jonathan *
157 1.1 jonathan * This scheme is not intended for SMP machines.
158 1.10 perry */
159 1.1 jonathan static void cryptointr(void); /* swi thread to dispatch ops */
160 1.1 jonathan static void cryptoret(void); /* kernel thread for callbacks*/
161 1.1 jonathan static struct proc *cryptoproc;
162 1.1 jonathan static void crypto_destroy(void);
163 1.1 jonathan static int crypto_invoke(struct cryptop *crp, int hint);
164 1.1 jonathan static int crypto_kinvoke(struct cryptkop *krp, int hint);
165 1.1 jonathan
166 1.1 jonathan static struct cryptostats cryptostats;
167 1.1 jonathan static int crypto_timing = 0;
168 1.1 jonathan
169 1.1 jonathan #ifdef __FreeBSD__
170 1.1 jonathan SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
171 1.1 jonathan cryptostats, "Crypto system statistics");
172 1.1 jonathan
173 1.1 jonathan SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
174 1.1 jonathan &crypto_timing, 0, "Enable/disable crypto timing support");
175 1.1 jonathan SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
176 1.1 jonathan cryptostats, "Crypto system statistics");
177 1.4 lha #endif /* __FreeBSD__ */
178 1.1 jonathan
179 1.2 jonathan int
180 1.1 jonathan crypto_init(void)
181 1.1 jonathan {
182 1.1 jonathan int error;
183 1.1 jonathan
184 1.1 jonathan #ifdef __FreeBSD__
185 1.1 jonathan
186 1.1 jonathan cryptop_zone = zinit("cryptop", sizeof (struct cryptop), 0, 0, 1);
187 1.1 jonathan cryptodesc_zone = zinit("cryptodesc", sizeof (struct cryptodesc),
188 1.1 jonathan 0, 0, 1);
189 1.1 jonathan if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
190 1.1 jonathan printf("crypto_init: cannot setup crypto zones\n");
191 1.1 jonathan return ENOMEM;
192 1.1 jonathan }
193 1.1 jonathan #endif
194 1.1 jonathan
195 1.1 jonathan crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
196 1.1 jonathan crypto_drivers = malloc(crypto_drivers_num *
197 1.1 jonathan sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
198 1.1 jonathan if (crypto_drivers == NULL) {
199 1.1 jonathan printf("crypto_init: cannot malloc driver table\n");
200 1.1 jonathan return ENOMEM;
201 1.1 jonathan }
202 1.1 jonathan
203 1.1 jonathan TAILQ_INIT(&crp_q);
204 1.1 jonathan TAILQ_INIT(&crp_kq);
205 1.1 jonathan
206 1.1 jonathan TAILQ_INIT(&crp_ret_q);
207 1.1 jonathan TAILQ_INIT(&crp_ret_kq);
208 1.1 jonathan
209 1.1 jonathan softintr_cookie = register_swi(SWI_CRYPTO, cryptointr);
210 1.1 jonathan #ifdef __FreeBSD__
211 1.1 jonathan error = kthread_create((void (*)(void *)) cryptoret, NULL,
212 1.1 jonathan &cryptoproc, "cryptoret");
213 1.1 jonathan if (error) {
214 1.1 jonathan printf("crypto_init: cannot start cryptoret thread; error %d",
215 1.1 jonathan error);
216 1.1 jonathan crypto_destroy();
217 1.1 jonathan }
218 1.1 jonathan #else
219 1.1 jonathan /* defer thread creation until after boot */
220 1.1 jonathan kthread_create( deferred_crypto_thread, NULL);
221 1.5 scw error = 0;
222 1.1 jonathan #endif
223 1.1 jonathan return error;
224 1.1 jonathan }
225 1.1 jonathan
226 1.1 jonathan static void
227 1.1 jonathan crypto_destroy(void)
228 1.1 jonathan {
229 1.1 jonathan /* XXX no wait to reclaim zones */
230 1.1 jonathan if (crypto_drivers != NULL)
231 1.1 jonathan free(crypto_drivers, M_CRYPTO_DATA);
232 1.1 jonathan unregister_swi(SWI_CRYPTO, cryptointr);
233 1.1 jonathan }
234 1.1 jonathan
235 1.1 jonathan /*
236 1.1 jonathan * Create a new session.
237 1.1 jonathan */
238 1.1 jonathan int
239 1.1 jonathan crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
240 1.1 jonathan {
241 1.1 jonathan struct cryptoini *cr;
242 1.1 jonathan u_int32_t hid, lid;
243 1.1 jonathan int err = EINVAL;
244 1.1 jonathan int s;
245 1.1 jonathan
246 1.1 jonathan s = splcrypto();
247 1.1 jonathan
248 1.1 jonathan if (crypto_drivers == NULL)
249 1.1 jonathan goto done;
250 1.1 jonathan
251 1.1 jonathan /*
252 1.1 jonathan * The algorithm we use here is pretty stupid; just use the
253 1.1 jonathan * first driver that supports all the algorithms we need.
254 1.1 jonathan *
255 1.1 jonathan * XXX We need more smarts here (in real life too, but that's
256 1.1 jonathan * XXX another story altogether).
257 1.1 jonathan */
258 1.1 jonathan
259 1.1 jonathan for (hid = 0; hid < crypto_drivers_num; hid++) {
260 1.1 jonathan /*
261 1.1 jonathan * If it's not initialized or has remaining sessions
262 1.1 jonathan * referencing it, skip.
263 1.1 jonathan */
264 1.1 jonathan if (crypto_drivers[hid].cc_newsession == NULL ||
265 1.1 jonathan (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP))
266 1.1 jonathan continue;
267 1.1 jonathan
268 1.1 jonathan /* Hardware required -- ignore software drivers. */
269 1.1 jonathan if (hard > 0 &&
270 1.1 jonathan (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE))
271 1.1 jonathan continue;
272 1.1 jonathan /* Software required -- ignore hardware drivers. */
273 1.1 jonathan if (hard < 0 &&
274 1.1 jonathan (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0)
275 1.1 jonathan continue;
276 1.1 jonathan
277 1.1 jonathan /* See if all the algorithms are supported. */
278 1.1 jonathan for (cr = cri; cr; cr = cr->cri_next)
279 1.1 jonathan if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0)
280 1.1 jonathan break;
281 1.1 jonathan
282 1.1 jonathan if (cr == NULL) {
283 1.1 jonathan /* Ok, all algorithms are supported. */
284 1.1 jonathan
285 1.1 jonathan /*
286 1.1 jonathan * Can't do everything in one session.
287 1.1 jonathan *
288 1.1 jonathan * XXX Fix this. We need to inject a "virtual" session layer right
289 1.1 jonathan * XXX about here.
290 1.1 jonathan */
291 1.1 jonathan
292 1.1 jonathan /* Call the driver initialization routine. */
293 1.1 jonathan lid = hid; /* Pass the driver ID. */
294 1.1 jonathan err = crypto_drivers[hid].cc_newsession(
295 1.1 jonathan crypto_drivers[hid].cc_arg, &lid, cri);
296 1.1 jonathan if (err == 0) {
297 1.1 jonathan (*sid) = hid;
298 1.1 jonathan (*sid) <<= 32;
299 1.1 jonathan (*sid) |= (lid & 0xffffffff);
300 1.1 jonathan crypto_drivers[hid].cc_sessions++;
301 1.1 jonathan }
302 1.1 jonathan goto done;
303 1.1 jonathan /*break;*/
304 1.1 jonathan }
305 1.1 jonathan }
306 1.1 jonathan done:
307 1.1 jonathan splx(s);
308 1.1 jonathan return err;
309 1.1 jonathan }
310 1.1 jonathan
311 1.1 jonathan /*
312 1.1 jonathan * Delete an existing session (or a reserved session on an unregistered
313 1.1 jonathan * driver).
314 1.1 jonathan */
315 1.1 jonathan int
316 1.1 jonathan crypto_freesession(u_int64_t sid)
317 1.1 jonathan {
318 1.1 jonathan u_int32_t hid;
319 1.1 jonathan int err = 0;
320 1.1 jonathan int s;
321 1.1 jonathan
322 1.1 jonathan s = splcrypto();
323 1.1 jonathan
324 1.1 jonathan if (crypto_drivers == NULL) {
325 1.1 jonathan err = EINVAL;
326 1.1 jonathan goto done;
327 1.1 jonathan }
328 1.1 jonathan
329 1.1 jonathan /* Determine two IDs. */
330 1.1 jonathan hid = SESID2HID(sid);
331 1.1 jonathan
332 1.1 jonathan if (hid >= crypto_drivers_num) {
333 1.1 jonathan err = ENOENT;
334 1.1 jonathan goto done;
335 1.1 jonathan }
336 1.1 jonathan
337 1.1 jonathan if (crypto_drivers[hid].cc_sessions)
338 1.1 jonathan crypto_drivers[hid].cc_sessions--;
339 1.1 jonathan
340 1.1 jonathan /* Call the driver cleanup routine, if available. */
341 1.1 jonathan if (crypto_drivers[hid].cc_freesession)
342 1.1 jonathan err = crypto_drivers[hid].cc_freesession(
343 1.1 jonathan crypto_drivers[hid].cc_arg, sid);
344 1.1 jonathan else
345 1.1 jonathan err = 0;
346 1.1 jonathan
347 1.1 jonathan /*
348 1.1 jonathan * If this was the last session of a driver marked as invalid,
349 1.1 jonathan * make the entry available for reuse.
350 1.1 jonathan */
351 1.1 jonathan if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) &&
352 1.1 jonathan crypto_drivers[hid].cc_sessions == 0)
353 1.1 jonathan bzero(&crypto_drivers[hid], sizeof(struct cryptocap));
354 1.1 jonathan
355 1.1 jonathan done:
356 1.1 jonathan splx(s);
357 1.1 jonathan return err;
358 1.1 jonathan }
359 1.1 jonathan
360 1.1 jonathan /*
361 1.1 jonathan * Return an unused driver id. Used by drivers prior to registering
362 1.1 jonathan * support for the algorithms they handle.
363 1.1 jonathan */
364 1.1 jonathan int32_t
365 1.1 jonathan crypto_get_driverid(u_int32_t flags)
366 1.1 jonathan {
367 1.1 jonathan struct cryptocap *newdrv;
368 1.1 jonathan int i, s;
369 1.1 jonathan
370 1.1 jonathan s = splcrypto();
371 1.1 jonathan for (i = 0; i < crypto_drivers_num; i++)
372 1.1 jonathan if (crypto_drivers[i].cc_process == NULL &&
373 1.1 jonathan (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
374 1.1 jonathan crypto_drivers[i].cc_sessions == 0)
375 1.1 jonathan break;
376 1.1 jonathan
377 1.1 jonathan /* Out of entries, allocate some more. */
378 1.1 jonathan if (i == crypto_drivers_num) {
379 1.1 jonathan /* Be careful about wrap-around. */
380 1.1 jonathan if (2 * crypto_drivers_num <= crypto_drivers_num) {
381 1.1 jonathan splx(s);
382 1.1 jonathan printf("crypto: driver count wraparound!\n");
383 1.1 jonathan return -1;
384 1.1 jonathan }
385 1.1 jonathan
386 1.1 jonathan newdrv = malloc(2 * crypto_drivers_num *
387 1.1 jonathan sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
388 1.1 jonathan if (newdrv == NULL) {
389 1.1 jonathan splx(s);
390 1.1 jonathan printf("crypto: no space to expand driver table!\n");
391 1.1 jonathan return -1;
392 1.1 jonathan }
393 1.1 jonathan
394 1.1 jonathan bcopy(crypto_drivers, newdrv,
395 1.1 jonathan crypto_drivers_num * sizeof(struct cryptocap));
396 1.1 jonathan
397 1.1 jonathan crypto_drivers_num *= 2;
398 1.1 jonathan
399 1.1 jonathan free(crypto_drivers, M_CRYPTO_DATA);
400 1.1 jonathan crypto_drivers = newdrv;
401 1.1 jonathan }
402 1.1 jonathan
403 1.1 jonathan /* NB: state is zero'd on free */
404 1.1 jonathan crypto_drivers[i].cc_sessions = 1; /* Mark */
405 1.1 jonathan crypto_drivers[i].cc_flags = flags;
406 1.1 jonathan
407 1.1 jonathan if (bootverbose)
408 1.1 jonathan printf("crypto: assign driver %u, flags %u\n", i, flags);
409 1.1 jonathan
410 1.1 jonathan splx(s);
411 1.1 jonathan
412 1.1 jonathan return i;
413 1.1 jonathan }
414 1.1 jonathan
415 1.1 jonathan static struct cryptocap *
416 1.1 jonathan crypto_checkdriver(u_int32_t hid)
417 1.1 jonathan {
418 1.1 jonathan if (crypto_drivers == NULL)
419 1.1 jonathan return NULL;
420 1.1 jonathan return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
421 1.1 jonathan }
422 1.1 jonathan
423 1.1 jonathan /*
424 1.1 jonathan * Register support for a key-related algorithm. This routine
425 1.1 jonathan * is called once for each algorithm supported a driver.
426 1.1 jonathan */
427 1.1 jonathan int
428 1.1 jonathan crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags,
429 1.1 jonathan int (*kprocess)(void*, struct cryptkop *, int),
430 1.1 jonathan void *karg)
431 1.1 jonathan {
432 1.1 jonathan int s;
433 1.1 jonathan struct cryptocap *cap;
434 1.1 jonathan int err;
435 1.1 jonathan
436 1.1 jonathan s = splcrypto();
437 1.1 jonathan
438 1.1 jonathan cap = crypto_checkdriver(driverid);
439 1.1 jonathan if (cap != NULL &&
440 1.1 jonathan (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
441 1.1 jonathan /*
442 1.1 jonathan * XXX Do some performance testing to determine placing.
443 1.1 jonathan * XXX We probably need an auxiliary data structure that
444 1.1 jonathan * XXX describes relative performances.
445 1.1 jonathan */
446 1.1 jonathan
447 1.1 jonathan cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
448 1.1 jonathan if (bootverbose)
449 1.1 jonathan printf("crypto: driver %u registers key alg %u flags %u\n"
450 1.1 jonathan , driverid
451 1.1 jonathan , kalg
452 1.1 jonathan , flags
453 1.1 jonathan );
454 1.1 jonathan
455 1.1 jonathan if (cap->cc_kprocess == NULL) {
456 1.1 jonathan cap->cc_karg = karg;
457 1.1 jonathan cap->cc_kprocess = kprocess;
458 1.1 jonathan }
459 1.1 jonathan err = 0;
460 1.1 jonathan } else
461 1.1 jonathan err = EINVAL;
462 1.1 jonathan
463 1.1 jonathan splx(s);
464 1.1 jonathan return err;
465 1.1 jonathan }
466 1.1 jonathan
467 1.1 jonathan /*
468 1.1 jonathan * Register support for a non-key-related algorithm. This routine
469 1.1 jonathan * is called once for each such algorithm supported by a driver.
470 1.1 jonathan */
471 1.1 jonathan int
472 1.1 jonathan crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
473 1.1 jonathan u_int32_t flags,
474 1.1 jonathan int (*newses)(void*, u_int32_t*, struct cryptoini*),
475 1.1 jonathan int (*freeses)(void*, u_int64_t),
476 1.1 jonathan int (*process)(void*, struct cryptop *, int),
477 1.1 jonathan void *arg)
478 1.1 jonathan {
479 1.1 jonathan struct cryptocap *cap;
480 1.1 jonathan int s, err;
481 1.1 jonathan
482 1.1 jonathan s = splcrypto();
483 1.1 jonathan
484 1.1 jonathan cap = crypto_checkdriver(driverid);
485 1.1 jonathan /* NB: algorithms are in the range [1..max] */
486 1.1 jonathan if (cap != NULL &&
487 1.1 jonathan (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
488 1.1 jonathan /*
489 1.1 jonathan * XXX Do some performance testing to determine placing.
490 1.1 jonathan * XXX We probably need an auxiliary data structure that
491 1.1 jonathan * XXX describes relative performances.
492 1.1 jonathan */
493 1.1 jonathan
494 1.1 jonathan cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
495 1.1 jonathan cap->cc_max_op_len[alg] = maxoplen;
496 1.1 jonathan if (bootverbose)
497 1.1 jonathan printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n"
498 1.1 jonathan , driverid
499 1.1 jonathan , alg
500 1.1 jonathan , flags
501 1.1 jonathan , maxoplen
502 1.1 jonathan );
503 1.1 jonathan
504 1.1 jonathan if (cap->cc_process == NULL) {
505 1.1 jonathan cap->cc_arg = arg;
506 1.1 jonathan cap->cc_newsession = newses;
507 1.1 jonathan cap->cc_process = process;
508 1.1 jonathan cap->cc_freesession = freeses;
509 1.1 jonathan cap->cc_sessions = 0; /* Unmark */
510 1.1 jonathan }
511 1.1 jonathan err = 0;
512 1.1 jonathan } else
513 1.1 jonathan err = EINVAL;
514 1.1 jonathan
515 1.1 jonathan splx(s);
516 1.1 jonathan return err;
517 1.1 jonathan }
518 1.1 jonathan
519 1.1 jonathan /*
520 1.1 jonathan * Unregister a crypto driver. If there are pending sessions using it,
521 1.1 jonathan * leave enough information around so that subsequent calls using those
522 1.1 jonathan * sessions will correctly detect the driver has been unregistered and
523 1.1 jonathan * reroute requests.
524 1.1 jonathan */
525 1.1 jonathan int
526 1.1 jonathan crypto_unregister(u_int32_t driverid, int alg)
527 1.1 jonathan {
528 1.1 jonathan int i, err, s;
529 1.1 jonathan u_int32_t ses;
530 1.1 jonathan struct cryptocap *cap;
531 1.1 jonathan
532 1.1 jonathan s = splcrypto();
533 1.1 jonathan
534 1.1 jonathan cap = crypto_checkdriver(driverid);
535 1.1 jonathan if (cap != NULL &&
536 1.1 jonathan (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
537 1.1 jonathan cap->cc_alg[alg] != 0) {
538 1.1 jonathan cap->cc_alg[alg] = 0;
539 1.1 jonathan cap->cc_max_op_len[alg] = 0;
540 1.1 jonathan
541 1.1 jonathan /* Was this the last algorithm ? */
542 1.1 jonathan for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
543 1.1 jonathan if (cap->cc_alg[i] != 0)
544 1.1 jonathan break;
545 1.1 jonathan
546 1.1 jonathan if (i == CRYPTO_ALGORITHM_MAX + 1) {
547 1.1 jonathan ses = cap->cc_sessions;
548 1.1 jonathan bzero(cap, sizeof(struct cryptocap));
549 1.1 jonathan if (ses != 0) {
550 1.1 jonathan /*
551 1.1 jonathan * If there are pending sessions, just mark as invalid.
552 1.1 jonathan */
553 1.1 jonathan cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
554 1.1 jonathan cap->cc_sessions = ses;
555 1.1 jonathan }
556 1.1 jonathan }
557 1.1 jonathan err = 0;
558 1.1 jonathan } else
559 1.1 jonathan err = EINVAL;
560 1.1 jonathan
561 1.1 jonathan splx(s);
562 1.1 jonathan return err;
563 1.1 jonathan }
564 1.1 jonathan
565 1.1 jonathan /*
566 1.1 jonathan * Unregister all algorithms associated with a crypto driver.
567 1.1 jonathan * If there are pending sessions using it, leave enough information
568 1.1 jonathan * around so that subsequent calls using those sessions will
569 1.1 jonathan * correctly detect the driver has been unregistered and reroute
570 1.1 jonathan * requests.
571 1.1 jonathan */
572 1.1 jonathan int
573 1.1 jonathan crypto_unregister_all(u_int32_t driverid)
574 1.1 jonathan {
575 1.1 jonathan int i, err, s = splcrypto();
576 1.1 jonathan u_int32_t ses;
577 1.1 jonathan struct cryptocap *cap;
578 1.1 jonathan
579 1.1 jonathan cap = crypto_checkdriver(driverid);
580 1.1 jonathan if (cap != NULL) {
581 1.1 jonathan for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
582 1.1 jonathan cap->cc_alg[i] = 0;
583 1.1 jonathan cap->cc_max_op_len[i] = 0;
584 1.1 jonathan }
585 1.1 jonathan ses = cap->cc_sessions;
586 1.1 jonathan bzero(cap, sizeof(struct cryptocap));
587 1.1 jonathan if (ses != 0) {
588 1.1 jonathan /*
589 1.1 jonathan * If there are pending sessions, just mark as invalid.
590 1.1 jonathan */
591 1.1 jonathan cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
592 1.1 jonathan cap->cc_sessions = ses;
593 1.1 jonathan }
594 1.1 jonathan err = 0;
595 1.1 jonathan } else
596 1.1 jonathan err = EINVAL;
597 1.1 jonathan
598 1.1 jonathan splx(s);
599 1.1 jonathan return err;
600 1.1 jonathan }
601 1.1 jonathan
602 1.1 jonathan /*
603 1.1 jonathan * Clear blockage on a driver. The what parameter indicates whether
604 1.1 jonathan * the driver is now ready for cryptop's and/or cryptokop's.
605 1.1 jonathan */
606 1.1 jonathan int
607 1.1 jonathan crypto_unblock(u_int32_t driverid, int what)
608 1.1 jonathan {
609 1.1 jonathan struct cryptocap *cap;
610 1.1 jonathan int needwakeup, err, s;
611 1.1 jonathan
612 1.1 jonathan s = splcrypto();
613 1.1 jonathan cap = crypto_checkdriver(driverid);
614 1.1 jonathan if (cap != NULL) {
615 1.1 jonathan needwakeup = 0;
616 1.1 jonathan if (what & CRYPTO_SYMQ) {
617 1.1 jonathan needwakeup |= cap->cc_qblocked;
618 1.1 jonathan cap->cc_qblocked = 0;
619 1.1 jonathan }
620 1.1 jonathan if (what & CRYPTO_ASYMQ) {
621 1.1 jonathan needwakeup |= cap->cc_kqblocked;
622 1.1 jonathan cap->cc_kqblocked = 0;
623 1.1 jonathan }
624 1.1 jonathan if (needwakeup) {
625 1.1 jonathan setsoftcrypto(softintr_cookie);
626 1.1 jonathan }
627 1.1 jonathan err = 0;
628 1.1 jonathan } else
629 1.1 jonathan err = EINVAL;
630 1.1 jonathan splx(s);
631 1.1 jonathan
632 1.1 jonathan return err;
633 1.1 jonathan }
634 1.1 jonathan
635 1.1 jonathan /*
636 1.1 jonathan * Dispatch a crypto request to a driver or queue
637 1.1 jonathan * it, to be processed by the kernel thread.
638 1.1 jonathan */
639 1.1 jonathan int
640 1.1 jonathan crypto_dispatch(struct cryptop *crp)
641 1.1 jonathan {
642 1.1 jonathan u_int32_t hid = SESID2HID(crp->crp_sid);
643 1.1 jonathan int s, result;
644 1.1 jonathan
645 1.1 jonathan s = splcrypto();
646 1.1 jonathan
647 1.1 jonathan cryptostats.cs_ops++;
648 1.1 jonathan
649 1.1 jonathan #ifdef CRYPTO_TIMING
650 1.1 jonathan if (crypto_timing)
651 1.1 jonathan nanouptime(&crp->crp_tstamp);
652 1.1 jonathan #endif
653 1.1 jonathan if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
654 1.1 jonathan struct cryptocap *cap;
655 1.1 jonathan /*
656 1.1 jonathan * Caller marked the request to be processed
657 1.1 jonathan * immediately; dispatch it directly to the
658 1.1 jonathan * driver unless the driver is currently blocked.
659 1.1 jonathan */
660 1.1 jonathan cap = crypto_checkdriver(hid);
661 1.1 jonathan if (cap && !cap->cc_qblocked) {
662 1.1 jonathan result = crypto_invoke(crp, 0);
663 1.1 jonathan if (result == ERESTART) {
664 1.1 jonathan /*
665 1.1 jonathan * The driver ran out of resources, mark the
666 1.1 jonathan * driver ``blocked'' for cryptop's and put
667 1.1 jonathan * the op on the queue.
668 1.1 jonathan */
669 1.1 jonathan crypto_drivers[hid].cc_qblocked = 1;
670 1.1 jonathan TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
671 1.1 jonathan cryptostats.cs_blocks++;
672 1.1 jonathan }
673 1.1 jonathan } else {
674 1.1 jonathan /*
675 1.1 jonathan * The driver is blocked, just queue the op until
676 1.1 jonathan * it unblocks and the swi thread gets kicked.
677 1.1 jonathan */
678 1.1 jonathan TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
679 1.1 jonathan result = 0;
680 1.1 jonathan }
681 1.1 jonathan } else {
682 1.1 jonathan int wasempty = TAILQ_EMPTY(&crp_q);
683 1.1 jonathan /*
684 1.1 jonathan * Caller marked the request as ``ok to delay'';
685 1.1 jonathan * queue it for the swi thread. This is desirable
686 1.1 jonathan * when the operation is low priority and/or suitable
687 1.1 jonathan * for batching.
688 1.1 jonathan */
689 1.1 jonathan TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
690 1.1 jonathan if (wasempty) {
691 1.1 jonathan setsoftcrypto(softintr_cookie);
692 1.1 jonathan }
693 1.1 jonathan
694 1.1 jonathan result = 0;
695 1.1 jonathan }
696 1.1 jonathan splx(s);
697 1.1 jonathan
698 1.1 jonathan return result;
699 1.1 jonathan }
700 1.1 jonathan
701 1.1 jonathan /*
702 1.1 jonathan * Add an asymetric crypto request to a queue,
703 1.1 jonathan * to be processed by the kernel thread.
704 1.1 jonathan */
705 1.1 jonathan int
706 1.1 jonathan crypto_kdispatch(struct cryptkop *krp)
707 1.1 jonathan {
708 1.1 jonathan struct cryptocap *cap;
709 1.1 jonathan int s, result;
710 1.1 jonathan
711 1.1 jonathan s = splcrypto();
712 1.1 jonathan cryptostats.cs_kops++;
713 1.1 jonathan
714 1.1 jonathan cap = crypto_checkdriver(krp->krp_hid);
715 1.1 jonathan if (cap && !cap->cc_kqblocked) {
716 1.1 jonathan result = crypto_kinvoke(krp, 0);
717 1.1 jonathan if (result == ERESTART) {
718 1.1 jonathan /*
719 1.1 jonathan * The driver ran out of resources, mark the
720 1.1 jonathan * driver ``blocked'' for cryptop's and put
721 1.1 jonathan * the op on the queue.
722 1.1 jonathan */
723 1.1 jonathan crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
724 1.1 jonathan TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
725 1.1 jonathan cryptostats.cs_kblocks++;
726 1.1 jonathan }
727 1.1 jonathan } else {
728 1.1 jonathan /*
729 1.1 jonathan * The driver is blocked, just queue the op until
730 1.1 jonathan * it unblocks and the swi thread gets kicked.
731 1.1 jonathan */
732 1.1 jonathan TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
733 1.1 jonathan result = 0;
734 1.1 jonathan }
735 1.1 jonathan splx(s);
736 1.1 jonathan
737 1.1 jonathan return result;
738 1.1 jonathan }
739 1.1 jonathan
740 1.1 jonathan /*
741 1.1 jonathan * Dispatch an assymetric crypto request to the appropriate crypto devices.
742 1.1 jonathan */
743 1.1 jonathan static int
744 1.1 jonathan crypto_kinvoke(struct cryptkop *krp, int hint)
745 1.1 jonathan {
746 1.1 jonathan u_int32_t hid;
747 1.1 jonathan int error;
748 1.1 jonathan
749 1.1 jonathan /* Sanity checks. */
750 1.1 jonathan if (krp == NULL)
751 1.1 jonathan return EINVAL;
752 1.1 jonathan if (krp->krp_callback == NULL) {
753 1.1 jonathan free(krp, M_XDATA); /* XXX allocated in cryptodev */
754 1.1 jonathan return EINVAL;
755 1.1 jonathan }
756 1.1 jonathan
757 1.1 jonathan for (hid = 0; hid < crypto_drivers_num; hid++) {
758 1.1 jonathan if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
759 1.1 jonathan crypto_devallowsoft == 0)
760 1.1 jonathan continue;
761 1.1 jonathan if (crypto_drivers[hid].cc_kprocess == NULL)
762 1.1 jonathan continue;
763 1.1 jonathan if ((crypto_drivers[hid].cc_kalg[krp->krp_op] &
764 1.1 jonathan CRYPTO_ALG_FLAG_SUPPORTED) == 0)
765 1.1 jonathan continue;
766 1.1 jonathan break;
767 1.1 jonathan }
768 1.1 jonathan if (hid < crypto_drivers_num) {
769 1.1 jonathan krp->krp_hid = hid;
770 1.1 jonathan error = crypto_drivers[hid].cc_kprocess(
771 1.1 jonathan crypto_drivers[hid].cc_karg, krp, hint);
772 1.1 jonathan } else {
773 1.1 jonathan error = ENODEV;
774 1.1 jonathan }
775 1.1 jonathan
776 1.1 jonathan if (error) {
777 1.1 jonathan krp->krp_status = error;
778 1.1 jonathan crypto_kdone(krp);
779 1.1 jonathan }
780 1.1 jonathan return 0;
781 1.1 jonathan }
782 1.1 jonathan
783 1.1 jonathan #ifdef CRYPTO_TIMING
784 1.1 jonathan static void
785 1.1 jonathan crypto_tstat(struct cryptotstat *ts, struct timespec *tv)
786 1.1 jonathan {
787 1.1 jonathan struct timespec now, t;
788 1.1 jonathan
789 1.1 jonathan nanouptime(&now);
790 1.1 jonathan t.tv_sec = now.tv_sec - tv->tv_sec;
791 1.1 jonathan t.tv_nsec = now.tv_nsec - tv->tv_nsec;
792 1.1 jonathan if (t.tv_nsec < 0) {
793 1.1 jonathan t.tv_sec--;
794 1.1 jonathan t.tv_nsec += 1000000000;
795 1.1 jonathan }
796 1.1 jonathan timespecadd(&ts->acc, &t, &t);
797 1.1 jonathan if (timespeccmp(&t, &ts->min, <))
798 1.1 jonathan ts->min = t;
799 1.1 jonathan if (timespeccmp(&t, &ts->max, >))
800 1.1 jonathan ts->max = t;
801 1.1 jonathan ts->count++;
802 1.1 jonathan
803 1.1 jonathan *tv = now;
804 1.1 jonathan }
805 1.1 jonathan #endif
806 1.1 jonathan
807 1.1 jonathan /*
808 1.1 jonathan * Dispatch a crypto request to the appropriate crypto devices.
809 1.1 jonathan */
810 1.1 jonathan static int
811 1.1 jonathan crypto_invoke(struct cryptop *crp, int hint)
812 1.1 jonathan {
813 1.1 jonathan u_int32_t hid;
814 1.1 jonathan int (*process)(void*, struct cryptop *, int);
815 1.1 jonathan
816 1.1 jonathan #ifdef CRYPTO_TIMING
817 1.1 jonathan if (crypto_timing)
818 1.1 jonathan crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
819 1.1 jonathan #endif
820 1.1 jonathan /* Sanity checks. */
821 1.1 jonathan if (crp == NULL)
822 1.1 jonathan return EINVAL;
823 1.1 jonathan if (crp->crp_callback == NULL) {
824 1.1 jonathan crypto_freereq(crp);
825 1.1 jonathan return EINVAL;
826 1.1 jonathan }
827 1.1 jonathan if (crp->crp_desc == NULL) {
828 1.1 jonathan crp->crp_etype = EINVAL;
829 1.1 jonathan crypto_done(crp);
830 1.1 jonathan return 0;
831 1.1 jonathan }
832 1.1 jonathan
833 1.1 jonathan hid = SESID2HID(crp->crp_sid);
834 1.1 jonathan if (hid < crypto_drivers_num) {
835 1.1 jonathan if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)
836 1.1 jonathan crypto_freesession(crp->crp_sid);
837 1.1 jonathan process = crypto_drivers[hid].cc_process;
838 1.1 jonathan } else {
839 1.1 jonathan process = NULL;
840 1.1 jonathan }
841 1.1 jonathan
842 1.1 jonathan if (process == NULL) {
843 1.1 jonathan struct cryptodesc *crd;
844 1.1 jonathan u_int64_t nid;
845 1.1 jonathan
846 1.1 jonathan /*
847 1.1 jonathan * Driver has unregistered; migrate the session and return
848 1.1 jonathan * an error to the caller so they'll resubmit the op.
849 1.1 jonathan */
850 1.1 jonathan for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
851 1.1 jonathan crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
852 1.1 jonathan
853 1.1 jonathan if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
854 1.1 jonathan crp->crp_sid = nid;
855 1.1 jonathan
856 1.1 jonathan crp->crp_etype = EAGAIN;
857 1.1 jonathan crypto_done(crp);
858 1.1 jonathan return 0;
859 1.1 jonathan } else {
860 1.1 jonathan /*
861 1.1 jonathan * Invoke the driver to process the request.
862 1.1 jonathan */
863 1.1 jonathan return (*process)(crypto_drivers[hid].cc_arg, crp, hint);
864 1.1 jonathan }
865 1.1 jonathan }
866 1.1 jonathan
867 1.1 jonathan /*
868 1.1 jonathan * Release a set of crypto descriptors.
869 1.1 jonathan */
870 1.1 jonathan void
871 1.1 jonathan crypto_freereq(struct cryptop *crp)
872 1.1 jonathan {
873 1.1 jonathan struct cryptodesc *crd;
874 1.1 jonathan int s;
875 1.1 jonathan
876 1.1 jonathan if (crp == NULL)
877 1.1 jonathan return;
878 1.1 jonathan
879 1.1 jonathan s = splcrypto();
880 1.1 jonathan
881 1.1 jonathan while ((crd = crp->crp_desc) != NULL) {
882 1.1 jonathan crp->crp_desc = crd->crd_next;
883 1.1 jonathan pool_put(&cryptodesc_pool, crd);
884 1.1 jonathan }
885 1.1 jonathan
886 1.1 jonathan pool_put(&cryptop_pool, crp);
887 1.1 jonathan splx(s);
888 1.1 jonathan }
889 1.1 jonathan
890 1.1 jonathan /*
891 1.1 jonathan * Acquire a set of crypto descriptors.
892 1.1 jonathan */
893 1.1 jonathan struct cryptop *
894 1.1 jonathan crypto_getreq(int num)
895 1.1 jonathan {
896 1.1 jonathan struct cryptodesc *crd;
897 1.1 jonathan struct cryptop *crp;
898 1.1 jonathan int s;
899 1.1 jonathan
900 1.1 jonathan s = splcrypto();
901 1.1 jonathan
902 1.1 jonathan if (crypto_pool_initialized == 0) {
903 1.1 jonathan pool_init(&cryptop_pool, sizeof(struct cryptop), 0, 0,
904 1.1 jonathan 0, "cryptop", NULL);
905 1.1 jonathan pool_init(&cryptodesc_pool, sizeof(struct cryptodesc), 0, 0,
906 1.1 jonathan 0, "cryptodesc", NULL);
907 1.1 jonathan crypto_pool_initialized = 1;
908 1.1 jonathan }
909 1.1 jonathan
910 1.1 jonathan crp = pool_get(&cryptop_pool, 0);
911 1.1 jonathan if (crp == NULL) {
912 1.1 jonathan splx(s);
913 1.1 jonathan return NULL;
914 1.1 jonathan }
915 1.1 jonathan bzero(crp, sizeof(struct cryptop));
916 1.1 jonathan
917 1.1 jonathan while (num--) {
918 1.1 jonathan crd = pool_get(&cryptodesc_pool, 0);
919 1.1 jonathan if (crd == NULL) {
920 1.1 jonathan splx(s);
921 1.1 jonathan crypto_freereq(crp);
922 1.1 jonathan return NULL;
923 1.1 jonathan }
924 1.1 jonathan
925 1.1 jonathan bzero(crd, sizeof(struct cryptodesc));
926 1.1 jonathan crd->crd_next = crp->crp_desc;
927 1.1 jonathan crp->crp_desc = crd;
928 1.1 jonathan }
929 1.1 jonathan
930 1.1 jonathan splx(s);
931 1.1 jonathan return crp;
932 1.1 jonathan }
933 1.1 jonathan
934 1.1 jonathan /*
935 1.1 jonathan * Invoke the callback on behalf of the driver.
936 1.1 jonathan */
937 1.1 jonathan void
938 1.1 jonathan crypto_done(struct cryptop *crp)
939 1.1 jonathan {
940 1.1 jonathan if (crp->crp_etype != 0)
941 1.1 jonathan cryptostats.cs_errs++;
942 1.1 jonathan #ifdef CRYPTO_TIMING
943 1.1 jonathan if (crypto_timing)
944 1.1 jonathan crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
945 1.1 jonathan #endif
946 1.1 jonathan /*
947 1.1 jonathan * On netbsd 1.6O, CBIMM does its wake_one() before the requestor
948 1.1 jonathan * has done its tsleep().
949 1.1 jonathan */
950 1.1 jonathan #ifndef __NetBSD__
951 1.1 jonathan if (crp->crp_flags & CRYPTO_F_CBIMM) {
952 1.1 jonathan /*
953 1.1 jonathan * Do the callback directly. This is ok when the
954 1.1 jonathan * callback routine does very little (e.g. the
955 1.1 jonathan * /dev/crypto callback method just does a wakeup).
956 1.1 jonathan */
957 1.1 jonathan #ifdef CRYPTO_TIMING
958 1.1 jonathan if (crypto_timing) {
959 1.1 jonathan /*
960 1.1 jonathan * NB: We must copy the timestamp before
961 1.1 jonathan * doing the callback as the cryptop is
962 1.1 jonathan * likely to be reclaimed.
963 1.1 jonathan */
964 1.1 jonathan struct timespec t = crp->crp_tstamp;
965 1.1 jonathan crypto_tstat(&cryptostats.cs_cb, &t);
966 1.1 jonathan crp->crp_callback(crp);
967 1.1 jonathan crypto_tstat(&cryptostats.cs_finis, &t);
968 1.1 jonathan } else
969 1.1 jonathan #endif
970 1.1 jonathan crp->crp_callback(crp);
971 1.1 jonathan } else
972 1.1 jonathan #endif /* __NetBSD__ */
973 1.1 jonathan {
974 1.1 jonathan int s, wasempty;
975 1.1 jonathan /*
976 1.1 jonathan * Normal case; queue the callback for the thread.
977 1.1 jonathan *
978 1.1 jonathan * The return queue is manipulated by the swi thread
979 1.1 jonathan * and, potentially, by crypto device drivers calling
980 1.1 jonathan * back to mark operations completed. Thus we need
981 1.1 jonathan * to mask both while manipulating the return queue.
982 1.1 jonathan */
983 1.1 jonathan s = splcrypto();
984 1.1 jonathan wasempty = TAILQ_EMPTY(&crp_ret_q);
985 1.1 jonathan TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
986 1.1 jonathan if (wasempty)
987 1.1 jonathan wakeup_one(&crp_ret_q);
988 1.1 jonathan splx(s);
989 1.1 jonathan }
990 1.1 jonathan }
991 1.1 jonathan
992 1.1 jonathan /*
993 1.1 jonathan * Invoke the callback on behalf of the driver.
994 1.1 jonathan */
995 1.1 jonathan void
996 1.1 jonathan crypto_kdone(struct cryptkop *krp)
997 1.1 jonathan {
998 1.1 jonathan int s, wasempty;
999 1.1 jonathan
1000 1.1 jonathan if (krp->krp_status != 0)
1001 1.1 jonathan cryptostats.cs_kerrs++;
1002 1.1 jonathan /*
1003 1.1 jonathan * The return queue is manipulated by the swi thread
1004 1.1 jonathan * and, potentially, by crypto device drivers calling
1005 1.1 jonathan * back to mark operations completed. Thus we need
1006 1.1 jonathan * to mask both while manipulating the return queue.
1007 1.1 jonathan */
1008 1.1 jonathan s = splcrypto();
1009 1.1 jonathan wasempty = TAILQ_EMPTY(&crp_ret_kq);
1010 1.1 jonathan TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
1011 1.1 jonathan if (wasempty)
1012 1.1 jonathan wakeup_one(&crp_ret_q);
1013 1.1 jonathan splx(s);
1014 1.1 jonathan }
1015 1.1 jonathan
1016 1.1 jonathan int
1017 1.1 jonathan crypto_getfeat(int *featp)
1018 1.1 jonathan {
1019 1.1 jonathan int hid, kalg, feat = 0;
1020 1.1 jonathan int s;
1021 1.1 jonathan
1022 1.1 jonathan s = splcrypto();
1023 1.1 jonathan
1024 1.1 jonathan if (crypto_userasymcrypto == 0)
1025 1.10 perry goto out;
1026 1.1 jonathan
1027 1.1 jonathan for (hid = 0; hid < crypto_drivers_num; hid++) {
1028 1.1 jonathan if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1029 1.7 jonathan crypto_devallowsoft == 0) {
1030 1.1 jonathan continue;
1031 1.1 jonathan }
1032 1.1 jonathan if (crypto_drivers[hid].cc_kprocess == NULL)
1033 1.1 jonathan continue;
1034 1.1 jonathan for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1035 1.1 jonathan if ((crypto_drivers[hid].cc_kalg[kalg] &
1036 1.1 jonathan CRYPTO_ALG_FLAG_SUPPORTED) != 0)
1037 1.1 jonathan feat |= 1 << kalg;
1038 1.1 jonathan }
1039 1.1 jonathan out:
1040 1.1 jonathan splx(s);
1041 1.1 jonathan *featp = feat;
1042 1.1 jonathan return (0);
1043 1.1 jonathan }
1044 1.1 jonathan
1045 1.1 jonathan /*
1046 1.1 jonathan * Software interrupt thread to dispatch crypto requests.
1047 1.1 jonathan */
1048 1.1 jonathan static void
1049 1.1 jonathan cryptointr(void)
1050 1.1 jonathan {
1051 1.1 jonathan struct cryptop *crp, *submit;
1052 1.1 jonathan struct cryptkop *krp;
1053 1.1 jonathan struct cryptocap *cap;
1054 1.1 jonathan int result, hint, s;
1055 1.1 jonathan
1056 1.1 jonathan printf("crypto softint\n");
1057 1.1 jonathan cryptostats.cs_intrs++;
1058 1.1 jonathan s = splcrypto();
1059 1.1 jonathan do {
1060 1.1 jonathan /*
1061 1.1 jonathan * Find the first element in the queue that can be
1062 1.1 jonathan * processed and look-ahead to see if multiple ops
1063 1.1 jonathan * are ready for the same driver.
1064 1.1 jonathan */
1065 1.1 jonathan submit = NULL;
1066 1.1 jonathan hint = 0;
1067 1.1 jonathan TAILQ_FOREACH(crp, &crp_q, crp_next) {
1068 1.1 jonathan u_int32_t hid = SESID2HID(crp->crp_sid);
1069 1.1 jonathan cap = crypto_checkdriver(hid);
1070 1.1 jonathan if (cap == NULL || cap->cc_process == NULL) {
1071 1.1 jonathan /* Op needs to be migrated, process it. */
1072 1.1 jonathan if (submit == NULL)
1073 1.1 jonathan submit = crp;
1074 1.1 jonathan break;
1075 1.1 jonathan }
1076 1.1 jonathan if (!cap->cc_qblocked) {
1077 1.1 jonathan if (submit != NULL) {
1078 1.1 jonathan /*
1079 1.1 jonathan * We stop on finding another op,
1080 1.1 jonathan * regardless whether its for the same
1081 1.1 jonathan * driver or not. We could keep
1082 1.1 jonathan * searching the queue but it might be
1083 1.1 jonathan * better to just use a per-driver
1084 1.1 jonathan * queue instead.
1085 1.1 jonathan */
1086 1.1 jonathan if (SESID2HID(submit->crp_sid) == hid)
1087 1.1 jonathan hint = CRYPTO_HINT_MORE;
1088 1.1 jonathan break;
1089 1.1 jonathan } else {
1090 1.1 jonathan submit = crp;
1091 1.1 jonathan if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1092 1.1 jonathan break;
1093 1.1 jonathan /* keep scanning for more are q'd */
1094 1.1 jonathan }
1095 1.1 jonathan }
1096 1.1 jonathan }
1097 1.1 jonathan if (submit != NULL) {
1098 1.1 jonathan TAILQ_REMOVE(&crp_q, submit, crp_next);
1099 1.1 jonathan result = crypto_invoke(submit, hint);
1100 1.1 jonathan if (result == ERESTART) {
1101 1.1 jonathan /*
1102 1.1 jonathan * The driver ran out of resources, mark the
1103 1.1 jonathan * driver ``blocked'' for cryptop's and put
1104 1.1 jonathan * the request back in the queue. It would
1105 1.1 jonathan * best to put the request back where we got
1106 1.1 jonathan * it but that's hard so for now we put it
1107 1.1 jonathan * at the front. This should be ok; putting
1108 1.1 jonathan * it at the end does not work.
1109 1.1 jonathan */
1110 1.1 jonathan /* XXX validate sid again? */
1111 1.1 jonathan crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1112 1.1 jonathan TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1113 1.1 jonathan cryptostats.cs_blocks++;
1114 1.1 jonathan }
1115 1.1 jonathan }
1116 1.1 jonathan
1117 1.1 jonathan /* As above, but for key ops */
1118 1.1 jonathan TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1119 1.1 jonathan cap = crypto_checkdriver(krp->krp_hid);
1120 1.1 jonathan if (cap == NULL || cap->cc_kprocess == NULL) {
1121 1.1 jonathan /* Op needs to be migrated, process it. */
1122 1.1 jonathan break;
1123 1.1 jonathan }
1124 1.1 jonathan if (!cap->cc_kqblocked)
1125 1.1 jonathan break;
1126 1.1 jonathan }
1127 1.1 jonathan if (krp != NULL) {
1128 1.1 jonathan TAILQ_REMOVE(&crp_kq, krp, krp_next);
1129 1.1 jonathan result = crypto_kinvoke(krp, 0);
1130 1.1 jonathan if (result == ERESTART) {
1131 1.1 jonathan /*
1132 1.1 jonathan * The driver ran out of resources, mark the
1133 1.1 jonathan * driver ``blocked'' for cryptkop's and put
1134 1.1 jonathan * the request back in the queue. It would
1135 1.1 jonathan * best to put the request back where we got
1136 1.1 jonathan * it but that's hard so for now we put it
1137 1.1 jonathan * at the front. This should be ok; putting
1138 1.1 jonathan * it at the end does not work.
1139 1.1 jonathan */
1140 1.1 jonathan /* XXX validate sid again? */
1141 1.1 jonathan crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1142 1.1 jonathan TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1143 1.1 jonathan cryptostats.cs_kblocks++;
1144 1.1 jonathan }
1145 1.1 jonathan }
1146 1.1 jonathan } while (submit != NULL || krp != NULL);
1147 1.1 jonathan splx(s);
1148 1.1 jonathan }
1149 1.1 jonathan
1150 1.1 jonathan /*
1151 1.1 jonathan * Kernel thread to do callbacks.
1152 1.1 jonathan */
1153 1.1 jonathan static void
1154 1.1 jonathan cryptoret(void)
1155 1.1 jonathan {
1156 1.1 jonathan struct cryptop *crp;
1157 1.1 jonathan struct cryptkop *krp;
1158 1.1 jonathan int s;
1159 1.1 jonathan
1160 1.1 jonathan s = splcrypto();
1161 1.1 jonathan for (;;) {
1162 1.1 jonathan crp = TAILQ_FIRST(&crp_ret_q);
1163 1.1 jonathan if (crp != NULL)
1164 1.1 jonathan TAILQ_REMOVE(&crp_ret_q, crp, crp_next);
1165 1.1 jonathan krp = TAILQ_FIRST(&crp_ret_kq);
1166 1.1 jonathan if (krp != NULL)
1167 1.1 jonathan TAILQ_REMOVE(&crp_ret_kq, krp, krp_next);
1168 1.1 jonathan
1169 1.1 jonathan if (crp != NULL || krp != NULL) {
1170 1.1 jonathan splx(s); /* lower ipl for callbacks */
1171 1.1 jonathan if (crp != NULL) {
1172 1.1 jonathan #ifdef CRYPTO_TIMING
1173 1.1 jonathan if (crypto_timing) {
1174 1.1 jonathan /*
1175 1.1 jonathan * NB: We must copy the timestamp before
1176 1.1 jonathan * doing the callback as the cryptop is
1177 1.1 jonathan * likely to be reclaimed.
1178 1.1 jonathan */
1179 1.1 jonathan struct timespec t = crp->crp_tstamp;
1180 1.1 jonathan crypto_tstat(&cryptostats.cs_cb, &t);
1181 1.1 jonathan crp->crp_callback(crp);
1182 1.1 jonathan crypto_tstat(&cryptostats.cs_finis, &t);
1183 1.1 jonathan } else
1184 1.1 jonathan #endif
1185 1.1 jonathan crp->crp_callback(crp);
1186 1.1 jonathan }
1187 1.1 jonathan if (krp != NULL)
1188 1.1 jonathan krp->krp_callback(krp);
1189 1.1 jonathan s = splcrypto();
1190 1.1 jonathan } else {
1191 1.1 jonathan (void) tsleep(&crp_ret_q, PLOCK, "crypto_wait", 0);
1192 1.1 jonathan cryptostats.cs_rets++;
1193 1.1 jonathan }
1194 1.1 jonathan }
1195 1.1 jonathan }
1196 1.1 jonathan
1197 1.1 jonathan static void
1199 1.1 jonathan deferred_crypto_thread(void *arg)
1200 1.1 jonathan {
1201 1.1 jonathan int error;
1202 1.1 jonathan
1203 1.1 jonathan error = kthread_create1((void (*)(void*)) cryptoret, NULL,
1204 1.1 jonathan &cryptoproc, "cryptoret");
1205 1.8 jonathan if (error) {
1206 1.8 jonathan printf("crypto_init: cannot start cryptoret thread; error %d",
1207 1.8 jonathan error);
1208 1.1 jonathan crypto_destroy();
1209 1.1 jonathan }
1210 1.8 jonathan
1211 1.8 jonathan /*
1212 1.8 jonathan * XXX in absence of FreeBSD mod_init(), call init hooks here,
1213 1.8 jonathan * now that the thread used by software crypto is up and running.
1214 1.8 jonathan */
1215 1.1 jonathan swcr_init();
1216 1.1 jonathan }
1217 1.1 jonathan
1218 1.8 jonathan void
1219 1.1 jonathan cryptoattach(int n)
1220 1.8 jonathan {
1221 1.1 jonathan /* Nothing to do. */
1222 1.1 jonathan }
1223 1.1 jonathan
1224 1.1 jonathan #ifdef __FreeBSD__
1225 1.1 jonathan /*
1226 1.1 jonathan * Initialization code, both for static and dynamic loading.
1227 1.1 jonathan */
1228 1.1 jonathan static int
1229 1.1 jonathan crypto_modevent(module_t mod, int type, void *unused)
1230 1.1 jonathan {
1231 1.1 jonathan int error = EINVAL;
1232 1.1 jonathan
1233 1.1 jonathan switch (type) {
1234 1.1 jonathan case MOD_LOAD:
1235 1.1 jonathan error = crypto_init();
1236 1.1 jonathan if (error == 0 && bootverbose)
1237 1.1 jonathan printf("crypto: <crypto core>\n");
1238 1.1 jonathan break;
1239 1.1 jonathan case MOD_UNLOAD:
1240 1.1 jonathan /*XXX disallow if active sessions */
1241 1.1 jonathan error = 0;
1242 1.1 jonathan crypto_destroy();
1243 1.1 jonathan break;
1244 1.1 jonathan }
1245 1.1 jonathan return error;
1246 1.1 jonathan }
1247 1.1 jonathan static moduledata_t crypto_mod = {
1248 1.1 jonathan "crypto",
1249 1.1 jonathan crypto_modevent,
1250 1.1 jonathan 0
1251 1.1 jonathan };
1252 1.1 jonathan
1253 1.1 jonathan MODULE_VERSION(crypto, 1);
1254 1.4 lha DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
1255 1.1 jonathan #endif /* __FreeBSD__ */
1256 1.1 jonathan
1257
1258