ubsec.c revision 1.40 1 /* $NetBSD: ubsec.c,v 1.40 2014/04/19 12:29:24 bad Exp $ */
2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */
3 /* $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ */
4
5 /*
6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net)
7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org)
8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com)
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 *
31 * Effort sponsored in part by the Defense Advanced Research Projects
32 * Agency (DARPA) and Air Force Research Laboratory, Air Force
33 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
34 *
35 */
36
37 #include <sys/cdefs.h>
38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.40 2014/04/19 12:29:24 bad Exp $");
39
40 #undef UBSEC_DEBUG
41
42 /*
43 * uBsec 5[56]01, 58xx hardware crypto accelerator
44 */
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/proc.h>
49 #include <sys/endian.h>
50 #ifdef __NetBSD__
51 #define UBSEC_NO_RNG /* hangs on attach */
52 #define letoh16 htole16
53 #define letoh32 htole32
54 #endif
55 #include <sys/errno.h>
56 #include <sys/malloc.h>
57 #include <sys/kernel.h>
58 #include <sys/mbuf.h>
59 #include <sys/device.h>
60 #include <sys/module.h>
61 #include <sys/queue.h>
62 #include <sys/sysctl.h>
63
64 #include <opencrypto/cryptodev.h>
65 #include <opencrypto/xform.h>
66 #ifdef __OpenBSD__
67 #include <dev/rndvar.h>
68 #include <sys/md5k.h>
69 #else
70 #include <sys/cprng.h>
71 #include <sys/md5.h>
72 #endif
73 #include <sys/sha1.h>
74
75 #include <dev/pci/pcireg.h>
76 #include <dev/pci/pcivar.h>
77 #include <dev/pci/pcidevs.h>
78
79 #include <dev/pci/ubsecreg.h>
80 #include <dev/pci/ubsecvar.h>
81
82 /*
83 * Prototypes and count for the pci_device structure
84 */
85 static int ubsec_probe(device_t, cfdata_t, void *);
86 static void ubsec_attach(device_t, device_t, void *);
87 static int ubsec_detach(device_t, int);
88 static int ubsec_sysctl_init(void);
89 static void ubsec_reset_board(struct ubsec_softc *);
90 static void ubsec_init_board(struct ubsec_softc *);
91 static void ubsec_init_pciregs(struct pci_attach_args *pa);
92 static void ubsec_cleanchip(struct ubsec_softc *);
93 static void ubsec_totalreset(struct ubsec_softc *);
94 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *);
95
96 #ifdef __OpenBSD__
97 struct cfattach ubsec_ca = {
98 sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
99 };
100
101 struct cfdriver ubsec_cd = {
102 0, "ubsec", DV_DULL
103 };
104 #else
105 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
106 ubsec_detach, NULL);
107 extern struct cfdriver ubsec_cd;
108 #endif
109
110 /* patchable */
111 #ifdef UBSEC_DEBUG
112 extern int ubsec_debug;
113 int ubsec_debug=1;
114 #endif
115
116 static int ubsec_intr(void *);
117 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *);
118 static int ubsec_freesession(void*, u_int64_t);
119 static int ubsec_process(void*, struct cryptop *, int hint);
120 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
121 static void ubsec_feed(struct ubsec_softc *);
122 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
123 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
124 static void ubsec_feed2(struct ubsec_softc *);
125 static void ubsec_feed4(struct ubsec_softc *);
126 #ifndef UBSEC_NO_RNG
127 static void ubsec_rng(void *);
128 static void ubsec_rng_locked(void *);
129 static void ubsec_rng_get(size_t, void *);
130 #endif /* UBSEC_NO_RNG */
131 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
132 struct ubsec_dma_alloc *, int);
133 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
134 static int ubsec_dmamap_aligned(bus_dmamap_t);
135
136 static int ubsec_kprocess(void*, struct cryptkop *, int);
137 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *,
138 struct cryptkop *, int);
139 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *,
140 struct cryptkop *, int);
141 static int ubsec_kprocess_rsapriv(struct ubsec_softc *,
142 struct cryptkop *, int);
143 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
144 static int ubsec_ksigbits(struct crparam *);
145 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
146 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
147
148 #ifdef UBSEC_DEBUG
149 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
150 static void ubsec_dump_mcr(struct ubsec_mcr *);
151 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *);
152 #endif
153
154 #define READ_REG(sc,r) \
155 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
156
157 #define WRITE_REG(sc,reg,val) \
158 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
159
160 #define SWAP32(x) (x) = htole32(ntohl((x)))
161 #ifndef HTOLE32
162 #define HTOLE32(x) (x) = htole32(x)
163 #endif
164
165 struct ubsec_stats ubsecstats;
166
167 static struct sysctllog *ubsec_sysctllog;
168
169 /*
170 * ubsec_maxbatch controls the number of crypto ops to voluntarily
171 * collect into one submission to the hardware. This batching happens
172 * when ops are dispatched from the crypto subsystem with a hint that
173 * more are to follow immediately. These ops must also not be marked
174 * with a ``no delay'' flag.
175 */
176 static int ubsec_maxbatch = 1;
177
178 /*
179 * ubsec_maxaggr controls the number of crypto ops to submit to the
180 * hardware as a unit. This aggregation reduces the number of interrupts
181 * to the host at the expense of increased latency (for all but the last
182 * operation). For network traffic setting this to one yields the highest
183 * performance but at the expense of more interrupt processing.
184 */
185 static int ubsec_maxaggr = 1;
186
187 static const struct ubsec_product {
188 pci_vendor_id_t ubsec_vendor;
189 pci_product_id_t ubsec_product;
190 int ubsec_flags;
191 int ubsec_statmask;
192 int ubsec_maxaggr;
193 const char *ubsec_name;
194 } ubsec_products[] = {
195 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501,
196 0,
197 BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
198 UBS_MIN_AGGR,
199 "Bluesteel 5501"
200 },
201 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601,
202 UBS_FLAGS_KEY | UBS_FLAGS_RNG,
203 BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
204 UBS_MIN_AGGR,
205 "Bluesteel 5601"
206 },
207
208 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801,
209 0,
210 BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
211 UBS_MIN_AGGR,
212 "Broadcom BCM5801"
213 },
214
215 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802,
216 UBS_FLAGS_KEY | UBS_FLAGS_RNG,
217 BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
218 UBS_MIN_AGGR,
219 "Broadcom BCM5802"
220 },
221
222 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805,
223 UBS_FLAGS_KEY | UBS_FLAGS_RNG,
224 BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
225 UBS_MIN_AGGR,
226 "Broadcom BCM5805"
227 },
228
229 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820,
230 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
231 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
232 BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
233 UBS_MIN_AGGR,
234 "Broadcom BCM5820"
235 },
236
237 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821,
238 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
239 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
240 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
241 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
242 UBS_MIN_AGGR,
243 "Broadcom BCM5821"
244 },
245 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K,
246 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
247 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
248 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
249 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
250 UBS_MIN_AGGR,
251 "Sun Crypto Accelerator 1000"
252 },
253 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821,
254 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
255 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
256 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
257 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
258 UBS_MIN_AGGR,
259 "Broadcom BCM5821 (Sun)"
260 },
261
262 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822,
263 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
264 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
265 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
266 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
267 UBS_MIN_AGGR,
268 "Broadcom BCM5822"
269 },
270
271 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823,
272 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
273 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
274 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
275 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
276 UBS_MIN_AGGR,
277 "Broadcom BCM5823"
278 },
279
280 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825,
281 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
282 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
283 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
284 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
285 UBS_MIN_AGGR,
286 "Broadcom BCM5825"
287 },
288
289 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860,
290 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
291 UBS_FLAGS_LONGCTX |
292 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
293 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
294 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
295 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
296 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
297 UBS_MAX_AGGR,
298 "Broadcom BCM5860"
299 },
300
301 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861,
302 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
303 UBS_FLAGS_LONGCTX |
304 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
305 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
306 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
307 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
308 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
309 UBS_MAX_AGGR,
310 "Broadcom BCM5861"
311 },
312
313 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862,
314 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
315 UBS_FLAGS_LONGCTX |
316 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
317 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
318 BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
319 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
320 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
321 UBS_MAX_AGGR,
322 "Broadcom BCM5862"
323 },
324
325 { 0, 0,
326 0,
327 0,
328 0,
329 NULL
330 }
331 };
332
333 static const struct ubsec_product *
334 ubsec_lookup(const struct pci_attach_args *pa)
335 {
336 const struct ubsec_product *up;
337
338 for (up = ubsec_products; up->ubsec_name != NULL; up++) {
339 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor &&
340 PCI_PRODUCT(pa->pa_id) == up->ubsec_product)
341 return (up);
342 }
343 return (NULL);
344 }
345
346 static int
347 ubsec_probe(device_t parent, cfdata_t match, void *aux)
348 {
349 struct pci_attach_args *pa = (struct pci_attach_args *)aux;
350
351 if (ubsec_lookup(pa) != NULL)
352 return (1);
353
354 return (0);
355 }
356
357 static void
358 ubsec_attach(device_t parent, device_t self, void *aux)
359 {
360 struct ubsec_softc *sc = device_private(self);
361 struct pci_attach_args *pa = aux;
362 const struct ubsec_product *up;
363 pci_chipset_tag_t pc = pa->pa_pc;
364 pci_intr_handle_t ih;
365 const char *intrstr = NULL;
366 pcireg_t memtype;
367 struct ubsec_dma *dmap;
368 u_int32_t cmd, i;
369 char intrbuf[PCI_INTRSTR_LEN];
370
371 sc->sc_dev = self;
372 sc->sc_pct = pc;
373
374 up = ubsec_lookup(pa);
375 if (up == NULL) {
376 printf("\n");
377 panic("ubsec_attach: impossible");
378 }
379
380 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1);
381
382 SIMPLEQ_INIT(&sc->sc_queue);
383 SIMPLEQ_INIT(&sc->sc_qchip);
384 SIMPLEQ_INIT(&sc->sc_queue2);
385 SIMPLEQ_INIT(&sc->sc_qchip2);
386 SIMPLEQ_INIT(&sc->sc_queue4);
387 SIMPLEQ_INIT(&sc->sc_qchip4);
388 SIMPLEQ_INIT(&sc->sc_q2free);
389
390 sc->sc_flags = up->ubsec_flags;
391 sc->sc_statmask = up->ubsec_statmask;
392 sc->sc_maxaggr = up->ubsec_maxaggr;
393
394 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
395 cmd |= PCI_COMMAND_MASTER_ENABLE;
396 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
397
398 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR);
399 if (pci_mapreg_map(pa, BS_BAR, memtype, 0,
400 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) {
401 aprint_error_dev(self, "can't find mem space");
402 return;
403 }
404
405 sc->sc_dmat = pa->pa_dmat;
406
407 if (pci_intr_map(pa, &ih)) {
408 aprint_error_dev(self, "couldn't map interrupt\n");
409 return;
410 }
411 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
412 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc);
413 if (sc->sc_ih == NULL) {
414 aprint_error_dev(self, "couldn't establish interrupt");
415 if (intrstr != NULL)
416 aprint_error(" at %s", intrstr);
417 aprint_error("\n");
418 return;
419 }
420 aprint_normal_dev(self, "interrupting at %s\n", intrstr);
421
422 sc->sc_cid = crypto_get_driverid(0);
423 if (sc->sc_cid < 0) {
424 aprint_error_dev(self, "couldn't get crypto driver id\n");
425 pci_intr_disestablish(pc, sc->sc_ih);
426 return;
427 }
428
429 sc->sc_rng_need = RND_POOLBITS / NBBY;
430 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM);
431
432 SIMPLEQ_INIT(&sc->sc_freequeue);
433 dmap = sc->sc_dmaa;
434 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
435 struct ubsec_q *q;
436
437 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
438 M_DEVBUF, M_ZERO|M_NOWAIT);
439 if (q == NULL) {
440 aprint_error_dev(self, "can't allocate queue buffers\n");
441 break;
442 }
443
444 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
445 &dmap->d_alloc, 0)) {
446 aprint_error_dev(self, "can't allocate dma buffers\n");
447 free(q, M_DEVBUF);
448 break;
449 }
450 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
451
452 q->q_dma = dmap;
453 sc->sc_queuea[i] = q;
454
455 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
456 }
457
458 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
459 ubsec_newsession, ubsec_freesession, ubsec_process, sc);
460 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
461 ubsec_newsession, ubsec_freesession, ubsec_process, sc);
462 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0,
463 ubsec_newsession, ubsec_freesession, ubsec_process, sc);
464 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0,
465 ubsec_newsession, ubsec_freesession, ubsec_process, sc);
466 if (sc->sc_flags & UBS_FLAGS_AES) {
467 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
468 ubsec_newsession, ubsec_freesession, ubsec_process, sc);
469 }
470
471 /*
472 * Reset Broadcom chip
473 */
474 ubsec_reset_board(sc);
475
476 /*
477 * Init Broadcom specific PCI settings
478 */
479 ubsec_init_pciregs(pa);
480
481 /*
482 * Init Broadcom chip
483 */
484 ubsec_init_board(sc);
485
486 #ifndef UBSEC_NO_RNG
487 if (sc->sc_flags & UBS_FLAGS_RNG) {
488 if (sc->sc_flags & UBS_FLAGS_RNG4)
489 sc->sc_statmask |= BS_STAT_MCR4_DONE;
490 else
491 sc->sc_statmask |= BS_STAT_MCR2_DONE;
492
493 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
494 &sc->sc_rng.rng_q.q_mcr, 0))
495 goto skip_rng;
496
497 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
498 &sc->sc_rng.rng_q.q_ctx, 0)) {
499 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
500 goto skip_rng;
501 }
502
503 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
504 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
505 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
506 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
507 goto skip_rng;
508 }
509
510 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc);
511 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev),
512 RND_TYPE_RNG,
513 RND_FLAG_NO_ESTIMATE|RND_FLAG_HASCB);
514 if (hz >= 100)
515 sc->sc_rnghz = hz / 100;
516 else
517 sc->sc_rnghz = 1;
518 #ifdef __OpenBSD__
519 timeout_set(&sc->sc_rngto, ubsec_rng, sc);
520 timeout_add(&sc->sc_rngto, sc->sc_rnghz);
521 #else
522 callout_init(&sc->sc_rngto, 0);
523 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc);
524 callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
525 #endif
526 skip_rng:
527 if (sc->sc_rnghz)
528 aprint_normal_dev(self, "random number generator enabled\n");
529 else
530 aprint_error_dev(self, "WARNING: random number generator "
531 "disabled\n");
532 }
533 #endif /* UBSEC_NO_RNG */
534
535 if (sc->sc_flags & UBS_FLAGS_KEY) {
536 sc->sc_statmask |= BS_STAT_MCR2_DONE;
537
538 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
539 ubsec_kprocess, sc);
540 #if 0
541 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
542 ubsec_kprocess, sc);
543 #endif
544 }
545 }
546
547 static int
548 ubsec_detach(device_t self, int flags)
549 {
550 struct ubsec_softc *sc = device_private(self);
551 struct ubsec_q *q, *qtmp;
552 volatile u_int32_t ctrl;
553
554 /* disable interrupts */
555 /* XXX wait/abort current ops? where is DMAERR enabled? */
556 ctrl = READ_REG(sc, BS_CTRL);
557
558 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR);
559 if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
560 ctrl &= ~BS_CTRL_MCR4INT;
561
562 WRITE_REG(sc, BS_CTRL, ctrl);
563
564 #ifndef UBSEC_NO_RNG
565 if (sc->sc_flags & UBS_FLAGS_RNG) {
566 callout_halt(&sc->sc_rngto, NULL);
567 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
568 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
569 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
570 rnd_detach_source(&sc->sc_rnd_source);
571 }
572 #endif /* UBSEC_NO_RNG */
573
574 crypto_unregister_all(sc->sc_cid);
575
576 mutex_spin_enter(&sc->sc_mtx);
577
578 ubsec_totalreset(sc); /* XXX leaves the chip running */
579
580 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) {
581 ubsec_dma_free(sc, &q->q_dma->d_alloc);
582 if (q->q_src_map != NULL)
583 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
584 if (q->q_cached_dst_map != NULL)
585 bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map);
586 free(q, M_DEVBUF);
587 }
588
589 mutex_spin_exit(&sc->sc_mtx);
590
591 if (sc->sc_ih != NULL) {
592 pci_intr_disestablish(sc->sc_pct, sc->sc_ih);
593 sc->sc_ih = NULL;
594 }
595
596 if (sc->sc_memsize != 0) {
597 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize);
598 sc->sc_memsize = 0;
599 }
600
601 return 0;
602 }
603
604 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto");
605
606 #ifdef _MODULE
607 #include "ioconf.c"
608 #endif
609
610 static int
611 ubsec_modcmd(modcmd_t cmd, void *data)
612 {
613 int error = 0;
614
615 switch (cmd) {
616 case MODULE_CMD_INIT:
617 #ifdef _MODULE
618 error = config_init_component(cfdriver_ioconf_ubsec,
619 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
620 #endif
621 if (error == 0)
622 error = ubsec_sysctl_init();
623 return error;
624 case MODULE_CMD_FINI:
625 if (ubsec_sysctllog != NULL)
626 sysctl_teardown(&ubsec_sysctllog);
627 #ifdef _MODULE
628 error = config_fini_component(cfdriver_ioconf_ubsec,
629 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
630 #endif
631 return error;
632 default:
633 return ENOTTY;
634 }
635 }
636
637 static int
638 ubsec_sysctl_init(void)
639 {
640 const struct sysctlnode *node = NULL;
641
642 ubsec_sysctllog = NULL;
643
644 sysctl_createv(&ubsec_sysctllog, 0, NULL, &node,
645 CTLFLAG_PERMANENT,
646 CTLTYPE_NODE, "ubsec",
647 SYSCTL_DESCR("ubsec opetions"),
648 NULL, 0, NULL, 0,
649 CTL_HW, CTL_CREATE, CTL_EOL);
650 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL,
651 CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
652 CTLTYPE_INT, "maxbatch",
653 SYSCTL_DESCR("max ops to batch w/o interrupt"),
654 NULL, 0, &ubsec_maxbatch, 0,
655 CTL_CREATE, CTL_EOL);
656 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL,
657 CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
658 CTLTYPE_INT, "maxaggr",
659 SYSCTL_DESCR("max ops to aggregate under one interrupt"),
660 NULL, 0, &ubsec_maxaggr, 0,
661 CTL_CREATE, CTL_EOL);
662
663 return 0;
664 }
665
666 /*
667 * UBSEC Interrupt routine
668 */
669 static int
670 ubsec_intr(void *arg)
671 {
672 struct ubsec_softc *sc = arg;
673 volatile u_int32_t stat;
674 struct ubsec_q *q;
675 struct ubsec_dma *dmap;
676 int flags;
677 int npkts = 0, i;
678
679 mutex_spin_enter(&sc->sc_mtx);
680 stat = READ_REG(sc, BS_STAT);
681 stat &= sc->sc_statmask;
682 if (stat == 0) {
683 mutex_spin_exit(&sc->sc_mtx);
684 return (0);
685 }
686
687 WRITE_REG(sc, BS_STAT, stat); /* IACK */
688
689 /*
690 * Check to see if we have any packets waiting for us
691 */
692 if ((stat & BS_STAT_MCR1_DONE)) {
693 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
694 q = SIMPLEQ_FIRST(&sc->sc_qchip);
695 dmap = q->q_dma;
696
697 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
698 break;
699
700 q = SIMPLEQ_FIRST(&sc->sc_qchip);
701 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next);
702
703 npkts = q->q_nstacked_mcrs;
704 sc->sc_nqchip -= 1+npkts;
705 /*
706 * search for further sc_qchip ubsec_q's that share
707 * the same MCR, and complete them too, they must be
708 * at the top.
709 */
710 for (i = 0; i < npkts; i++) {
711 if(q->q_stacked_mcr[i])
712 ubsec_callback(sc, q->q_stacked_mcr[i]);
713 else
714 break;
715 }
716 ubsec_callback(sc, q);
717 }
718
719 /*
720 * Don't send any more packet to chip if there has been
721 * a DMAERR.
722 */
723 if (!(stat & BS_STAT_DMAERR))
724 ubsec_feed(sc);
725 }
726
727 /*
728 * Check to see if we have any key setups/rng's waiting for us
729 */
730 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
731 (stat & BS_STAT_MCR2_DONE)) {
732 struct ubsec_q2 *q2;
733 struct ubsec_mcr *mcr;
734
735 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
736 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
737
738 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
739 0, q2->q_mcr.dma_map->dm_mapsize,
740 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
741
742 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
743
744 /* A bug in new devices requires to swap this field */
745 if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
746 flags = htole16(mcr->mcr_flags);
747 else
748 flags = mcr->mcr_flags;
749 if ((flags & htole16(UBS_MCR_DONE)) == 0) {
750 bus_dmamap_sync(sc->sc_dmat,
751 q2->q_mcr.dma_map, 0,
752 q2->q_mcr.dma_map->dm_mapsize,
753 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
754 break;
755 }
756 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
757 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next);
758 ubsec_callback2(sc, q2);
759 /*
760 * Don't send any more packet to chip if there has been
761 * a DMAERR.
762 */
763 if (!(stat & BS_STAT_DMAERR))
764 ubsec_feed2(sc);
765 }
766 }
767 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) {
768 struct ubsec_q2 *q2;
769 struct ubsec_mcr *mcr;
770
771 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) {
772 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4);
773
774 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
775 0, q2->q_mcr.dma_map->dm_mapsize,
776 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
777
778 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
779
780 /* A bug in new devices requires to swap this field */
781 flags = htole16(mcr->mcr_flags);
782
783 if ((flags & htole16(UBS_MCR_DONE)) == 0) {
784 bus_dmamap_sync(sc->sc_dmat,
785 q2->q_mcr.dma_map, 0,
786 q2->q_mcr.dma_map->dm_mapsize,
787 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
788 break;
789 }
790 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next);
791 ubsec_callback2(sc, q2);
792 /*
793 * Don't send any more packet to chip if there has been
794 * a DMAERR.
795 */
796 if (!(stat & BS_STAT_DMAERR))
797 ubsec_feed4(sc);
798 }
799 }
800
801 /*
802 * Check to see if we got any DMA Error
803 */
804 if (stat & BS_STAT_DMAERR) {
805 #ifdef UBSEC_DEBUG
806 if (ubsec_debug) {
807 volatile u_int32_t a = READ_REG(sc, BS_ERR);
808
809 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev),
810 (a & BS_ERR_READ) ? "read" : "write",
811 a & BS_ERR_ADDR);
812 }
813 #endif /* UBSEC_DEBUG */
814 ubsecstats.hst_dmaerr++;
815 ubsec_totalreset(sc);
816 ubsec_feed(sc);
817 }
818
819 if (sc->sc_needwakeup) { /* XXX check high watermark */
820 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
821 #ifdef UBSEC_DEBUG
822 if (ubsec_debug)
823 printf("%s: wakeup crypto (%x)\n", device_xname(sc->sc_dev),
824 sc->sc_needwakeup);
825 #endif /* UBSEC_DEBUG */
826 sc->sc_needwakeup &= ~wkeup;
827 crypto_unblock(sc->sc_cid, wkeup);
828 }
829 mutex_spin_exit(&sc->sc_mtx);
830 return (1);
831 }
832
833 /*
834 * ubsec_feed() - aggregate and post requests to chip
835 * OpenBSD comments:
836 * It is assumed that the caller set splnet()
837 */
838 static void
839 ubsec_feed(struct ubsec_softc *sc)
840 {
841 struct ubsec_q *q, *q2;
842 int npkts, i;
843 void *v;
844 u_int32_t stat;
845 #ifdef UBSEC_DEBUG
846 static int max;
847 #endif /* UBSEC_DEBUG */
848
849 npkts = sc->sc_nqueue;
850 if (npkts > ubsecstats.hst_maxqueue)
851 ubsecstats.hst_maxqueue = npkts;
852 if (npkts < 2)
853 goto feed1;
854
855 /*
856 * Decide how many ops to combine in a single MCR. We cannot
857 * aggregate more than UBS_MAX_AGGR because this is the number
858 * of slots defined in the data structure. Otherwise we clamp
859 * based on the tunable parameter ubsec_maxaggr. Note that
860 * aggregation can happen in two ways: either by batching ops
861 * from above or because the h/w backs up and throttles us.
862 * Aggregating ops reduces the number of interrupts to the host
863 * but also (potentially) increases the latency for processing
864 * completed ops as we only get an interrupt when all aggregated
865 * ops have completed.
866 */
867 if (npkts > sc->sc_maxaggr)
868 npkts = sc->sc_maxaggr;
869 if (npkts > ubsec_maxaggr)
870 npkts = ubsec_maxaggr;
871 if (npkts > ubsecstats.hst_maxbatch)
872 ubsecstats.hst_maxbatch = npkts;
873 if (npkts < 2)
874 goto feed1;
875 ubsecstats.hst_totbatch += npkts-1;
876
877 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
878 if (stat & BS_STAT_DMAERR) {
879 ubsec_totalreset(sc);
880 ubsecstats.hst_dmaerr++;
881 } else {
882 ubsecstats.hst_mcr1full++;
883 }
884 return;
885 }
886
887 #ifdef UBSEC_DEBUG
888 if (ubsec_debug)
889 printf("merging %d records\n", npkts);
890 /* XXX temporary aggregation statistics reporting code */
891 if (max < npkts) {
892 max = npkts;
893 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), max);
894 }
895 #endif /* UBSEC_DEBUG */
896
897 q = SIMPLEQ_FIRST(&sc->sc_queue);
898 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next);
899 --sc->sc_nqueue;
900
901 bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
902 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
903 if (q->q_dst_map != NULL)
904 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
905 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
906
907 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */
908
909 for (i = 0; i < q->q_nstacked_mcrs; i++) {
910 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
911 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
912 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
913 if (q2->q_dst_map != NULL)
914 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
915 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
916 q2= SIMPLEQ_FIRST(&sc->sc_queue);
917 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next);
918 --sc->sc_nqueue;
919
920 v = ((void *)&q2->q_dma->d_dma->d_mcr);
921 v = (char*)v + (sizeof(struct ubsec_mcr) -
922 sizeof(struct ubsec_mcr_add));
923 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, sizeof(struct ubsec_mcr_add));
924 q->q_stacked_mcr[i] = q2;
925 }
926 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
927 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
928 sc->sc_nqchip += npkts;
929 if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
930 ubsecstats.hst_maxqchip = sc->sc_nqchip;
931 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
932 0, q->q_dma->d_alloc.dma_map->dm_mapsize,
933 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
934 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
935 offsetof(struct ubsec_dmachunk, d_mcr));
936 return;
937
938 feed1:
939 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
940 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
941 if (stat & BS_STAT_DMAERR) {
942 ubsec_totalreset(sc);
943 ubsecstats.hst_dmaerr++;
944 } else {
945 ubsecstats.hst_mcr1full++;
946 }
947 break;
948 }
949
950 q = SIMPLEQ_FIRST(&sc->sc_queue);
951
952 bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
953 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
954 if (q->q_dst_map != NULL)
955 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
956 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
957 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
958 0, q->q_dma->d_alloc.dma_map->dm_mapsize,
959 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
960
961 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
962 offsetof(struct ubsec_dmachunk, d_mcr));
963 #ifdef UBSEC_DEBUG
964 if (ubsec_debug)
965 printf("feed: q->chip %p %08x stat %08x\n",
966 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr,
967 stat);
968 #endif /* UBSEC_DEBUG */
969 q = SIMPLEQ_FIRST(&sc->sc_queue);
970 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next);
971 --sc->sc_nqueue;
972 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
973 sc->sc_nqchip++;
974 }
975 if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
976 ubsecstats.hst_maxqchip = sc->sc_nqchip;
977 }
978
979 /*
980 * Allocate a new 'session' and return an encoded session id. 'sidp'
981 * contains our registration id, and should contain an encoded session
982 * id on successful allocation.
983 */
984 static int
985 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
986 {
987 struct cryptoini *c, *encini = NULL, *macini = NULL;
988 struct ubsec_softc *sc;
989 struct ubsec_session *ses = NULL;
990 MD5_CTX md5ctx;
991 SHA1_CTX sha1ctx;
992 int i, sesn;
993
994 sc = arg;
995 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/);
996
997 if (sidp == NULL || cri == NULL || sc == NULL)
998 return (EINVAL);
999
1000 for (c = cri; c != NULL; c = c->cri_next) {
1001 if (c->cri_alg == CRYPTO_MD5_HMAC_96 ||
1002 c->cri_alg == CRYPTO_SHA1_HMAC_96) {
1003 if (macini)
1004 return (EINVAL);
1005 macini = c;
1006 } else if (c->cri_alg == CRYPTO_DES_CBC ||
1007 c->cri_alg == CRYPTO_3DES_CBC ||
1008 c->cri_alg == CRYPTO_AES_CBC) {
1009 if (encini)
1010 return (EINVAL);
1011 encini = c;
1012 } else
1013 return (EINVAL);
1014 }
1015 if (encini == NULL && macini == NULL)
1016 return (EINVAL);
1017
1018 if (encini && encini->cri_alg == CRYPTO_AES_CBC) {
1019 switch (encini->cri_klen) {
1020 case 128:
1021 case 192:
1022 case 256:
1023 break;
1024 default:
1025 return (EINVAL);
1026 }
1027 }
1028
1029 if (sc->sc_sessions == NULL) {
1030 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
1031 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1032 if (ses == NULL)
1033 return (ENOMEM);
1034 sesn = 0;
1035 sc->sc_nsessions = 1;
1036 } else {
1037 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
1038 if (sc->sc_sessions[sesn].ses_used == 0) {
1039 ses = &sc->sc_sessions[sesn];
1040 break;
1041 }
1042 }
1043
1044 if (ses == NULL) {
1045 sesn = sc->sc_nsessions;
1046 ses = (struct ubsec_session *)malloc((sesn + 1) *
1047 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1048 if (ses == NULL)
1049 return (ENOMEM);
1050 memcpy(ses, sc->sc_sessions, sesn *
1051 sizeof(struct ubsec_session));
1052 memset(sc->sc_sessions, 0, sesn *
1053 sizeof(struct ubsec_session));
1054 free(sc->sc_sessions, M_DEVBUF);
1055 sc->sc_sessions = ses;
1056 ses = &sc->sc_sessions[sesn];
1057 sc->sc_nsessions++;
1058 }
1059 }
1060
1061 memset(ses, 0, sizeof(struct ubsec_session));
1062 ses->ses_used = 1;
1063 if (encini) {
1064 /* get an IV, network byte order */
1065 #ifdef __NetBSD__
1066 cprng_fast(ses->ses_iv, sizeof(ses->ses_iv));
1067 #else
1068 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
1069 #endif
1070
1071 /* Go ahead and compute key in ubsec's byte order */
1072 if (encini->cri_alg == CRYPTO_AES_CBC) {
1073 memcpy(ses->ses_key, encini->cri_key,
1074 encini->cri_klen / 8);
1075 }
1076 if (encini->cri_alg == CRYPTO_DES_CBC) {
1077 memcpy(&ses->ses_key[0], encini->cri_key, 8);
1078 memcpy(&ses->ses_key[2], encini->cri_key, 8);
1079 memcpy(&ses->ses_key[4], encini->cri_key, 8);
1080 } else
1081 memcpy(ses->ses_key, encini->cri_key, 24);
1082
1083 SWAP32(ses->ses_key[0]);
1084 SWAP32(ses->ses_key[1]);
1085 SWAP32(ses->ses_key[2]);
1086 SWAP32(ses->ses_key[3]);
1087 SWAP32(ses->ses_key[4]);
1088 SWAP32(ses->ses_key[5]);
1089 }
1090
1091 if (macini) {
1092 for (i = 0; i < macini->cri_klen / 8; i++)
1093 macini->cri_key[i] ^= HMAC_IPAD_VAL;
1094
1095 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1096 MD5Init(&md5ctx);
1097 MD5Update(&md5ctx, macini->cri_key,
1098 macini->cri_klen / 8);
1099 MD5Update(&md5ctx, hmac_ipad_buffer,
1100 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1101 memcpy(ses->ses_hminner, md5ctx.state,
1102 sizeof(md5ctx.state));
1103 } else {
1104 SHA1Init(&sha1ctx);
1105 SHA1Update(&sha1ctx, macini->cri_key,
1106 macini->cri_klen / 8);
1107 SHA1Update(&sha1ctx, hmac_ipad_buffer,
1108 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1109 memcpy(ses->ses_hminner, sha1ctx.state,
1110 sizeof(sha1ctx.state));
1111 }
1112
1113 for (i = 0; i < macini->cri_klen / 8; i++)
1114 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
1115
1116 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1117 MD5Init(&md5ctx);
1118 MD5Update(&md5ctx, macini->cri_key,
1119 macini->cri_klen / 8);
1120 MD5Update(&md5ctx, hmac_opad_buffer,
1121 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1122 memcpy(ses->ses_hmouter, md5ctx.state,
1123 sizeof(md5ctx.state));
1124 } else {
1125 SHA1Init(&sha1ctx);
1126 SHA1Update(&sha1ctx, macini->cri_key,
1127 macini->cri_klen / 8);
1128 SHA1Update(&sha1ctx, hmac_opad_buffer,
1129 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1130 memcpy(ses->ses_hmouter, sha1ctx.state,
1131 sizeof(sha1ctx.state));
1132 }
1133
1134 for (i = 0; i < macini->cri_klen / 8; i++)
1135 macini->cri_key[i] ^= HMAC_OPAD_VAL;
1136 }
1137
1138 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn);
1139 return (0);
1140 }
1141
1142 /*
1143 * Deallocate a session.
1144 */
1145 static int
1146 ubsec_freesession(void *arg, u_int64_t tid)
1147 {
1148 struct ubsec_softc *sc;
1149 int session;
1150 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
1151
1152 sc = arg;
1153 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/);
1154
1155 session = UBSEC_SESSION(sid);
1156 if (session >= sc->sc_nsessions)
1157 return (EINVAL);
1158
1159 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session]));
1160 return (0);
1161 }
1162
1163 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */
1164 static void
1165 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1166 {
1167 struct ubsec_operand *op = arg;
1168
1169 KASSERT(nsegs <= UBS_MAX_SCATTER
1170 /*, ("Too many DMA segments returned when mapping operand")*/);
1171 #ifdef UBSEC_DEBUG
1172 if (ubsec_debug)
1173 printf("ubsec_op_cb: mapsize %u nsegs %d\n",
1174 (u_int) mapsize, nsegs);
1175 #endif
1176 op->mapsize = mapsize;
1177 op->nsegs = nsegs;
1178 memcpy(op->segs, seg, nsegs * sizeof (seg[0]));
1179 }
1180 #endif
1181
1182 static int
1183 ubsec_process(void *arg, struct cryptop *crp, int hint)
1184 {
1185 struct ubsec_q *q = NULL;
1186 #ifdef __OpenBSD__
1187 int card;
1188 #endif
1189 int err = 0, i, j, nicealign;
1190 struct ubsec_softc *sc;
1191 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1192 int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1193 int sskip, dskip, stheend, dtheend;
1194 int16_t coffset;
1195 struct ubsec_session *ses, key;
1196 struct ubsec_dma *dmap = NULL;
1197 u_int16_t flags = 0;
1198 int ivlen = 0, keylen = 0;
1199
1200 sc = arg;
1201 KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/);
1202
1203 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1204 ubsecstats.hst_invalid++;
1205 return (EINVAL);
1206 }
1207 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1208 ubsecstats.hst_badsession++;
1209 return (EINVAL);
1210 }
1211
1212 mutex_spin_enter(&sc->sc_mtx);
1213
1214 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1215 ubsecstats.hst_queuefull++;
1216 sc->sc_needwakeup |= CRYPTO_SYMQ;
1217 mutex_spin_exit(&sc->sc_mtx);
1218 return(ERESTART);
1219 }
1220
1221 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1222 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next);
1223 mutex_spin_exit(&sc->sc_mtx);
1224
1225 dmap = q->q_dma; /* Save dma pointer */
1226 /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */
1227 memset(q, 0, offsetof(struct ubsec_q, q_src_map));
1228 memset(&key, 0, sizeof(key));
1229
1230 q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1231 q->q_dma = dmap;
1232 ses = &sc->sc_sessions[q->q_sesn];
1233
1234 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1235 q->q_src_m = (struct mbuf *)crp->crp_buf;
1236 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1237 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1238 q->q_src_io = (struct uio *)crp->crp_buf;
1239 q->q_dst_io = (struct uio *)crp->crp_buf;
1240 } else {
1241 ubsecstats.hst_badflags++;
1242 err = EINVAL;
1243 goto errout; /* XXX we don't handle contiguous blocks! */
1244 }
1245
1246 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr));
1247
1248 dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1249 dmap->d_dma->d_mcr.mcr_flags = 0;
1250 q->q_crp = crp;
1251
1252 crd1 = crp->crp_desc;
1253 if (crd1 == NULL) {
1254 ubsecstats.hst_nodesc++;
1255 err = EINVAL;
1256 goto errout;
1257 }
1258 crd2 = crd1->crd_next;
1259
1260 if (crd2 == NULL) {
1261 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
1262 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) {
1263 maccrd = crd1;
1264 enccrd = NULL;
1265 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1266 crd1->crd_alg == CRYPTO_3DES_CBC ||
1267 crd1->crd_alg == CRYPTO_AES_CBC) {
1268 maccrd = NULL;
1269 enccrd = crd1;
1270 } else {
1271 ubsecstats.hst_badalg++;
1272 err = EINVAL;
1273 goto errout;
1274 }
1275 } else {
1276 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
1277 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1278 (crd2->crd_alg == CRYPTO_DES_CBC ||
1279 crd2->crd_alg == CRYPTO_3DES_CBC ||
1280 crd2->crd_alg == CRYPTO_AES_CBC) &&
1281 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1282 maccrd = crd1;
1283 enccrd = crd2;
1284 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1285 crd1->crd_alg == CRYPTO_3DES_CBC ||
1286 crd1->crd_alg == CRYPTO_AES_CBC) &&
1287 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 ||
1288 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1289 (crd1->crd_flags & CRD_F_ENCRYPT)) {
1290 enccrd = crd1;
1291 maccrd = crd2;
1292 } else {
1293 /*
1294 * We cannot order the ubsec as requested
1295 */
1296 ubsecstats.hst_badalg++;
1297 err = EINVAL;
1298 goto errout;
1299 }
1300 }
1301
1302 if (enccrd) {
1303 if (enccrd->crd_alg == CRYPTO_AES_CBC) {
1304 if ((sc->sc_flags & UBS_FLAGS_AES) == 0) {
1305 /*
1306 * We cannot order the ubsec as requested
1307 */
1308 ubsecstats.hst_badalg++;
1309 err = EINVAL;
1310 goto errout;
1311 }
1312 flags |= htole16(UBS_PKTCTX_ENC_AES);
1313 switch (enccrd->crd_klen) {
1314 case 128:
1315 case 192:
1316 case 256:
1317 keylen = enccrd->crd_klen / 8;
1318 break;
1319 default:
1320 err = EINVAL;
1321 goto errout;
1322 }
1323 ivlen = 16;
1324 } else {
1325 flags |= htole16(UBS_PKTCTX_ENC_3DES);
1326 ivlen = 8;
1327 keylen = 24;
1328 }
1329
1330 encoffset = enccrd->crd_skip;
1331
1332 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1333 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1334
1335 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1336 memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1337 else {
1338 for (i = 0; i < (ivlen / 4); i++)
1339 key.ses_iv[i] = ses->ses_iv[i];
1340 }
1341
1342 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1343 if (crp->crp_flags & CRYPTO_F_IMBUF)
1344 m_copyback(q->q_src_m,
1345 enccrd->crd_inject,
1346 ivlen, (void *)key.ses_iv);
1347 else if (crp->crp_flags & CRYPTO_F_IOV)
1348 cuio_copyback(q->q_src_io,
1349 enccrd->crd_inject,
1350 ivlen, (void *)key.ses_iv);
1351 }
1352 } else {
1353 flags |= htole16(UBS_PKTCTX_INBOUND);
1354
1355 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1356 memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1357 else if (crp->crp_flags & CRYPTO_F_IMBUF)
1358 m_copydata(q->q_src_m, enccrd->crd_inject,
1359 ivlen, (void *)key.ses_iv);
1360 else if (crp->crp_flags & CRYPTO_F_IOV)
1361 cuio_copydata(q->q_src_io,
1362 enccrd->crd_inject, 8,
1363 (void *)key.ses_iv);
1364 }
1365
1366 for (i = 0; i < (keylen / 4); i++)
1367 key.ses_key[i] = ses->ses_key[i];
1368 for (i = 0; i < (ivlen / 4); i++)
1369 SWAP32(key.ses_iv[i]);
1370 }
1371
1372 if (maccrd) {
1373 macoffset = maccrd->crd_skip;
1374
1375 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96)
1376 flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1377 else
1378 flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1379
1380 for (i = 0; i < 5; i++) {
1381 key.ses_hminner[i] = ses->ses_hminner[i];
1382 key.ses_hmouter[i] = ses->ses_hmouter[i];
1383
1384 HTOLE32(key.ses_hminner[i]);
1385 HTOLE32(key.ses_hmouter[i]);
1386 }
1387 }
1388
1389 if (enccrd && maccrd) {
1390 /*
1391 * ubsec cannot handle packets where the end of encryption
1392 * and authentication are not the same, or where the
1393 * encrypted part begins before the authenticated part.
1394 */
1395 if ((encoffset + enccrd->crd_len) !=
1396 (macoffset + maccrd->crd_len)) {
1397 ubsecstats.hst_lenmismatch++;
1398 err = EINVAL;
1399 goto errout;
1400 }
1401 if (enccrd->crd_skip < maccrd->crd_skip) {
1402 ubsecstats.hst_skipmismatch++;
1403 err = EINVAL;
1404 goto errout;
1405 }
1406 sskip = maccrd->crd_skip;
1407 cpskip = dskip = enccrd->crd_skip;
1408 stheend = maccrd->crd_len;
1409 dtheend = enccrd->crd_len;
1410 coffset = enccrd->crd_skip - maccrd->crd_skip;
1411 cpoffset = cpskip + dtheend;
1412 #ifdef UBSEC_DEBUG
1413 if (ubsec_debug) {
1414 printf("mac: skip %d, len %d, inject %d\n",
1415 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1416 printf("enc: skip %d, len %d, inject %d\n",
1417 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1418 printf("src: skip %d, len %d\n", sskip, stheend);
1419 printf("dst: skip %d, len %d\n", dskip, dtheend);
1420 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1421 coffset, stheend, cpskip, cpoffset);
1422 }
1423 #endif
1424 } else {
1425 cpskip = dskip = sskip = macoffset + encoffset;
1426 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1427 cpoffset = cpskip + dtheend;
1428 coffset = 0;
1429 }
1430
1431 if (q->q_src_map == NULL) {
1432 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */
1433 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER,
1434 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) {
1435 err = ENOMEM;
1436 goto errout;
1437 }
1438 }
1439 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1440 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1441 q->q_src_m, BUS_DMA_NOWAIT) != 0) {
1442 ubsecstats.hst_noload++;
1443 err = ENOMEM;
1444 goto errout;
1445 }
1446 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1447 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1448 q->q_src_io, BUS_DMA_NOWAIT) != 0) {
1449 ubsecstats.hst_noload++;
1450 err = ENOMEM;
1451 goto errout;
1452 }
1453 }
1454 nicealign = ubsec_dmamap_aligned(q->q_src_map);
1455
1456 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1457
1458 #ifdef UBSEC_DEBUG
1459 if (ubsec_debug)
1460 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1461 #endif
1462 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) {
1463 struct ubsec_pktbuf *pb;
1464 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len;
1465 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr;
1466
1467 if (sskip >= packl) {
1468 sskip -= packl;
1469 continue;
1470 }
1471
1472 packl -= sskip;
1473 packp += sskip;
1474 sskip = 0;
1475
1476 if (packl > 0xfffc) {
1477 err = EIO;
1478 goto errout;
1479 }
1480
1481 if (j == 0)
1482 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1483 else
1484 pb = &dmap->d_dma->d_sbuf[j - 1];
1485
1486 pb->pb_addr = htole32(packp);
1487
1488 if (stheend) {
1489 if (packl > stheend) {
1490 pb->pb_len = htole32(stheend);
1491 stheend = 0;
1492 } else {
1493 pb->pb_len = htole32(packl);
1494 stheend -= packl;
1495 }
1496 } else
1497 pb->pb_len = htole32(packl);
1498
1499 if ((i + 1) == q->q_src_map->dm_nsegs)
1500 pb->pb_next = 0;
1501 else
1502 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1503 offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1504 j++;
1505 }
1506
1507 if (enccrd == NULL && maccrd != NULL) {
1508 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1509 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1510 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1511 offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1512 #ifdef UBSEC_DEBUG
1513 if (ubsec_debug)
1514 printf("opkt: %x %x %x\n",
1515 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1516 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1517 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1518
1519 #endif
1520 } else {
1521 if (crp->crp_flags & CRYPTO_F_IOV) {
1522 if (!nicealign) {
1523 ubsecstats.hst_iovmisaligned++;
1524 err = EINVAL;
1525 goto errout;
1526 }
1527 if (q->q_dst_map == NULL) {
1528 if (q->q_cached_dst_map == NULL) {
1529 /* XXX: ``what the heck's that'' 0xfff0? */
1530 if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1531 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1532 &q->q_cached_dst_map) != 0) {
1533 ubsecstats.hst_nomap++;
1534 err = ENOMEM;
1535 goto errout;
1536 }
1537 }
1538 q->q_dst_map = q->q_cached_dst_map;
1539 }
1540 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1541 q->q_dst_io, BUS_DMA_NOWAIT) != 0) {
1542 ubsecstats.hst_noload++;
1543 err = ENOMEM;
1544 goto errout;
1545 }
1546 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1547 if (nicealign) {
1548 q->q_dst_m = q->q_src_m;
1549 q->q_dst_map = q->q_src_map;
1550 } else {
1551 int totlen, len;
1552 struct mbuf *m, *top, **mp;
1553
1554 ubsecstats.hst_unaligned++;
1555 totlen = q->q_src_map->dm_mapsize;
1556 if (q->q_src_m->m_flags & M_PKTHDR) {
1557 len = MHLEN;
1558 MGETHDR(m, M_DONTWAIT, MT_DATA);
1559 /*XXX FIXME: m_dup_pkthdr */
1560 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) {
1561 m_free(m);
1562 m = NULL;
1563 }
1564 } else {
1565 len = MLEN;
1566 MGET(m, M_DONTWAIT, MT_DATA);
1567 }
1568 if (m == NULL) {
1569 ubsecstats.hst_nombuf++;
1570 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1571 goto errout;
1572 }
1573 if (len == MHLEN)
1574 /*XXX was M_DUP_PKTHDR*/
1575 M_COPY_PKTHDR(m, q->q_src_m);
1576 if (totlen >= MINCLSIZE) {
1577 MCLGET(m, M_DONTWAIT);
1578 if ((m->m_flags & M_EXT) == 0) {
1579 m_free(m);
1580 ubsecstats.hst_nomcl++;
1581 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1582 goto errout;
1583 }
1584 len = MCLBYTES;
1585 }
1586 m->m_len = len;
1587 top = NULL;
1588 mp = ⊤
1589
1590 while (totlen > 0) {
1591 if (top) {
1592 MGET(m, M_DONTWAIT, MT_DATA);
1593 if (m == NULL) {
1594 m_freem(top);
1595 ubsecstats.hst_nombuf++;
1596 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1597 goto errout;
1598 }
1599 len = MLEN;
1600 }
1601 if (top && totlen >= MINCLSIZE) {
1602 MCLGET(m, M_DONTWAIT);
1603 if ((m->m_flags & M_EXT) == 0) {
1604 *mp = m;
1605 m_freem(top);
1606 ubsecstats.hst_nomcl++;
1607 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1608 goto errout;
1609 }
1610 len = MCLBYTES;
1611 }
1612 m->m_len = len = min(totlen, len);
1613 totlen -= len;
1614 *mp = m;
1615 mp = &m->m_next;
1616 }
1617 q->q_dst_m = top;
1618 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1619 cpskip, cpoffset);
1620 if (q->q_dst_map == NULL) {
1621 if (q->q_cached_dst_map == NULL) {
1622 /* XXX again, what the heck is that 0xfff0? */
1623 if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1624 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1625 &q->q_cached_dst_map) != 0) {
1626 ubsecstats.hst_nomap++;
1627 err = ENOMEM;
1628 goto errout;
1629 }
1630 }
1631 q->q_dst_map = q->q_cached_dst_map;
1632 }
1633 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1634 q->q_dst_map, q->q_dst_m,
1635 BUS_DMA_NOWAIT) != 0) {
1636 ubsecstats.hst_noload++;
1637 err = ENOMEM;
1638 goto errout;
1639 }
1640 }
1641 } else {
1642 ubsecstats.hst_badflags++;
1643 err = EINVAL;
1644 goto errout;
1645 }
1646
1647 #ifdef UBSEC_DEBUG
1648 if (ubsec_debug)
1649 printf("dst skip: %d\n", dskip);
1650 #endif
1651 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) {
1652 struct ubsec_pktbuf *pb;
1653 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len;
1654 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr;
1655
1656 if (dskip >= packl) {
1657 dskip -= packl;
1658 continue;
1659 }
1660
1661 packl -= dskip;
1662 packp += dskip;
1663 dskip = 0;
1664
1665 if (packl > 0xfffc) {
1666 err = EIO;
1667 goto errout;
1668 }
1669
1670 if (j == 0)
1671 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1672 else
1673 pb = &dmap->d_dma->d_dbuf[j - 1];
1674
1675 pb->pb_addr = htole32(packp);
1676
1677 if (dtheend) {
1678 if (packl > dtheend) {
1679 pb->pb_len = htole32(dtheend);
1680 dtheend = 0;
1681 } else {
1682 pb->pb_len = htole32(packl);
1683 dtheend -= packl;
1684 }
1685 } else
1686 pb->pb_len = htole32(packl);
1687
1688 if ((i + 1) == q->q_dst_map->dm_nsegs) {
1689 if (maccrd)
1690 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1691 offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1692 else
1693 pb->pb_next = 0;
1694 } else
1695 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1696 offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1697 j++;
1698 }
1699 }
1700
1701 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1702 offsetof(struct ubsec_dmachunk, d_ctx));
1703
1704 if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) {
1705 struct ubsec_pktctx_aes128 *aes128;
1706 struct ubsec_pktctx_aes192 *aes192;
1707 struct ubsec_pktctx_aes256 *aes256;
1708 struct ubsec_pktctx_hdr *ph;
1709 u_int8_t *ctx;
1710
1711 ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1712 offsetof(struct ubsec_dmachunk, d_ctx);
1713
1714 ph = (struct ubsec_pktctx_hdr *)ctx;
1715 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES);
1716 ph->ph_flags = flags;
1717 ph->ph_offset = htole16(coffset >> 2);
1718
1719 switch (enccrd->crd_klen) {
1720 case 128:
1721 aes128 = (struct ubsec_pktctx_aes128 *)ctx;
1722 ph->ph_len = htole16(sizeof(*aes128));
1723 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128);
1724 for (i = 0; i < 4; i++)
1725 aes128->pc_aeskey[i] = key.ses_key[i];
1726 for (i = 0; i < 5; i++)
1727 aes128->pc_hminner[i] = key.ses_hminner[i];
1728 for (i = 0; i < 5; i++)
1729 aes128->pc_hmouter[i] = key.ses_hmouter[i];
1730 for (i = 0; i < 4; i++)
1731 aes128->pc_iv[i] = key.ses_iv[i];
1732 break;
1733 case 192:
1734 aes192 = (struct ubsec_pktctx_aes192 *)ctx;
1735 ph->ph_len = htole16(sizeof(*aes192));
1736 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192);
1737 for (i = 0; i < 6; i++)
1738 aes192->pc_aeskey[i] = key.ses_key[i];
1739 for (i = 0; i < 5; i++)
1740 aes192->pc_hminner[i] = key.ses_hminner[i];
1741 for (i = 0; i < 5; i++)
1742 aes192->pc_hmouter[i] = key.ses_hmouter[i];
1743 for (i = 0; i < 4; i++)
1744 aes192->pc_iv[i] = key.ses_iv[i];
1745 break;
1746 case 256:
1747 aes256 = (struct ubsec_pktctx_aes256 *)ctx;
1748 ph->ph_len = htole16(sizeof(*aes256));
1749 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256);
1750 for (i = 0; i < 8; i++)
1751 aes256->pc_aeskey[i] = key.ses_key[i];
1752 for (i = 0; i < 5; i++)
1753 aes256->pc_hminner[i] = key.ses_hminner[i];
1754 for (i = 0; i < 5; i++)
1755 aes256->pc_hmouter[i] = key.ses_hmouter[i];
1756 for (i = 0; i < 4; i++)
1757 aes256->pc_iv[i] = key.ses_iv[i];
1758 break;
1759 }
1760 } else if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1761 struct ubsec_pktctx_3des *ctx;
1762 struct ubsec_pktctx_hdr *ph;
1763
1764 ctx = (struct ubsec_pktctx_3des *)
1765 ((u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1766 offsetof(struct ubsec_dmachunk, d_ctx));
1767
1768 ph = (struct ubsec_pktctx_hdr *)ctx;
1769 ph->ph_len = htole16(sizeof(*ctx));
1770 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES);
1771 ph->ph_flags = flags;
1772 ph->ph_offset = htole16(coffset >> 2);
1773
1774 for (i = 0; i < 6; i++)
1775 ctx->pc_deskey[i] = key.ses_key[i];
1776 for (i = 0; i < 5; i++)
1777 ctx->pc_hminner[i] = key.ses_hminner[i];
1778 for (i = 0; i < 5; i++)
1779 ctx->pc_hmouter[i] = key.ses_hmouter[i];
1780 for (i = 0; i < 2; i++)
1781 ctx->pc_iv[i] = key.ses_iv[i];
1782 } else {
1783 struct ubsec_pktctx *ctx = (struct ubsec_pktctx *)
1784 ((u_int8_t *)dmap->d_alloc.dma_vaddr +
1785 offsetof(struct ubsec_dmachunk, d_ctx));
1786
1787 ctx->pc_flags = flags;
1788 ctx->pc_offset = htole16(coffset >> 2);
1789 for (i = 0; i < 6; i++)
1790 ctx->pc_deskey[i] = key.ses_key[i];
1791 for (i = 0; i < 5; i++)
1792 ctx->pc_hminner[i] = key.ses_hminner[i];
1793 for (i = 0; i < 5; i++)
1794 ctx->pc_hmouter[i] = key.ses_hmouter[i];
1795 for (i = 0; i < 2; i++)
1796 ctx->pc_iv[i] = key.ses_iv[i];
1797 }
1798
1799 mutex_spin_enter(&sc->sc_mtx);
1800 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1801 sc->sc_nqueue++;
1802 ubsecstats.hst_ipackets++;
1803 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize;
1804 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch)
1805 ubsec_feed(sc);
1806 mutex_spin_exit(&sc->sc_mtx);
1807 return (0);
1808
1809 errout:
1810 if (q != NULL) {
1811 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1812 m_freem(q->q_dst_m);
1813
1814 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1815 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1816 }
1817 if (q->q_src_map != NULL) {
1818 bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1819 }
1820
1821 mutex_spin_enter(&sc->sc_mtx);
1822 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1823 mutex_spin_exit(&sc->sc_mtx);
1824 }
1825 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */
1826 if (err == EINVAL)
1827 ubsecstats.hst_invalid++;
1828 else
1829 ubsecstats.hst_nomem++;
1830 #endif
1831 if (err != ERESTART) {
1832 crp->crp_etype = err;
1833 crypto_done(crp);
1834 } else {
1835 sc->sc_needwakeup |= CRYPTO_SYMQ;
1836 }
1837 return (err);
1838 }
1839
1840 static void
1841 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1842 {
1843 struct cryptop *crp = (struct cryptop *)q->q_crp;
1844 struct cryptodesc *crd;
1845 struct ubsec_dma *dmap = q->q_dma;
1846
1847 ubsecstats.hst_opackets++;
1848 ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1849
1850 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0,
1851 dmap->d_alloc.dma_map->dm_mapsize,
1852 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1853 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1854 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1855 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1856 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1857 }
1858 bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
1859 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1860 bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1861
1862 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1863 m_freem(q->q_src_m);
1864 crp->crp_buf = (void *)q->q_dst_m;
1865 }
1866
1867 /* copy out IV for future use */
1868 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1869 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1870 if (crd->crd_alg != CRYPTO_DES_CBC &&
1871 crd->crd_alg != CRYPTO_3DES_CBC &&
1872 crd->crd_alg != CRYPTO_AES_CBC)
1873 continue;
1874 if (crp->crp_flags & CRYPTO_F_IMBUF)
1875 m_copydata((struct mbuf *)crp->crp_buf,
1876 crd->crd_skip + crd->crd_len - 8, 8,
1877 (void *)sc->sc_sessions[q->q_sesn].ses_iv);
1878 else if (crp->crp_flags & CRYPTO_F_IOV) {
1879 cuio_copydata((struct uio *)crp->crp_buf,
1880 crd->crd_skip + crd->crd_len - 8, 8,
1881 (void *)sc->sc_sessions[q->q_sesn].ses_iv);
1882 }
1883 break;
1884 }
1885 }
1886
1887 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1888 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 &&
1889 crd->crd_alg != CRYPTO_SHA1_HMAC_96)
1890 continue;
1891 if (crp->crp_flags & CRYPTO_F_IMBUF)
1892 m_copyback((struct mbuf *)crp->crp_buf,
1893 crd->crd_inject, 12,
1894 (void *)dmap->d_dma->d_macbuf);
1895 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1896 bcopy((void *)dmap->d_dma->d_macbuf,
1897 crp->crp_mac, 12);
1898 break;
1899 }
1900 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1901 crypto_done(crp);
1902 }
1903
1904 static void
1905 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1906 {
1907 int i, j, dlen, slen;
1908 char *dptr, *sptr;
1909
1910 j = 0;
1911 sptr = srcm->m_data;
1912 slen = srcm->m_len;
1913 dptr = dstm->m_data;
1914 dlen = dstm->m_len;
1915
1916 while (1) {
1917 for (i = 0; i < min(slen, dlen); i++) {
1918 if (j < hoffset || j >= toffset)
1919 *dptr++ = *sptr++;
1920 slen--;
1921 dlen--;
1922 j++;
1923 }
1924 if (slen == 0) {
1925 srcm = srcm->m_next;
1926 if (srcm == NULL)
1927 return;
1928 sptr = srcm->m_data;
1929 slen = srcm->m_len;
1930 }
1931 if (dlen == 0) {
1932 dstm = dstm->m_next;
1933 if (dstm == NULL)
1934 return;
1935 dptr = dstm->m_data;
1936 dlen = dstm->m_len;
1937 }
1938 }
1939 }
1940
1941 /*
1942 * feed the key generator, must be called at splnet() or higher.
1943 */
1944 static void
1945 ubsec_feed2(struct ubsec_softc *sc)
1946 {
1947 struct ubsec_q2 *q;
1948
1949 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1950 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1951 break;
1952 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1953
1954 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
1955 q->q_mcr.dma_map->dm_mapsize,
1956 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1957 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1958 q->q_ctx.dma_map->dm_mapsize,
1959 BUS_DMASYNC_PREWRITE);
1960
1961 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1962 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1963 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next);
1964 --sc->sc_nqueue2;
1965 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1966 }
1967 }
1968
1969 /*
1970 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices)
1971 */
1972 void
1973 ubsec_feed4(struct ubsec_softc *sc)
1974 {
1975 struct ubsec_q2 *q;
1976
1977 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) {
1978 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL)
1979 break;
1980 q = SIMPLEQ_FIRST(&sc->sc_queue4);
1981
1982 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
1983 q->q_mcr.dma_map->dm_mapsize,
1984 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1985 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1986 q->q_ctx.dma_map->dm_mapsize,
1987 BUS_DMASYNC_PREWRITE);
1988
1989 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr);
1990 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next);
1991 --sc->sc_nqueue4;
1992 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next);
1993 }
1994 }
1995
1996 /*
1997 * Callback for handling random numbers
1998 */
1999 static void
2000 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
2001 {
2002 struct cryptkop *krp;
2003 struct ubsec_ctx_keyop *ctx;
2004
2005 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
2006 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
2007 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2008
2009 switch (q->q_type) {
2010 #ifndef UBSEC_NO_RNG
2011 case UBS_CTXOP_RNGSHA1:
2012 case UBS_CTXOP_RNGBYPASS: {
2013 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
2014 u_int32_t *p;
2015 int i;
2016
2017 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
2018 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2019 p = (u_int32_t *)rng->rng_buf.dma_vaddr;
2020 #ifndef __NetBSD__
2021 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++)
2022 add_true_randomness(letoh32(*p));
2023 #else
2024 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t);
2025 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY);
2026 sc->sc_rng_need -= i;
2027 #endif
2028 rng->rng_used = 0;
2029 #ifdef __OpenBSD__
2030 timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2031 #else
2032 if (sc->sc_rng_need > 0) {
2033 callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2034 }
2035 #endif
2036 break;
2037 }
2038 #endif
2039 case UBS_CTXOP_MODEXP: {
2040 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2041 u_int rlen, clen;
2042
2043 krp = me->me_krp;
2044 rlen = (me->me_modbits + 7) / 8;
2045 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
2046
2047 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2048 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2049 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2050 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2051 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2052 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2053 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2054 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2055
2056 if (clen < rlen)
2057 krp->krp_status = E2BIG;
2058 else {
2059 if (sc->sc_flags & UBS_FLAGS_HWNORM) {
2060 memset(krp->krp_param[krp->krp_iparams].crp_p, 0,
2061 (krp->krp_param[krp->krp_iparams].crp_nbits
2062 + 7) / 8);
2063 bcopy(me->me_C.dma_vaddr,
2064 krp->krp_param[krp->krp_iparams].crp_p,
2065 (me->me_modbits + 7) / 8);
2066 } else
2067 ubsec_kshift_l(me->me_shiftbits,
2068 me->me_C.dma_vaddr, me->me_normbits,
2069 krp->krp_param[krp->krp_iparams].crp_p,
2070 krp->krp_param[krp->krp_iparams].crp_nbits);
2071 }
2072
2073 crypto_kdone(krp);
2074
2075 /* bzero all potentially sensitive data */
2076 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2077 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2078 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2079 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2080
2081 /* Can't free here, so put us on the free list. */
2082 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
2083 break;
2084 }
2085 case UBS_CTXOP_RSAPRIV: {
2086 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2087 u_int len;
2088
2089 krp = rp->rpr_krp;
2090 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0,
2091 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2092 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0,
2093 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2094
2095 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
2096 bcopy(rp->rpr_msgout.dma_vaddr,
2097 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
2098
2099 crypto_kdone(krp);
2100
2101 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size);
2102 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size);
2103 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size);
2104
2105 /* Can't free here, so put us on the free list. */
2106 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
2107 break;
2108 }
2109 default:
2110 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev),
2111 letoh16(ctx->ctx_op));
2112 break;
2113 }
2114 }
2115
2116 #ifndef UBSEC_NO_RNG
2117
2118 static void
2119 ubsec_rng_get(size_t bytes, void *vsc)
2120 {
2121 struct ubsec_softc *sc = vsc;
2122
2123 mutex_spin_enter(&sc->sc_mtx);
2124 sc->sc_rng_need = bytes;
2125 ubsec_rng_locked(sc);
2126 mutex_spin_exit(&sc->sc_mtx);
2127
2128 }
2129
2130 static void
2131 ubsec_rng(void *vsc)
2132 {
2133 struct ubsec_softc *sc = vsc;
2134 mutex_spin_enter(&sc->sc_mtx);
2135 ubsec_rng_locked(sc);
2136 mutex_spin_exit(&sc->sc_mtx);
2137 }
2138
2139 static void
2140 ubsec_rng_locked(void *vsc)
2141 {
2142 struct ubsec_softc *sc = vsc;
2143 struct ubsec_q2_rng *rng = &sc->sc_rng;
2144 struct ubsec_mcr *mcr;
2145 struct ubsec_ctx_rngbypass *ctx;
2146 int *nqueue;
2147
2148 /* Caller is responsible to lock and release sc_mtx. */
2149 KASSERT(mutex_owned(&sc->sc_mtx));
2150
2151 if (rng->rng_used) {
2152 return;
2153 }
2154
2155 if (sc->sc_rng_need < 1) {
2156 callout_stop(&sc->sc_rngto);
2157 return;
2158 }
2159
2160 if (sc->sc_flags & UBS_FLAGS_RNG4)
2161 nqueue = &sc->sc_nqueue4;
2162 else
2163 nqueue = &sc->sc_nqueue2;
2164
2165 (*nqueue)++;
2166 if (*nqueue >= UBS_MAX_NQUEUE)
2167 goto out;
2168
2169 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
2170 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
2171
2172 mcr->mcr_pkts = htole16(1);
2173 mcr->mcr_flags = 0;
2174 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
2175 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
2176 mcr->mcr_ipktbuf.pb_len = 0;
2177 mcr->mcr_reserved = mcr->mcr_pktlen = 0;
2178 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
2179 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
2180 UBS_PKTBUF_LEN);
2181 mcr->mcr_opktbuf.pb_next = 0;
2182
2183 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
2184 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1);
2185 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1;
2186
2187 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
2188 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2189
2190 if (sc->sc_flags & UBS_FLAGS_RNG4) {
2191 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next);
2192 ubsec_feed4(sc);
2193 } else {
2194 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
2195 ubsec_feed2(sc);
2196 }
2197 rng->rng_used = 1;
2198 ubsecstats.hst_rng++;
2199
2200 return;
2201
2202 out:
2203 /*
2204 * Something weird happened, generate our own call back.
2205 */
2206 (*nqueue)--;
2207 #ifdef __OpenBSD__
2208 timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2209 #else
2210 callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2211 #endif
2212 }
2213 #endif /* UBSEC_NO_RNG */
2214
2215 static int
2216 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size,
2217 struct ubsec_dma_alloc *dma,int mapflags)
2218 {
2219 int r;
2220
2221 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
2222 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
2223 goto fail_0;
2224
2225 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
2226 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
2227 goto fail_1;
2228
2229 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2230 BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
2231 goto fail_2;
2232
2233 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
2234 size, NULL, BUS_DMA_NOWAIT)) != 0)
2235 goto fail_3;
2236
2237 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
2238 dma->dma_size = size;
2239 return (0);
2240
2241 fail_3:
2242 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2243 fail_2:
2244 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
2245 fail_1:
2246 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2247 fail_0:
2248 dma->dma_map = NULL;
2249 return (r);
2250 }
2251
2252 static void
2253 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
2254 {
2255 bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
2256 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
2257 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2258 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2259 }
2260
2261 /*
2262 * Resets the board. Values in the regesters are left as is
2263 * from the reset (i.e. initial values are assigned elsewhere).
2264 */
2265 static void
2266 ubsec_reset_board(struct ubsec_softc *sc)
2267 {
2268 volatile u_int32_t ctrl;
2269
2270 ctrl = READ_REG(sc, BS_CTRL);
2271 ctrl |= BS_CTRL_RESET;
2272 WRITE_REG(sc, BS_CTRL, ctrl);
2273
2274 /*
2275 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
2276 */
2277 DELAY(10);
2278
2279 /* Enable RNG and interrupts on newer devices */
2280 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2281 #ifndef UBSEC_NO_RNG
2282 WRITE_REG(sc, BS_CFG, BS_CFG_RNG);
2283 #endif
2284 WRITE_REG(sc, BS_INT, BS_INT_DMAINT);
2285 }
2286 }
2287
2288 /*
2289 * Init Broadcom registers
2290 */
2291 static void
2292 ubsec_init_board(struct ubsec_softc *sc)
2293 {
2294 u_int32_t ctrl;
2295
2296 ctrl = READ_REG(sc, BS_CTRL);
2297 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
2298 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
2299
2300 /*
2301 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)).
2302 * anyone got hw docs?
2303 */
2304 if (sc->sc_flags & UBS_FLAGS_KEY)
2305 ctrl |= BS_CTRL_MCR2INT;
2306 else
2307 ctrl &= ~BS_CTRL_MCR2INT;
2308
2309 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2310 ctrl &= ~BS_CTRL_SWNORM;
2311
2312 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2313 ctrl |= BS_CTRL_BSIZE240;
2314 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */
2315
2316 if (sc->sc_flags & UBS_FLAGS_RNG4)
2317 ctrl |= BS_CTRL_MCR4INT;
2318 else
2319 ctrl &= ~BS_CTRL_MCR4INT;
2320 }
2321
2322 WRITE_REG(sc, BS_CTRL, ctrl);
2323 }
2324
2325 /*
2326 * Init Broadcom PCI registers
2327 */
2328 static void
2329 ubsec_init_pciregs(struct pci_attach_args *pa)
2330 {
2331 pci_chipset_tag_t pc = pa->pa_pc;
2332 u_int32_t misc;
2333
2334 /*
2335 * This will set the cache line size to 1, this will
2336 * force the BCM58xx chip just to do burst read/writes.
2337 * Cache line read/writes are to slow
2338 */
2339 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG);
2340 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT))
2341 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT);
2342 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc);
2343 }
2344
2345 /*
2346 * Clean up after a chip crash.
2347 * It is assumed that the caller in splnet()
2348 */
2349 static void
2350 ubsec_cleanchip(struct ubsec_softc *sc)
2351 {
2352 struct ubsec_q *q;
2353
2354 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2355 q = SIMPLEQ_FIRST(&sc->sc_qchip);
2356 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next);
2357 ubsec_free_q(sc, q);
2358 }
2359 sc->sc_nqchip = 0;
2360 }
2361
2362 /*
2363 * free a ubsec_q
2364 * It is assumed that the caller is within splnet()
2365 */
2366 static int
2367 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2368 {
2369 struct ubsec_q *q2;
2370 struct cryptop *crp;
2371 int npkts;
2372 int i;
2373
2374 npkts = q->q_nstacked_mcrs;
2375
2376 for (i = 0; i < npkts; i++) {
2377 if(q->q_stacked_mcr[i]) {
2378 q2 = q->q_stacked_mcr[i];
2379
2380 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2381 m_freem(q2->q_dst_m);
2382
2383 crp = (struct cryptop *)q2->q_crp;
2384
2385 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2386
2387 crp->crp_etype = EFAULT;
2388 crypto_done(crp);
2389 } else {
2390 break;
2391 }
2392 }
2393
2394 /*
2395 * Free header MCR
2396 */
2397 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2398 m_freem(q->q_dst_m);
2399
2400 crp = (struct cryptop *)q->q_crp;
2401
2402 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2403
2404 crp->crp_etype = EFAULT;
2405 crypto_done(crp);
2406 return(0);
2407 }
2408
2409 /*
2410 * Routine to reset the chip and clean up.
2411 * It is assumed that the caller is in splnet()
2412 */
2413 static void
2414 ubsec_totalreset(struct ubsec_softc *sc)
2415 {
2416 ubsec_reset_board(sc);
2417 ubsec_init_board(sc);
2418 ubsec_cleanchip(sc);
2419 }
2420
2421 static int
2422 ubsec_dmamap_aligned(bus_dmamap_t map)
2423 {
2424 int i;
2425
2426 for (i = 0; i < map->dm_nsegs; i++) {
2427 if (map->dm_segs[i].ds_addr & 3)
2428 return (0);
2429 if ((i != (map->dm_nsegs - 1)) &&
2430 (map->dm_segs[i].ds_len & 3))
2431 return (0);
2432 }
2433 return (1);
2434 }
2435
2436 #ifdef __OpenBSD__
2437 struct ubsec_softc *
2438 ubsec_kfind(struct cryptkop *krp)
2439 {
2440 struct ubsec_softc *sc;
2441 int i;
2442
2443 for (i = 0; i < ubsec_cd.cd_ndevs; i++) {
2444 sc = ubsec_cd.cd_devs[i];
2445 if (sc == NULL)
2446 continue;
2447 if (sc->sc_cid == krp->krp_hid)
2448 return (sc);
2449 }
2450 return (NULL);
2451 }
2452 #endif
2453
2454 static void
2455 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2456 {
2457 switch (q->q_type) {
2458 case UBS_CTXOP_MODEXP: {
2459 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2460
2461 ubsec_dma_free(sc, &me->me_q.q_mcr);
2462 ubsec_dma_free(sc, &me->me_q.q_ctx);
2463 ubsec_dma_free(sc, &me->me_M);
2464 ubsec_dma_free(sc, &me->me_E);
2465 ubsec_dma_free(sc, &me->me_C);
2466 ubsec_dma_free(sc, &me->me_epb);
2467 free(me, M_DEVBUF);
2468 break;
2469 }
2470 case UBS_CTXOP_RSAPRIV: {
2471 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2472
2473 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2474 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2475 ubsec_dma_free(sc, &rp->rpr_msgin);
2476 ubsec_dma_free(sc, &rp->rpr_msgout);
2477 free(rp, M_DEVBUF);
2478 break;
2479 }
2480 default:
2481 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev),
2482 q->q_type);
2483 break;
2484 }
2485 }
2486
2487 static int
2488 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2489 {
2490 struct ubsec_softc *sc;
2491 int r;
2492
2493 if (krp == NULL || krp->krp_callback == NULL)
2494 return (EINVAL);
2495 #ifdef __OpenBSD__
2496 if ((sc = ubsec_kfind(krp)) == NULL)
2497 return (EINVAL);
2498 #else
2499 sc = arg;
2500 KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/);
2501 #endif
2502
2503 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2504 struct ubsec_q2 *q;
2505
2506 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2507 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next);
2508 ubsec_kfree(sc, q);
2509 }
2510
2511 switch (krp->krp_op) {
2512 case CRK_MOD_EXP:
2513 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2514 r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2515 else
2516 r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2517 break;
2518 case CRK_MOD_EXP_CRT:
2519 r = ubsec_kprocess_rsapriv(sc, krp, hint);
2520 break;
2521 default:
2522 printf("%s: kprocess: invalid op 0x%x\n",
2523 device_xname(sc->sc_dev), krp->krp_op);
2524 krp->krp_status = EOPNOTSUPP;
2525 crypto_kdone(krp);
2526 r = 0;
2527 }
2528 return (r);
2529 }
2530
2531 /*
2532 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2533 */
2534 static int
2535 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp,
2536 int hint)
2537 {
2538 struct ubsec_q2_modexp *me;
2539 struct ubsec_mcr *mcr;
2540 struct ubsec_ctx_modexp *ctx;
2541 struct ubsec_pktbuf *epb;
2542 int err = 0;
2543 u_int nbits, normbits, mbits, shiftbits, ebits;
2544
2545 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2546 if (me == NULL) {
2547 err = ENOMEM;
2548 goto errout;
2549 }
2550 memset(me, 0, sizeof *me);
2551 me->me_krp = krp;
2552 me->me_q.q_type = UBS_CTXOP_MODEXP;
2553
2554 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2555 if (nbits <= 512)
2556 normbits = 512;
2557 else if (nbits <= 768)
2558 normbits = 768;
2559 else if (nbits <= 1024)
2560 normbits = 1024;
2561 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2562 normbits = 1536;
2563 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2564 normbits = 2048;
2565 else {
2566 err = E2BIG;
2567 goto errout;
2568 }
2569
2570 shiftbits = normbits - nbits;
2571
2572 me->me_modbits = nbits;
2573 me->me_shiftbits = shiftbits;
2574 me->me_normbits = normbits;
2575
2576 /* Sanity check: result bits must be >= true modulus bits. */
2577 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2578 err = ERANGE;
2579 goto errout;
2580 }
2581
2582 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2583 &me->me_q.q_mcr, 0)) {
2584 err = ENOMEM;
2585 goto errout;
2586 }
2587 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2588
2589 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2590 &me->me_q.q_ctx, 0)) {
2591 err = ENOMEM;
2592 goto errout;
2593 }
2594
2595 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2596 if (mbits > nbits) {
2597 err = E2BIG;
2598 goto errout;
2599 }
2600 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2601 err = ENOMEM;
2602 goto errout;
2603 }
2604 ubsec_kshift_r(shiftbits,
2605 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2606 me->me_M.dma_vaddr, normbits);
2607
2608 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2609 err = ENOMEM;
2610 goto errout;
2611 }
2612 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2613
2614 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2615 if (ebits > nbits) {
2616 err = E2BIG;
2617 goto errout;
2618 }
2619 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2620 err = ENOMEM;
2621 goto errout;
2622 }
2623 ubsec_kshift_r(shiftbits,
2624 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2625 me->me_E.dma_vaddr, normbits);
2626
2627 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2628 &me->me_epb, 0)) {
2629 err = ENOMEM;
2630 goto errout;
2631 }
2632 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2633 epb->pb_addr = htole32(me->me_E.dma_paddr);
2634 epb->pb_next = 0;
2635 epb->pb_len = htole32(normbits / 8);
2636
2637 #ifdef UBSEC_DEBUG
2638 if (ubsec_debug) {
2639 printf("Epb ");
2640 ubsec_dump_pb(epb);
2641 }
2642 #endif
2643
2644 mcr->mcr_pkts = htole16(1);
2645 mcr->mcr_flags = 0;
2646 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2647 mcr->mcr_reserved = 0;
2648 mcr->mcr_pktlen = 0;
2649
2650 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2651 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2652 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2653
2654 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2655 mcr->mcr_opktbuf.pb_next = 0;
2656 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2657
2658 #ifdef DIAGNOSTIC
2659 /* Misaligned output buffer will hang the chip. */
2660 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2661 panic("%s: modexp invalid addr 0x%x",
2662 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr));
2663 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2664 panic("%s: modexp invalid len 0x%x",
2665 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len));
2666 #endif
2667
2668 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2669 memset(ctx, 0, sizeof(*ctx));
2670 ubsec_kshift_r(shiftbits,
2671 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2672 ctx->me_N, normbits);
2673 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2674 ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2675 ctx->me_E_len = htole16(nbits);
2676 ctx->me_N_len = htole16(nbits);
2677
2678 #ifdef UBSEC_DEBUG
2679 if (ubsec_debug) {
2680 ubsec_dump_mcr(mcr);
2681 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2682 }
2683 #endif
2684
2685 /*
2686 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2687 * everything else.
2688 */
2689 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2690 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2691 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2692 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2693 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2694 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2695 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2696 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2697
2698 /* Enqueue and we're done... */
2699 mutex_spin_enter(&sc->sc_mtx);
2700 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2701 ubsec_feed2(sc);
2702 ubsecstats.hst_modexp++;
2703 mutex_spin_exit(&sc->sc_mtx);
2704
2705 return (0);
2706
2707 errout:
2708 if (me != NULL) {
2709 if (me->me_q.q_mcr.dma_map != NULL)
2710 ubsec_dma_free(sc, &me->me_q.q_mcr);
2711 if (me->me_q.q_ctx.dma_map != NULL) {
2712 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2713 ubsec_dma_free(sc, &me->me_q.q_ctx);
2714 }
2715 if (me->me_M.dma_map != NULL) {
2716 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2717 ubsec_dma_free(sc, &me->me_M);
2718 }
2719 if (me->me_E.dma_map != NULL) {
2720 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2721 ubsec_dma_free(sc, &me->me_E);
2722 }
2723 if (me->me_C.dma_map != NULL) {
2724 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2725 ubsec_dma_free(sc, &me->me_C);
2726 }
2727 if (me->me_epb.dma_map != NULL)
2728 ubsec_dma_free(sc, &me->me_epb);
2729 free(me, M_DEVBUF);
2730 }
2731 krp->krp_status = err;
2732 crypto_kdone(krp);
2733 return (0);
2734 }
2735
2736 /*
2737 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2738 */
2739 static int
2740 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp,
2741 int hint)
2742 {
2743 struct ubsec_q2_modexp *me;
2744 struct ubsec_mcr *mcr;
2745 struct ubsec_ctx_modexp *ctx;
2746 struct ubsec_pktbuf *epb;
2747 int err = 0;
2748 u_int nbits, normbits, mbits, shiftbits, ebits;
2749
2750 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2751 if (me == NULL) {
2752 err = ENOMEM;
2753 goto errout;
2754 }
2755 memset(me, 0, sizeof *me);
2756 me->me_krp = krp;
2757 me->me_q.q_type = UBS_CTXOP_MODEXP;
2758
2759 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2760 if (nbits <= 512)
2761 normbits = 512;
2762 else if (nbits <= 768)
2763 normbits = 768;
2764 else if (nbits <= 1024)
2765 normbits = 1024;
2766 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2767 normbits = 1536;
2768 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2769 normbits = 2048;
2770 else {
2771 err = E2BIG;
2772 goto errout;
2773 }
2774
2775 shiftbits = normbits - nbits;
2776
2777 /* XXX ??? */
2778 me->me_modbits = nbits;
2779 me->me_shiftbits = shiftbits;
2780 me->me_normbits = normbits;
2781
2782 /* Sanity check: result bits must be >= true modulus bits. */
2783 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2784 err = ERANGE;
2785 goto errout;
2786 }
2787
2788 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2789 &me->me_q.q_mcr, 0)) {
2790 err = ENOMEM;
2791 goto errout;
2792 }
2793 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2794
2795 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2796 &me->me_q.q_ctx, 0)) {
2797 err = ENOMEM;
2798 goto errout;
2799 }
2800
2801 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2802 if (mbits > nbits) {
2803 err = E2BIG;
2804 goto errout;
2805 }
2806 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2807 err = ENOMEM;
2808 goto errout;
2809 }
2810 memset(me->me_M.dma_vaddr, 0, normbits / 8);
2811 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2812 me->me_M.dma_vaddr, (mbits + 7) / 8);
2813
2814 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2815 err = ENOMEM;
2816 goto errout;
2817 }
2818 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2819
2820 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2821 if (ebits > nbits) {
2822 err = E2BIG;
2823 goto errout;
2824 }
2825 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2826 err = ENOMEM;
2827 goto errout;
2828 }
2829 memset(me->me_E.dma_vaddr, 0, normbits / 8);
2830 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2831 me->me_E.dma_vaddr, (ebits + 7) / 8);
2832
2833 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2834 &me->me_epb, 0)) {
2835 err = ENOMEM;
2836 goto errout;
2837 }
2838 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2839 epb->pb_addr = htole32(me->me_E.dma_paddr);
2840 epb->pb_next = 0;
2841 epb->pb_len = htole32((ebits + 7) / 8);
2842
2843 #ifdef UBSEC_DEBUG
2844 if (ubsec_debug) {
2845 printf("Epb ");
2846 ubsec_dump_pb(epb);
2847 }
2848 #endif
2849
2850 mcr->mcr_pkts = htole16(1);
2851 mcr->mcr_flags = 0;
2852 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2853 mcr->mcr_reserved = 0;
2854 mcr->mcr_pktlen = 0;
2855
2856 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2857 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2858 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2859
2860 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2861 mcr->mcr_opktbuf.pb_next = 0;
2862 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2863
2864 #ifdef DIAGNOSTIC
2865 /* Misaligned output buffer will hang the chip. */
2866 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2867 panic("%s: modexp invalid addr 0x%x",
2868 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr));
2869 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2870 panic("%s: modexp invalid len 0x%x",
2871 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len));
2872 #endif
2873
2874 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2875 memset(ctx, 0, sizeof(*ctx));
2876 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p,
2877 (nbits + 7) / 8);
2878 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2879 ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2880 ctx->me_E_len = htole16(ebits);
2881 ctx->me_N_len = htole16(nbits);
2882
2883 #ifdef UBSEC_DEBUG
2884 if (ubsec_debug) {
2885 ubsec_dump_mcr(mcr);
2886 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2887 }
2888 #endif
2889
2890 /*
2891 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2892 * everything else.
2893 */
2894 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2895 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2896 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2897 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2898 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2899 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2900 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2901 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2902
2903 /* Enqueue and we're done... */
2904 mutex_spin_enter(&sc->sc_mtx);
2905 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2906 ubsec_feed2(sc);
2907 mutex_spin_exit(&sc->sc_mtx);
2908
2909 return (0);
2910
2911 errout:
2912 if (me != NULL) {
2913 if (me->me_q.q_mcr.dma_map != NULL)
2914 ubsec_dma_free(sc, &me->me_q.q_mcr);
2915 if (me->me_q.q_ctx.dma_map != NULL) {
2916 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2917 ubsec_dma_free(sc, &me->me_q.q_ctx);
2918 }
2919 if (me->me_M.dma_map != NULL) {
2920 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2921 ubsec_dma_free(sc, &me->me_M);
2922 }
2923 if (me->me_E.dma_map != NULL) {
2924 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2925 ubsec_dma_free(sc, &me->me_E);
2926 }
2927 if (me->me_C.dma_map != NULL) {
2928 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2929 ubsec_dma_free(sc, &me->me_C);
2930 }
2931 if (me->me_epb.dma_map != NULL)
2932 ubsec_dma_free(sc, &me->me_epb);
2933 free(me, M_DEVBUF);
2934 }
2935 krp->krp_status = err;
2936 crypto_kdone(krp);
2937 return (0);
2938 }
2939
2940 static int
2941 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp,
2942 int hint)
2943 {
2944 struct ubsec_q2_rsapriv *rp = NULL;
2945 struct ubsec_mcr *mcr;
2946 struct ubsec_ctx_rsapriv *ctx;
2947 int err = 0;
2948 u_int padlen, msglen;
2949
2950 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2951 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2952 if (msglen > padlen)
2953 padlen = msglen;
2954
2955 if (padlen <= 256)
2956 padlen = 256;
2957 else if (padlen <= 384)
2958 padlen = 384;
2959 else if (padlen <= 512)
2960 padlen = 512;
2961 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2962 padlen = 768;
2963 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2964 padlen = 1024;
2965 else {
2966 err = E2BIG;
2967 goto errout;
2968 }
2969
2970 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2971 err = E2BIG;
2972 goto errout;
2973 }
2974
2975 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2976 err = E2BIG;
2977 goto errout;
2978 }
2979
2980 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2981 err = E2BIG;
2982 goto errout;
2983 }
2984
2985 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO);
2986 if (rp == NULL)
2987 return (ENOMEM);
2988 rp->rpr_krp = krp;
2989 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2990
2991 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2992 &rp->rpr_q.q_mcr, 0)) {
2993 err = ENOMEM;
2994 goto errout;
2995 }
2996 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2997
2998 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2999 &rp->rpr_q.q_ctx, 0)) {
3000 err = ENOMEM;
3001 goto errout;
3002 }
3003 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
3004 memset(ctx, 0, sizeof *ctx);
3005
3006 /* Copy in p */
3007 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
3008 &ctx->rpr_buf[0 * (padlen / 8)],
3009 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
3010
3011 /* Copy in q */
3012 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
3013 &ctx->rpr_buf[1 * (padlen / 8)],
3014 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
3015
3016 /* Copy in dp */
3017 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
3018 &ctx->rpr_buf[2 * (padlen / 8)],
3019 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
3020
3021 /* Copy in dq */
3022 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
3023 &ctx->rpr_buf[3 * (padlen / 8)],
3024 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
3025
3026 /* Copy in pinv */
3027 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
3028 &ctx->rpr_buf[4 * (padlen / 8)],
3029 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
3030
3031 msglen = padlen * 2;
3032
3033 /* Copy in input message (aligned buffer/length). */
3034 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
3035 /* Is this likely? */
3036 err = E2BIG;
3037 goto errout;
3038 }
3039 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
3040 err = ENOMEM;
3041 goto errout;
3042 }
3043 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8);
3044 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
3045 rp->rpr_msgin.dma_vaddr,
3046 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
3047
3048 /* Prepare space for output message (aligned buffer/length). */
3049 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
3050 /* Is this likely? */
3051 err = E2BIG;
3052 goto errout;
3053 }
3054 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
3055 err = ENOMEM;
3056 goto errout;
3057 }
3058 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8);
3059
3060 mcr->mcr_pkts = htole16(1);
3061 mcr->mcr_flags = 0;
3062 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
3063 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
3064 mcr->mcr_ipktbuf.pb_next = 0;
3065 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
3066 mcr->mcr_reserved = 0;
3067 mcr->mcr_pktlen = htole16(msglen);
3068 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
3069 mcr->mcr_opktbuf.pb_next = 0;
3070 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
3071
3072 #ifdef DIAGNOSTIC
3073 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
3074 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)",
3075 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr,
3076 (u_long) rp->rpr_msgin.dma_size);
3077 }
3078 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
3079 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)",
3080 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr,
3081 (u_long) rp->rpr_msgout.dma_size);
3082 }
3083 #endif
3084
3085 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
3086 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
3087 ctx->rpr_q_len = htole16(padlen);
3088 ctx->rpr_p_len = htole16(padlen);
3089
3090 /*
3091 * ubsec_feed2 will sync mcr and ctx, we just need to sync
3092 * everything else.
3093 */
3094 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map,
3095 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
3096 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map,
3097 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
3098
3099 /* Enqueue and we're done... */
3100 mutex_spin_enter(&sc->sc_mtx);
3101 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
3102 ubsec_feed2(sc);
3103 ubsecstats.hst_modexpcrt++;
3104 mutex_spin_exit(&sc->sc_mtx);
3105 return (0);
3106
3107 errout:
3108 if (rp != NULL) {
3109 if (rp->rpr_q.q_mcr.dma_map != NULL)
3110 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
3111 if (rp->rpr_msgin.dma_map != NULL) {
3112 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size);
3113 ubsec_dma_free(sc, &rp->rpr_msgin);
3114 }
3115 if (rp->rpr_msgout.dma_map != NULL) {
3116 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size);
3117 ubsec_dma_free(sc, &rp->rpr_msgout);
3118 }
3119 free(rp, M_DEVBUF);
3120 }
3121 krp->krp_status = err;
3122 crypto_kdone(krp);
3123 return (0);
3124 }
3125
3126 #ifdef UBSEC_DEBUG
3127 static void
3128 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
3129 {
3130 printf("addr 0x%x (0x%x) next 0x%x\n",
3131 pb->pb_addr, pb->pb_len, pb->pb_next);
3132 }
3133
3134 static void
3135 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c)
3136 {
3137 printf("CTX (0x%x):\n", c->ctx_len);
3138 switch (letoh16(c->ctx_op)) {
3139 case UBS_CTXOP_RNGBYPASS:
3140 case UBS_CTXOP_RNGSHA1:
3141 break;
3142 case UBS_CTXOP_MODEXP:
3143 {
3144 struct ubsec_ctx_modexp *cx = (void *)c;
3145 int i, len;
3146
3147 printf(" Elen %u, Nlen %u\n",
3148 letoh16(cx->me_E_len), letoh16(cx->me_N_len));
3149 len = (cx->me_N_len + 7)/8;
3150 for (i = 0; i < len; i++)
3151 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
3152 printf("\n");
3153 break;
3154 }
3155 default:
3156 printf("unknown context: %x\n", c->ctx_op);
3157 }
3158 printf("END CTX\n");
3159 }
3160
3161 static void
3162 ubsec_dump_mcr(struct ubsec_mcr *mcr)
3163 {
3164 volatile struct ubsec_mcr_add *ma;
3165 int i;
3166
3167 printf("MCR:\n");
3168 printf(" pkts: %u, flags 0x%x\n",
3169 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
3170 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
3171 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
3172 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
3173 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
3174 letoh16(ma->mcr_reserved));
3175 printf(" %d: ipkt ", i);
3176 ubsec_dump_pb(&ma->mcr_ipktbuf);
3177 printf(" %d: opkt ", i);
3178 ubsec_dump_pb(&ma->mcr_opktbuf);
3179 ma++;
3180 }
3181 printf("END MCR\n");
3182 }
3183 #endif /* UBSEC_DEBUG */
3184
3185 /*
3186 * Return the number of significant bits of a big number.
3187 */
3188 static int
3189 ubsec_ksigbits(struct crparam *cr)
3190 {
3191 u_int plen = (cr->crp_nbits + 7) / 8;
3192 int i, sig = plen * 8;
3193 u_int8_t c, *p = cr->crp_p;
3194
3195 for (i = plen - 1; i >= 0; i--) {
3196 c = p[i];
3197 if (c != 0) {
3198 while ((c & 0x80) == 0) {
3199 sig--;
3200 c <<= 1;
3201 }
3202 break;
3203 }
3204 sig -= 8;
3205 }
3206 return (sig);
3207 }
3208
3209 static void
3210 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits,
3211 u_int8_t *dst, u_int dstbits)
3212 {
3213 u_int slen, dlen;
3214 int i, si, di, n;
3215
3216 slen = (srcbits + 7) / 8;
3217 dlen = (dstbits + 7) / 8;
3218
3219 for (i = 0; i < slen; i++)
3220 dst[i] = src[i];
3221 for (i = 0; i < dlen - slen; i++)
3222 dst[slen + i] = 0;
3223
3224 n = shiftbits / 8;
3225 if (n != 0) {
3226 si = dlen - n - 1;
3227 di = dlen - 1;
3228 while (si >= 0)
3229 dst[di--] = dst[si--];
3230 while (di >= 0)
3231 dst[di--] = 0;
3232 }
3233
3234 n = shiftbits % 8;
3235 if (n != 0) {
3236 for (i = dlen - 1; i > 0; i--)
3237 dst[i] = (dst[i] << n) |
3238 (dst[i - 1] >> (8 - n));
3239 dst[0] = dst[0] << n;
3240 }
3241 }
3242
3243 static void
3244 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits,
3245 u_int8_t *dst, u_int dstbits)
3246 {
3247 int slen, dlen, i, n;
3248
3249 slen = (srcbits + 7) / 8;
3250 dlen = (dstbits + 7) / 8;
3251
3252 n = shiftbits / 8;
3253 for (i = 0; i < slen; i++)
3254 dst[i] = src[i + n];
3255 for (i = 0; i < dlen - slen; i++)
3256 dst[slen + i] = 0;
3257
3258 n = shiftbits % 8;
3259 if (n != 0) {
3260 for (i = 0; i < (dlen - 1); i++)
3261 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
3262 dst[dlen - 1] = dst[dlen - 1] >> n;
3263 }
3264 }
3265