ubsec.c revision 1.40 1 1.40 bad /* $NetBSD: ubsec.c,v 1.40 2014/04/19 12:29:24 bad Exp $ */
2 1.1 jonathan /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */
3 1.40 bad /* $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ */
4 1.1 jonathan
5 1.1 jonathan /*
6 1.1 jonathan * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net)
7 1.1 jonathan * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org)
8 1.1 jonathan * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com)
9 1.5 perry *
10 1.1 jonathan * Redistribution and use in source and binary forms, with or without
11 1.1 jonathan * modification, are permitted provided that the following conditions
12 1.1 jonathan * are met:
13 1.1 jonathan * 1. Redistributions of source code must retain the above copyright
14 1.1 jonathan * notice, this list of conditions and the following disclaimer.
15 1.1 jonathan * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 jonathan * notice, this list of conditions and the following disclaimer in the
17 1.1 jonathan * documentation and/or other materials provided with the distribution.
18 1.1 jonathan *
19 1.1 jonathan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 1.1 jonathan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 1.1 jonathan * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 1.1 jonathan * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
23 1.1 jonathan * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 1.1 jonathan * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 1.1 jonathan * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 1.1 jonathan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27 1.1 jonathan * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 1.1 jonathan * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 jonathan * POSSIBILITY OF SUCH DAMAGE.
30 1.1 jonathan *
31 1.1 jonathan * Effort sponsored in part by the Defense Advanced Research Projects
32 1.1 jonathan * Agency (DARPA) and Air Force Research Laboratory, Air Force
33 1.1 jonathan * Materiel Command, USAF, under agreement number F30602-01-2-0537.
34 1.1 jonathan *
35 1.1 jonathan */
36 1.1 jonathan
37 1.14 lukem #include <sys/cdefs.h>
38 1.40 bad __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.40 2014/04/19 12:29:24 bad Exp $");
39 1.14 lukem
40 1.1 jonathan #undef UBSEC_DEBUG
41 1.1 jonathan
42 1.1 jonathan /*
43 1.40 bad * uBsec 5[56]01, 58xx hardware crypto accelerator
44 1.1 jonathan */
45 1.1 jonathan
46 1.1 jonathan #include <sys/param.h>
47 1.1 jonathan #include <sys/systm.h>
48 1.1 jonathan #include <sys/proc.h>
49 1.1 jonathan #include <sys/endian.h>
50 1.1 jonathan #ifdef __NetBSD__
51 1.29 tls #define UBSEC_NO_RNG /* hangs on attach */
52 1.1 jonathan #define letoh16 htole16
53 1.1 jonathan #define letoh32 htole32
54 1.1 jonathan #endif
55 1.1 jonathan #include <sys/errno.h>
56 1.1 jonathan #include <sys/malloc.h>
57 1.1 jonathan #include <sys/kernel.h>
58 1.1 jonathan #include <sys/mbuf.h>
59 1.1 jonathan #include <sys/device.h>
60 1.32 bad #include <sys/module.h>
61 1.1 jonathan #include <sys/queue.h>
62 1.33 bad #include <sys/sysctl.h>
63 1.1 jonathan
64 1.1 jonathan #include <opencrypto/cryptodev.h>
65 1.8 thorpej #include <opencrypto/xform.h>
66 1.1 jonathan #ifdef __OpenBSD__
67 1.1 jonathan #include <dev/rndvar.h>
68 1.1 jonathan #include <sys/md5k.h>
69 1.1 jonathan #else
70 1.26 tls #include <sys/cprng.h>
71 1.1 jonathan #include <sys/md5.h>
72 1.1 jonathan #endif
73 1.1 jonathan #include <sys/sha1.h>
74 1.1 jonathan
75 1.1 jonathan #include <dev/pci/pcireg.h>
76 1.1 jonathan #include <dev/pci/pcivar.h>
77 1.1 jonathan #include <dev/pci/pcidevs.h>
78 1.1 jonathan
79 1.1 jonathan #include <dev/pci/ubsecreg.h>
80 1.1 jonathan #include <dev/pci/ubsecvar.h>
81 1.1 jonathan
82 1.1 jonathan /*
83 1.1 jonathan * Prototypes and count for the pci_device structure
84 1.1 jonathan */
85 1.22 cegger static int ubsec_probe(device_t, cfdata_t, void *);
86 1.22 cegger static void ubsec_attach(device_t, device_t, void *);
87 1.32 bad static int ubsec_detach(device_t, int);
88 1.33 bad static int ubsec_sysctl_init(void);
89 1.1 jonathan static void ubsec_reset_board(struct ubsec_softc *);
90 1.1 jonathan static void ubsec_init_board(struct ubsec_softc *);
91 1.1 jonathan static void ubsec_init_pciregs(struct pci_attach_args *pa);
92 1.1 jonathan static void ubsec_cleanchip(struct ubsec_softc *);
93 1.1 jonathan static void ubsec_totalreset(struct ubsec_softc *);
94 1.1 jonathan static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *);
95 1.1 jonathan
96 1.1 jonathan #ifdef __OpenBSD__
97 1.1 jonathan struct cfattach ubsec_ca = {
98 1.1 jonathan sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
99 1.1 jonathan };
100 1.1 jonathan
101 1.1 jonathan struct cfdriver ubsec_cd = {
102 1.1 jonathan 0, "ubsec", DV_DULL
103 1.1 jonathan };
104 1.1 jonathan #else
105 1.28 chs CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach,
106 1.32 bad ubsec_detach, NULL);
107 1.1 jonathan extern struct cfdriver ubsec_cd;
108 1.1 jonathan #endif
109 1.1 jonathan
110 1.1 jonathan /* patchable */
111 1.1 jonathan #ifdef UBSEC_DEBUG
112 1.1 jonathan extern int ubsec_debug;
113 1.1 jonathan int ubsec_debug=1;
114 1.1 jonathan #endif
115 1.1 jonathan
116 1.1 jonathan static int ubsec_intr(void *);
117 1.1 jonathan static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *);
118 1.1 jonathan static int ubsec_freesession(void*, u_int64_t);
119 1.1 jonathan static int ubsec_process(void*, struct cryptop *, int hint);
120 1.1 jonathan static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
121 1.1 jonathan static void ubsec_feed(struct ubsec_softc *);
122 1.1 jonathan static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
123 1.1 jonathan static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
124 1.1 jonathan static void ubsec_feed2(struct ubsec_softc *);
125 1.34 bad static void ubsec_feed4(struct ubsec_softc *);
126 1.1 jonathan #ifndef UBSEC_NO_RNG
127 1.29 tls static void ubsec_rng(void *);
128 1.29 tls static void ubsec_rng_locked(void *);
129 1.29 tls static void ubsec_rng_get(size_t, void *);
130 1.1 jonathan #endif /* UBSEC_NO_RNG */
131 1.1 jonathan static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
132 1.1 jonathan struct ubsec_dma_alloc *, int);
133 1.1 jonathan static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
134 1.1 jonathan static int ubsec_dmamap_aligned(bus_dmamap_t);
135 1.1 jonathan
136 1.1 jonathan static int ubsec_kprocess(void*, struct cryptkop *, int);
137 1.1 jonathan static int ubsec_kprocess_modexp_sw(struct ubsec_softc *,
138 1.1 jonathan struct cryptkop *, int);
139 1.1 jonathan static int ubsec_kprocess_modexp_hw(struct ubsec_softc *,
140 1.1 jonathan struct cryptkop *, int);
141 1.1 jonathan static int ubsec_kprocess_rsapriv(struct ubsec_softc *,
142 1.1 jonathan struct cryptkop *, int);
143 1.1 jonathan static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
144 1.1 jonathan static int ubsec_ksigbits(struct crparam *);
145 1.1 jonathan static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
146 1.1 jonathan static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
147 1.1 jonathan
148 1.1 jonathan #ifdef UBSEC_DEBUG
149 1.1 jonathan static void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
150 1.1 jonathan static void ubsec_dump_mcr(struct ubsec_mcr *);
151 1.1 jonathan static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *);
152 1.1 jonathan #endif
153 1.1 jonathan
154 1.1 jonathan #define READ_REG(sc,r) \
155 1.1 jonathan bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
156 1.1 jonathan
157 1.1 jonathan #define WRITE_REG(sc,reg,val) \
158 1.1 jonathan bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
159 1.1 jonathan
160 1.1 jonathan #define SWAP32(x) (x) = htole32(ntohl((x)))
161 1.1 jonathan #ifndef HTOLE32
162 1.1 jonathan #define HTOLE32(x) (x) = htole32(x)
163 1.1 jonathan #endif
164 1.1 jonathan
165 1.1 jonathan struct ubsec_stats ubsecstats;
166 1.1 jonathan
167 1.33 bad static struct sysctllog *ubsec_sysctllog;
168 1.33 bad
169 1.1 jonathan /*
170 1.5 perry * ubsec_maxbatch controls the number of crypto ops to voluntarily
171 1.1 jonathan * collect into one submission to the hardware. This batching happens
172 1.1 jonathan * when ops are dispatched from the crypto subsystem with a hint that
173 1.1 jonathan * more are to follow immediately. These ops must also not be marked
174 1.1 jonathan * with a ``no delay'' flag.
175 1.1 jonathan */
176 1.1 jonathan static int ubsec_maxbatch = 1;
177 1.1 jonathan
178 1.1 jonathan /*
179 1.1 jonathan * ubsec_maxaggr controls the number of crypto ops to submit to the
180 1.1 jonathan * hardware as a unit. This aggregation reduces the number of interrupts
181 1.1 jonathan * to the host at the expense of increased latency (for all but the last
182 1.1 jonathan * operation). For network traffic setting this to one yields the highest
183 1.1 jonathan * performance but at the expense of more interrupt processing.
184 1.1 jonathan */
185 1.1 jonathan static int ubsec_maxaggr = 1;
186 1.1 jonathan
187 1.4 thorpej static const struct ubsec_product {
188 1.4 thorpej pci_vendor_id_t ubsec_vendor;
189 1.4 thorpej pci_product_id_t ubsec_product;
190 1.4 thorpej int ubsec_flags;
191 1.4 thorpej int ubsec_statmask;
192 1.34 bad int ubsec_maxaggr;
193 1.4 thorpej const char *ubsec_name;
194 1.4 thorpej } ubsec_products[] = {
195 1.4 thorpej { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501,
196 1.4 thorpej 0,
197 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
198 1.34 bad UBS_MIN_AGGR,
199 1.4 thorpej "Bluesteel 5501"
200 1.4 thorpej },
201 1.4 thorpej { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601,
202 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG,
203 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
204 1.34 bad UBS_MIN_AGGR,
205 1.4 thorpej "Bluesteel 5601"
206 1.4 thorpej },
207 1.4 thorpej
208 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801,
209 1.4 thorpej 0,
210 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
211 1.34 bad UBS_MIN_AGGR,
212 1.4 thorpej "Broadcom BCM5801"
213 1.4 thorpej },
214 1.4 thorpej
215 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802,
216 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG,
217 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
218 1.34 bad UBS_MIN_AGGR,
219 1.4 thorpej "Broadcom BCM5802"
220 1.4 thorpej },
221 1.4 thorpej
222 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805,
223 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG,
224 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
225 1.34 bad UBS_MIN_AGGR,
226 1.4 thorpej "Broadcom BCM5805"
227 1.4 thorpej },
228 1.4 thorpej
229 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820,
230 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
231 1.4 thorpej UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
232 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR,
233 1.34 bad UBS_MIN_AGGR,
234 1.4 thorpej "Broadcom BCM5820"
235 1.4 thorpej },
236 1.4 thorpej
237 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821,
238 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
239 1.4 thorpej UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
240 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
241 1.4 thorpej BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
242 1.34 bad UBS_MIN_AGGR,
243 1.4 thorpej "Broadcom BCM5821"
244 1.4 thorpej },
245 1.4 thorpej { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K,
246 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
247 1.4 thorpej UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
248 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
249 1.4 thorpej BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
250 1.34 bad UBS_MIN_AGGR,
251 1.4 thorpej "Sun Crypto Accelerator 1000"
252 1.4 thorpej },
253 1.4 thorpej { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821,
254 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
255 1.4 thorpej UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
256 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
257 1.4 thorpej BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
258 1.34 bad UBS_MIN_AGGR,
259 1.4 thorpej "Broadcom BCM5821 (Sun)"
260 1.4 thorpej },
261 1.4 thorpej
262 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822,
263 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
264 1.4 thorpej UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY,
265 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
266 1.4 thorpej BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
267 1.34 bad UBS_MIN_AGGR,
268 1.4 thorpej "Broadcom BCM5822"
269 1.4 thorpej },
270 1.4 thorpej
271 1.4 thorpej { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823,
272 1.4 thorpej UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
273 1.40 bad UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
274 1.4 thorpej BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
275 1.4 thorpej BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
276 1.34 bad UBS_MIN_AGGR,
277 1.4 thorpej "Broadcom BCM5823"
278 1.4 thorpej },
279 1.4 thorpej
280 1.34 bad { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825,
281 1.34 bad UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX |
282 1.40 bad UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
283 1.34 bad BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
284 1.34 bad BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY,
285 1.34 bad UBS_MIN_AGGR,
286 1.34 bad "Broadcom BCM5825"
287 1.34 bad },
288 1.34 bad
289 1.34 bad { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860,
290 1.34 bad UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
291 1.34 bad UBS_FLAGS_LONGCTX |
292 1.34 bad UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
293 1.40 bad UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
294 1.34 bad BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
295 1.34 bad BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
296 1.34 bad BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
297 1.34 bad UBS_MAX_AGGR,
298 1.34 bad "Broadcom BCM5860"
299 1.34 bad },
300 1.34 bad
301 1.34 bad { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861,
302 1.34 bad UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
303 1.34 bad UBS_FLAGS_LONGCTX |
304 1.34 bad UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
305 1.40 bad UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
306 1.34 bad BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
307 1.34 bad BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
308 1.34 bad BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
309 1.34 bad UBS_MAX_AGGR,
310 1.34 bad "Broadcom BCM5861"
311 1.34 bad },
312 1.34 bad
313 1.34 bad { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862,
314 1.34 bad UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM |
315 1.34 bad UBS_FLAGS_LONGCTX |
316 1.34 bad UBS_FLAGS_RNG | UBS_FLAGS_RNG4 |
317 1.40 bad UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES,
318 1.34 bad BS_STAT_MCR1_DONE | BS_STAT_DMAERR |
319 1.34 bad BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY |
320 1.34 bad BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY,
321 1.34 bad UBS_MAX_AGGR,
322 1.34 bad "Broadcom BCM5862"
323 1.34 bad },
324 1.34 bad
325 1.4 thorpej { 0, 0,
326 1.4 thorpej 0,
327 1.4 thorpej 0,
328 1.34 bad 0,
329 1.4 thorpej NULL
330 1.4 thorpej }
331 1.4 thorpej };
332 1.4 thorpej
333 1.4 thorpej static const struct ubsec_product *
334 1.4 thorpej ubsec_lookup(const struct pci_attach_args *pa)
335 1.4 thorpej {
336 1.4 thorpej const struct ubsec_product *up;
337 1.4 thorpej
338 1.4 thorpej for (up = ubsec_products; up->ubsec_name != NULL; up++) {
339 1.4 thorpej if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor &&
340 1.4 thorpej PCI_PRODUCT(pa->pa_id) == up->ubsec_product)
341 1.4 thorpej return (up);
342 1.4 thorpej }
343 1.4 thorpej return (NULL);
344 1.4 thorpej }
345 1.4 thorpej
346 1.1 jonathan static int
347 1.22 cegger ubsec_probe(device_t parent, cfdata_t match, void *aux)
348 1.1 jonathan {
349 1.1 jonathan struct pci_attach_args *pa = (struct pci_attach_args *)aux;
350 1.1 jonathan
351 1.4 thorpej if (ubsec_lookup(pa) != NULL)
352 1.1 jonathan return (1);
353 1.1 jonathan
354 1.1 jonathan return (0);
355 1.1 jonathan }
356 1.1 jonathan
357 1.7 thorpej static void
358 1.22 cegger ubsec_attach(device_t parent, device_t self, void *aux)
359 1.1 jonathan {
360 1.23 cegger struct ubsec_softc *sc = device_private(self);
361 1.1 jonathan struct pci_attach_args *pa = aux;
362 1.4 thorpej const struct ubsec_product *up;
363 1.1 jonathan pci_chipset_tag_t pc = pa->pa_pc;
364 1.1 jonathan pci_intr_handle_t ih;
365 1.1 jonathan const char *intrstr = NULL;
366 1.34 bad pcireg_t memtype;
367 1.1 jonathan struct ubsec_dma *dmap;
368 1.1 jonathan u_int32_t cmd, i;
369 1.38 christos char intrbuf[PCI_INTRSTR_LEN];
370 1.1 jonathan
371 1.28 chs sc->sc_dev = self;
372 1.32 bad sc->sc_pct = pc;
373 1.32 bad
374 1.4 thorpej up = ubsec_lookup(pa);
375 1.4 thorpej if (up == NULL) {
376 1.4 thorpej printf("\n");
377 1.4 thorpej panic("ubsec_attach: impossible");
378 1.4 thorpej }
379 1.4 thorpej
380 1.27 drochner pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1);
381 1.4 thorpej
382 1.1 jonathan SIMPLEQ_INIT(&sc->sc_queue);
383 1.1 jonathan SIMPLEQ_INIT(&sc->sc_qchip);
384 1.1 jonathan SIMPLEQ_INIT(&sc->sc_queue2);
385 1.1 jonathan SIMPLEQ_INIT(&sc->sc_qchip2);
386 1.34 bad SIMPLEQ_INIT(&sc->sc_queue4);
387 1.34 bad SIMPLEQ_INIT(&sc->sc_qchip4);
388 1.1 jonathan SIMPLEQ_INIT(&sc->sc_q2free);
389 1.1 jonathan
390 1.4 thorpej sc->sc_flags = up->ubsec_flags;
391 1.4 thorpej sc->sc_statmask = up->ubsec_statmask;
392 1.34 bad sc->sc_maxaggr = up->ubsec_maxaggr;
393 1.1 jonathan
394 1.1 jonathan cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
395 1.4 thorpej cmd |= PCI_COMMAND_MASTER_ENABLE;
396 1.1 jonathan pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
397 1.1 jonathan
398 1.34 bad memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR);
399 1.34 bad if (pci_mapreg_map(pa, BS_BAR, memtype, 0,
400 1.32 bad &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) {
401 1.28 chs aprint_error_dev(self, "can't find mem space");
402 1.1 jonathan return;
403 1.1 jonathan }
404 1.1 jonathan
405 1.1 jonathan sc->sc_dmat = pa->pa_dmat;
406 1.1 jonathan
407 1.1 jonathan if (pci_intr_map(pa, &ih)) {
408 1.28 chs aprint_error_dev(self, "couldn't map interrupt\n");
409 1.1 jonathan return;
410 1.1 jonathan }
411 1.38 christos intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
412 1.4 thorpej sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc);
413 1.1 jonathan if (sc->sc_ih == NULL) {
414 1.28 chs aprint_error_dev(self, "couldn't establish interrupt");
415 1.1 jonathan if (intrstr != NULL)
416 1.24 njoly aprint_error(" at %s", intrstr);
417 1.24 njoly aprint_error("\n");
418 1.1 jonathan return;
419 1.1 jonathan }
420 1.28 chs aprint_normal_dev(self, "interrupting at %s\n", intrstr);
421 1.1 jonathan
422 1.1 jonathan sc->sc_cid = crypto_get_driverid(0);
423 1.1 jonathan if (sc->sc_cid < 0) {
424 1.28 chs aprint_error_dev(self, "couldn't get crypto driver id\n");
425 1.1 jonathan pci_intr_disestablish(pc, sc->sc_ih);
426 1.1 jonathan return;
427 1.1 jonathan }
428 1.1 jonathan
429 1.29 tls sc->sc_rng_need = RND_POOLBITS / NBBY;
430 1.29 tls mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM);
431 1.29 tls
432 1.1 jonathan SIMPLEQ_INIT(&sc->sc_freequeue);
433 1.1 jonathan dmap = sc->sc_dmaa;
434 1.1 jonathan for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
435 1.1 jonathan struct ubsec_q *q;
436 1.1 jonathan
437 1.1 jonathan q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
438 1.39 bad M_DEVBUF, M_ZERO|M_NOWAIT);
439 1.1 jonathan if (q == NULL) {
440 1.28 chs aprint_error_dev(self, "can't allocate queue buffers\n");
441 1.1 jonathan break;
442 1.1 jonathan }
443 1.1 jonathan
444 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
445 1.1 jonathan &dmap->d_alloc, 0)) {
446 1.28 chs aprint_error_dev(self, "can't allocate dma buffers\n");
447 1.1 jonathan free(q, M_DEVBUF);
448 1.1 jonathan break;
449 1.1 jonathan }
450 1.1 jonathan dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
451 1.1 jonathan
452 1.1 jonathan q->q_dma = dmap;
453 1.1 jonathan sc->sc_queuea[i] = q;
454 1.1 jonathan
455 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
456 1.1 jonathan }
457 1.1 jonathan
458 1.1 jonathan crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
459 1.1 jonathan ubsec_newsession, ubsec_freesession, ubsec_process, sc);
460 1.1 jonathan crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
461 1.1 jonathan ubsec_newsession, ubsec_freesession, ubsec_process, sc);
462 1.15 tls crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0,
463 1.1 jonathan ubsec_newsession, ubsec_freesession, ubsec_process, sc);
464 1.15 tls crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0,
465 1.1 jonathan ubsec_newsession, ubsec_freesession, ubsec_process, sc);
466 1.40 bad if (sc->sc_flags & UBS_FLAGS_AES) {
467 1.40 bad crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
468 1.40 bad ubsec_newsession, ubsec_freesession, ubsec_process, sc);
469 1.40 bad }
470 1.1 jonathan
471 1.1 jonathan /*
472 1.1 jonathan * Reset Broadcom chip
473 1.1 jonathan */
474 1.1 jonathan ubsec_reset_board(sc);
475 1.1 jonathan
476 1.1 jonathan /*
477 1.1 jonathan * Init Broadcom specific PCI settings
478 1.1 jonathan */
479 1.1 jonathan ubsec_init_pciregs(pa);
480 1.1 jonathan
481 1.1 jonathan /*
482 1.1 jonathan * Init Broadcom chip
483 1.1 jonathan */
484 1.1 jonathan ubsec_init_board(sc);
485 1.1 jonathan
486 1.1 jonathan #ifndef UBSEC_NO_RNG
487 1.1 jonathan if (sc->sc_flags & UBS_FLAGS_RNG) {
488 1.34 bad if (sc->sc_flags & UBS_FLAGS_RNG4)
489 1.34 bad sc->sc_statmask |= BS_STAT_MCR4_DONE;
490 1.34 bad else
491 1.34 bad sc->sc_statmask |= BS_STAT_MCR2_DONE;
492 1.1 jonathan
493 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
494 1.1 jonathan &sc->sc_rng.rng_q.q_mcr, 0))
495 1.1 jonathan goto skip_rng;
496 1.1 jonathan
497 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
498 1.1 jonathan &sc->sc_rng.rng_q.q_ctx, 0)) {
499 1.1 jonathan ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
500 1.1 jonathan goto skip_rng;
501 1.1 jonathan }
502 1.1 jonathan
503 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
504 1.1 jonathan UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
505 1.1 jonathan ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
506 1.1 jonathan ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
507 1.1 jonathan goto skip_rng;
508 1.1 jonathan }
509 1.1 jonathan
510 1.29 tls rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc);
511 1.29 tls rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev),
512 1.29 tls RND_TYPE_RNG,
513 1.29 tls RND_FLAG_NO_ESTIMATE|RND_FLAG_HASCB);
514 1.1 jonathan if (hz >= 100)
515 1.1 jonathan sc->sc_rnghz = hz / 100;
516 1.1 jonathan else
517 1.1 jonathan sc->sc_rnghz = 1;
518 1.1 jonathan #ifdef __OpenBSD__
519 1.1 jonathan timeout_set(&sc->sc_rngto, ubsec_rng, sc);
520 1.1 jonathan timeout_add(&sc->sc_rngto, sc->sc_rnghz);
521 1.1 jonathan #else
522 1.13 ad callout_init(&sc->sc_rngto, 0);
523 1.30 bad callout_setfunc(&sc->sc_rngto, ubsec_rng, sc);
524 1.30 bad callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
525 1.5 perry #endif
526 1.4 thorpej skip_rng:
527 1.4 thorpej if (sc->sc_rnghz)
528 1.28 chs aprint_normal_dev(self, "random number generator enabled\n");
529 1.4 thorpej else
530 1.28 chs aprint_error_dev(self, "WARNING: random number generator "
531 1.16 cegger "disabled\n");
532 1.1 jonathan }
533 1.1 jonathan #endif /* UBSEC_NO_RNG */
534 1.1 jonathan
535 1.1 jonathan if (sc->sc_flags & UBS_FLAGS_KEY) {
536 1.1 jonathan sc->sc_statmask |= BS_STAT_MCR2_DONE;
537 1.1 jonathan
538 1.1 jonathan crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
539 1.1 jonathan ubsec_kprocess, sc);
540 1.1 jonathan #if 0
541 1.1 jonathan crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
542 1.1 jonathan ubsec_kprocess, sc);
543 1.1 jonathan #endif
544 1.1 jonathan }
545 1.1 jonathan }
546 1.1 jonathan
547 1.32 bad static int
548 1.32 bad ubsec_detach(device_t self, int flags)
549 1.32 bad {
550 1.32 bad struct ubsec_softc *sc = device_private(self);
551 1.32 bad struct ubsec_q *q, *qtmp;
552 1.35 bad volatile u_int32_t ctrl;
553 1.32 bad
554 1.32 bad /* disable interrupts */
555 1.32 bad /* XXX wait/abort current ops? where is DMAERR enabled? */
556 1.35 bad ctrl = READ_REG(sc, BS_CTRL);
557 1.35 bad
558 1.35 bad ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR);
559 1.35 bad if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
560 1.35 bad ctrl &= ~BS_CTRL_MCR4INT;
561 1.35 bad
562 1.35 bad WRITE_REG(sc, BS_CTRL, ctrl);
563 1.32 bad
564 1.32 bad #ifndef UBSEC_NO_RNG
565 1.32 bad if (sc->sc_flags & UBS_FLAGS_RNG) {
566 1.32 bad callout_halt(&sc->sc_rngto, NULL);
567 1.32 bad ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
568 1.32 bad ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
569 1.32 bad ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
570 1.32 bad rnd_detach_source(&sc->sc_rnd_source);
571 1.32 bad }
572 1.32 bad #endif /* UBSEC_NO_RNG */
573 1.32 bad
574 1.32 bad crypto_unregister_all(sc->sc_cid);
575 1.32 bad
576 1.32 bad mutex_spin_enter(&sc->sc_mtx);
577 1.32 bad
578 1.32 bad ubsec_totalreset(sc); /* XXX leaves the chip running */
579 1.32 bad
580 1.32 bad SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) {
581 1.32 bad ubsec_dma_free(sc, &q->q_dma->d_alloc);
582 1.39 bad if (q->q_src_map != NULL)
583 1.39 bad bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
584 1.39 bad if (q->q_cached_dst_map != NULL)
585 1.39 bad bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map);
586 1.32 bad free(q, M_DEVBUF);
587 1.32 bad }
588 1.32 bad
589 1.32 bad mutex_spin_exit(&sc->sc_mtx);
590 1.32 bad
591 1.32 bad if (sc->sc_ih != NULL) {
592 1.32 bad pci_intr_disestablish(sc->sc_pct, sc->sc_ih);
593 1.32 bad sc->sc_ih = NULL;
594 1.32 bad }
595 1.32 bad
596 1.32 bad if (sc->sc_memsize != 0) {
597 1.32 bad bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize);
598 1.32 bad sc->sc_memsize = 0;
599 1.32 bad }
600 1.32 bad
601 1.32 bad return 0;
602 1.32 bad }
603 1.32 bad
604 1.36 pgoyette MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto");
605 1.32 bad
606 1.32 bad #ifdef _MODULE
607 1.32 bad #include "ioconf.c"
608 1.32 bad #endif
609 1.32 bad
610 1.32 bad static int
611 1.32 bad ubsec_modcmd(modcmd_t cmd, void *data)
612 1.32 bad {
613 1.32 bad int error = 0;
614 1.32 bad
615 1.32 bad switch (cmd) {
616 1.32 bad case MODULE_CMD_INIT:
617 1.32 bad #ifdef _MODULE
618 1.32 bad error = config_init_component(cfdriver_ioconf_ubsec,
619 1.32 bad cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
620 1.32 bad #endif
621 1.33 bad if (error == 0)
622 1.33 bad error = ubsec_sysctl_init();
623 1.32 bad return error;
624 1.32 bad case MODULE_CMD_FINI:
625 1.33 bad if (ubsec_sysctllog != NULL)
626 1.33 bad sysctl_teardown(&ubsec_sysctllog);
627 1.32 bad #ifdef _MODULE
628 1.32 bad error = config_fini_component(cfdriver_ioconf_ubsec,
629 1.32 bad cfattach_ioconf_ubsec, cfdata_ioconf_ubsec);
630 1.32 bad #endif
631 1.32 bad return error;
632 1.32 bad default:
633 1.32 bad return ENOTTY;
634 1.32 bad }
635 1.32 bad }
636 1.32 bad
637 1.33 bad static int
638 1.33 bad ubsec_sysctl_init(void)
639 1.33 bad {
640 1.33 bad const struct sysctlnode *node = NULL;
641 1.33 bad
642 1.33 bad ubsec_sysctllog = NULL;
643 1.33 bad
644 1.33 bad sysctl_createv(&ubsec_sysctllog, 0, NULL, &node,
645 1.33 bad CTLFLAG_PERMANENT,
646 1.33 bad CTLTYPE_NODE, "ubsec",
647 1.33 bad SYSCTL_DESCR("ubsec opetions"),
648 1.33 bad NULL, 0, NULL, 0,
649 1.33 bad CTL_HW, CTL_CREATE, CTL_EOL);
650 1.33 bad sysctl_createv(&ubsec_sysctllog, 0, &node, NULL,
651 1.33 bad CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
652 1.33 bad CTLTYPE_INT, "maxbatch",
653 1.33 bad SYSCTL_DESCR("max ops to batch w/o interrupt"),
654 1.33 bad NULL, 0, &ubsec_maxbatch, 0,
655 1.33 bad CTL_CREATE, CTL_EOL);
656 1.33 bad sysctl_createv(&ubsec_sysctllog, 0, &node, NULL,
657 1.33 bad CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
658 1.33 bad CTLTYPE_INT, "maxaggr",
659 1.33 bad SYSCTL_DESCR("max ops to aggregate under one interrupt"),
660 1.33 bad NULL, 0, &ubsec_maxaggr, 0,
661 1.33 bad CTL_CREATE, CTL_EOL);
662 1.33 bad
663 1.33 bad return 0;
664 1.33 bad }
665 1.33 bad
666 1.1 jonathan /*
667 1.1 jonathan * UBSEC Interrupt routine
668 1.1 jonathan */
669 1.7 thorpej static int
670 1.1 jonathan ubsec_intr(void *arg)
671 1.1 jonathan {
672 1.1 jonathan struct ubsec_softc *sc = arg;
673 1.1 jonathan volatile u_int32_t stat;
674 1.1 jonathan struct ubsec_q *q;
675 1.1 jonathan struct ubsec_dma *dmap;
676 1.34 bad int flags;
677 1.1 jonathan int npkts = 0, i;
678 1.1 jonathan
679 1.29 tls mutex_spin_enter(&sc->sc_mtx);
680 1.1 jonathan stat = READ_REG(sc, BS_STAT);
681 1.1 jonathan stat &= sc->sc_statmask;
682 1.1 jonathan if (stat == 0) {
683 1.29 tls mutex_spin_exit(&sc->sc_mtx);
684 1.1 jonathan return (0);
685 1.1 jonathan }
686 1.1 jonathan
687 1.1 jonathan WRITE_REG(sc, BS_STAT, stat); /* IACK */
688 1.1 jonathan
689 1.1 jonathan /*
690 1.1 jonathan * Check to see if we have any packets waiting for us
691 1.1 jonathan */
692 1.1 jonathan if ((stat & BS_STAT_MCR1_DONE)) {
693 1.1 jonathan while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
694 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_qchip);
695 1.1 jonathan dmap = q->q_dma;
696 1.1 jonathan
697 1.1 jonathan if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
698 1.1 jonathan break;
699 1.1 jonathan
700 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_qchip);
701 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next);
702 1.1 jonathan
703 1.1 jonathan npkts = q->q_nstacked_mcrs;
704 1.1 jonathan sc->sc_nqchip -= 1+npkts;
705 1.1 jonathan /*
706 1.1 jonathan * search for further sc_qchip ubsec_q's that share
707 1.1 jonathan * the same MCR, and complete them too, they must be
708 1.1 jonathan * at the top.
709 1.1 jonathan */
710 1.1 jonathan for (i = 0; i < npkts; i++) {
711 1.1 jonathan if(q->q_stacked_mcr[i])
712 1.1 jonathan ubsec_callback(sc, q->q_stacked_mcr[i]);
713 1.1 jonathan else
714 1.1 jonathan break;
715 1.1 jonathan }
716 1.1 jonathan ubsec_callback(sc, q);
717 1.1 jonathan }
718 1.1 jonathan
719 1.1 jonathan /*
720 1.1 jonathan * Don't send any more packet to chip if there has been
721 1.1 jonathan * a DMAERR.
722 1.1 jonathan */
723 1.1 jonathan if (!(stat & BS_STAT_DMAERR))
724 1.1 jonathan ubsec_feed(sc);
725 1.1 jonathan }
726 1.1 jonathan
727 1.1 jonathan /*
728 1.1 jonathan * Check to see if we have any key setups/rng's waiting for us
729 1.1 jonathan */
730 1.1 jonathan if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
731 1.1 jonathan (stat & BS_STAT_MCR2_DONE)) {
732 1.1 jonathan struct ubsec_q2 *q2;
733 1.1 jonathan struct ubsec_mcr *mcr;
734 1.1 jonathan
735 1.1 jonathan while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
736 1.1 jonathan q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
737 1.1 jonathan
738 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
739 1.1 jonathan 0, q2->q_mcr.dma_map->dm_mapsize,
740 1.1 jonathan BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
741 1.1 jonathan
742 1.1 jonathan mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
743 1.34 bad
744 1.34 bad /* A bug in new devices requires to swap this field */
745 1.34 bad if (sc->sc_flags & UBS_FLAGS_MULTIMCR)
746 1.34 bad flags = htole16(mcr->mcr_flags);
747 1.34 bad else
748 1.34 bad flags = mcr->mcr_flags;
749 1.34 bad if ((flags & htole16(UBS_MCR_DONE)) == 0) {
750 1.1 jonathan bus_dmamap_sync(sc->sc_dmat,
751 1.1 jonathan q2->q_mcr.dma_map, 0,
752 1.1 jonathan q2->q_mcr.dma_map->dm_mapsize,
753 1.1 jonathan BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
754 1.1 jonathan break;
755 1.1 jonathan }
756 1.1 jonathan q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
757 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next);
758 1.1 jonathan ubsec_callback2(sc, q2);
759 1.1 jonathan /*
760 1.1 jonathan * Don't send any more packet to chip if there has been
761 1.1 jonathan * a DMAERR.
762 1.1 jonathan */
763 1.1 jonathan if (!(stat & BS_STAT_DMAERR))
764 1.1 jonathan ubsec_feed2(sc);
765 1.1 jonathan }
766 1.1 jonathan }
767 1.34 bad if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) {
768 1.34 bad struct ubsec_q2 *q2;
769 1.34 bad struct ubsec_mcr *mcr;
770 1.34 bad
771 1.34 bad while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) {
772 1.34 bad q2 = SIMPLEQ_FIRST(&sc->sc_qchip4);
773 1.34 bad
774 1.34 bad bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map,
775 1.34 bad 0, q2->q_mcr.dma_map->dm_mapsize,
776 1.34 bad BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
777 1.34 bad
778 1.34 bad mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
779 1.34 bad
780 1.34 bad /* A bug in new devices requires to swap this field */
781 1.34 bad flags = htole16(mcr->mcr_flags);
782 1.34 bad
783 1.34 bad if ((flags & htole16(UBS_MCR_DONE)) == 0) {
784 1.34 bad bus_dmamap_sync(sc->sc_dmat,
785 1.34 bad q2->q_mcr.dma_map, 0,
786 1.34 bad q2->q_mcr.dma_map->dm_mapsize,
787 1.34 bad BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
788 1.34 bad break;
789 1.34 bad }
790 1.34 bad SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next);
791 1.34 bad ubsec_callback2(sc, q2);
792 1.34 bad /*
793 1.34 bad * Don't send any more packet to chip if there has been
794 1.34 bad * a DMAERR.
795 1.34 bad */
796 1.34 bad if (!(stat & BS_STAT_DMAERR))
797 1.34 bad ubsec_feed4(sc);
798 1.34 bad }
799 1.34 bad }
800 1.1 jonathan
801 1.1 jonathan /*
802 1.1 jonathan * Check to see if we got any DMA Error
803 1.1 jonathan */
804 1.1 jonathan if (stat & BS_STAT_DMAERR) {
805 1.1 jonathan #ifdef UBSEC_DEBUG
806 1.1 jonathan if (ubsec_debug) {
807 1.1 jonathan volatile u_int32_t a = READ_REG(sc, BS_ERR);
808 1.1 jonathan
809 1.28 chs printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev),
810 1.1 jonathan (a & BS_ERR_READ) ? "read" : "write",
811 1.1 jonathan a & BS_ERR_ADDR);
812 1.1 jonathan }
813 1.1 jonathan #endif /* UBSEC_DEBUG */
814 1.1 jonathan ubsecstats.hst_dmaerr++;
815 1.1 jonathan ubsec_totalreset(sc);
816 1.1 jonathan ubsec_feed(sc);
817 1.1 jonathan }
818 1.1 jonathan
819 1.1 jonathan if (sc->sc_needwakeup) { /* XXX check high watermark */
820 1.6 christos int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
821 1.1 jonathan #ifdef UBSEC_DEBUG
822 1.1 jonathan if (ubsec_debug)
823 1.28 chs printf("%s: wakeup crypto (%x)\n", device_xname(sc->sc_dev),
824 1.1 jonathan sc->sc_needwakeup);
825 1.1 jonathan #endif /* UBSEC_DEBUG */
826 1.6 christos sc->sc_needwakeup &= ~wkeup;
827 1.6 christos crypto_unblock(sc->sc_cid, wkeup);
828 1.1 jonathan }
829 1.29 tls mutex_spin_exit(&sc->sc_mtx);
830 1.1 jonathan return (1);
831 1.1 jonathan }
832 1.1 jonathan
833 1.1 jonathan /*
834 1.1 jonathan * ubsec_feed() - aggregate and post requests to chip
835 1.1 jonathan * OpenBSD comments:
836 1.1 jonathan * It is assumed that the caller set splnet()
837 1.1 jonathan */
838 1.1 jonathan static void
839 1.1 jonathan ubsec_feed(struct ubsec_softc *sc)
840 1.1 jonathan {
841 1.1 jonathan struct ubsec_q *q, *q2;
842 1.1 jonathan int npkts, i;
843 1.1 jonathan void *v;
844 1.1 jonathan u_int32_t stat;
845 1.1 jonathan #ifdef UBSEC_DEBUG
846 1.1 jonathan static int max;
847 1.1 jonathan #endif /* UBSEC_DEBUG */
848 1.1 jonathan
849 1.1 jonathan npkts = sc->sc_nqueue;
850 1.1 jonathan if (npkts > ubsecstats.hst_maxqueue)
851 1.1 jonathan ubsecstats.hst_maxqueue = npkts;
852 1.1 jonathan if (npkts < 2)
853 1.1 jonathan goto feed1;
854 1.1 jonathan
855 1.1 jonathan /*
856 1.1 jonathan * Decide how many ops to combine in a single MCR. We cannot
857 1.1 jonathan * aggregate more than UBS_MAX_AGGR because this is the number
858 1.1 jonathan * of slots defined in the data structure. Otherwise we clamp
859 1.1 jonathan * based on the tunable parameter ubsec_maxaggr. Note that
860 1.1 jonathan * aggregation can happen in two ways: either by batching ops
861 1.5 perry * from above or because the h/w backs up and throttles us.
862 1.1 jonathan * Aggregating ops reduces the number of interrupts to the host
863 1.1 jonathan * but also (potentially) increases the latency for processing
864 1.1 jonathan * completed ops as we only get an interrupt when all aggregated
865 1.1 jonathan * ops have completed.
866 1.1 jonathan */
867 1.34 bad if (npkts > sc->sc_maxaggr)
868 1.34 bad npkts = sc->sc_maxaggr;
869 1.1 jonathan if (npkts > ubsec_maxaggr)
870 1.1 jonathan npkts = ubsec_maxaggr;
871 1.1 jonathan if (npkts > ubsecstats.hst_maxbatch)
872 1.1 jonathan ubsecstats.hst_maxbatch = npkts;
873 1.1 jonathan if (npkts < 2)
874 1.1 jonathan goto feed1;
875 1.1 jonathan ubsecstats.hst_totbatch += npkts-1;
876 1.1 jonathan
877 1.1 jonathan if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
878 1.1 jonathan if (stat & BS_STAT_DMAERR) {
879 1.1 jonathan ubsec_totalreset(sc);
880 1.1 jonathan ubsecstats.hst_dmaerr++;
881 1.1 jonathan } else {
882 1.1 jonathan ubsecstats.hst_mcr1full++;
883 1.1 jonathan }
884 1.1 jonathan return;
885 1.1 jonathan }
886 1.1 jonathan
887 1.1 jonathan #ifdef UBSEC_DEBUG
888 1.1 jonathan if (ubsec_debug)
889 1.1 jonathan printf("merging %d records\n", npkts);
890 1.1 jonathan /* XXX temporary aggregation statistics reporting code */
891 1.1 jonathan if (max < npkts) {
892 1.1 jonathan max = npkts;
893 1.28 chs printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), max);
894 1.1 jonathan }
895 1.1 jonathan #endif /* UBSEC_DEBUG */
896 1.1 jonathan
897 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_queue);
898 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next);
899 1.1 jonathan --sc->sc_nqueue;
900 1.1 jonathan
901 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
902 1.1 jonathan 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
903 1.1 jonathan if (q->q_dst_map != NULL)
904 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
905 1.1 jonathan 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
906 1.1 jonathan
907 1.1 jonathan q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */
908 1.1 jonathan
909 1.1 jonathan for (i = 0; i < q->q_nstacked_mcrs; i++) {
910 1.1 jonathan q2 = SIMPLEQ_FIRST(&sc->sc_queue);
911 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
912 1.1 jonathan 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
913 1.1 jonathan if (q2->q_dst_map != NULL)
914 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
915 1.1 jonathan 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
916 1.1 jonathan q2= SIMPLEQ_FIRST(&sc->sc_queue);
917 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next);
918 1.1 jonathan --sc->sc_nqueue;
919 1.1 jonathan
920 1.1 jonathan v = ((void *)&q2->q_dma->d_dma->d_mcr);
921 1.1 jonathan v = (char*)v + (sizeof(struct ubsec_mcr) -
922 1.1 jonathan sizeof(struct ubsec_mcr_add));
923 1.20 tsutsui memcpy(&q->q_dma->d_dma->d_mcradd[i], v, sizeof(struct ubsec_mcr_add));
924 1.1 jonathan q->q_stacked_mcr[i] = q2;
925 1.1 jonathan }
926 1.1 jonathan q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
927 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
928 1.1 jonathan sc->sc_nqchip += npkts;
929 1.1 jonathan if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
930 1.1 jonathan ubsecstats.hst_maxqchip = sc->sc_nqchip;
931 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
932 1.1 jonathan 0, q->q_dma->d_alloc.dma_map->dm_mapsize,
933 1.1 jonathan BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
934 1.1 jonathan WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
935 1.1 jonathan offsetof(struct ubsec_dmachunk, d_mcr));
936 1.1 jonathan return;
937 1.1 jonathan
938 1.1 jonathan feed1:
939 1.1 jonathan while (!SIMPLEQ_EMPTY(&sc->sc_queue)) {
940 1.1 jonathan if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
941 1.1 jonathan if (stat & BS_STAT_DMAERR) {
942 1.1 jonathan ubsec_totalreset(sc);
943 1.1 jonathan ubsecstats.hst_dmaerr++;
944 1.1 jonathan } else {
945 1.1 jonathan ubsecstats.hst_mcr1full++;
946 1.1 jonathan }
947 1.1 jonathan break;
948 1.1 jonathan }
949 1.1 jonathan
950 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_queue);
951 1.1 jonathan
952 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
953 1.1 jonathan 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
954 1.1 jonathan if (q->q_dst_map != NULL)
955 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
956 1.1 jonathan 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
957 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map,
958 1.1 jonathan 0, q->q_dma->d_alloc.dma_map->dm_mapsize,
959 1.1 jonathan BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
960 1.1 jonathan
961 1.1 jonathan WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
962 1.1 jonathan offsetof(struct ubsec_dmachunk, d_mcr));
963 1.1 jonathan #ifdef UBSEC_DEBUG
964 1.1 jonathan if (ubsec_debug)
965 1.1 jonathan printf("feed: q->chip %p %08x stat %08x\n",
966 1.1 jonathan q, (u_int32_t)q->q_dma->d_alloc.dma_paddr,
967 1.1 jonathan stat);
968 1.1 jonathan #endif /* UBSEC_DEBUG */
969 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_queue);
970 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next);
971 1.1 jonathan --sc->sc_nqueue;
972 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
973 1.1 jonathan sc->sc_nqchip++;
974 1.1 jonathan }
975 1.1 jonathan if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
976 1.1 jonathan ubsecstats.hst_maxqchip = sc->sc_nqchip;
977 1.1 jonathan }
978 1.1 jonathan
979 1.1 jonathan /*
980 1.1 jonathan * Allocate a new 'session' and return an encoded session id. 'sidp'
981 1.1 jonathan * contains our registration id, and should contain an encoded session
982 1.1 jonathan * id on successful allocation.
983 1.1 jonathan */
984 1.1 jonathan static int
985 1.1 jonathan ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
986 1.1 jonathan {
987 1.1 jonathan struct cryptoini *c, *encini = NULL, *macini = NULL;
988 1.1 jonathan struct ubsec_softc *sc;
989 1.1 jonathan struct ubsec_session *ses = NULL;
990 1.1 jonathan MD5_CTX md5ctx;
991 1.1 jonathan SHA1_CTX sha1ctx;
992 1.1 jonathan int i, sesn;
993 1.1 jonathan
994 1.2 jonathan sc = arg;
995 1.2 jonathan KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/);
996 1.2 jonathan
997 1.1 jonathan if (sidp == NULL || cri == NULL || sc == NULL)
998 1.1 jonathan return (EINVAL);
999 1.1 jonathan
1000 1.1 jonathan for (c = cri; c != NULL; c = c->cri_next) {
1001 1.15 tls if (c->cri_alg == CRYPTO_MD5_HMAC_96 ||
1002 1.15 tls c->cri_alg == CRYPTO_SHA1_HMAC_96) {
1003 1.1 jonathan if (macini)
1004 1.1 jonathan return (EINVAL);
1005 1.1 jonathan macini = c;
1006 1.1 jonathan } else if (c->cri_alg == CRYPTO_DES_CBC ||
1007 1.40 bad c->cri_alg == CRYPTO_3DES_CBC ||
1008 1.40 bad c->cri_alg == CRYPTO_AES_CBC) {
1009 1.1 jonathan if (encini)
1010 1.1 jonathan return (EINVAL);
1011 1.1 jonathan encini = c;
1012 1.1 jonathan } else
1013 1.1 jonathan return (EINVAL);
1014 1.1 jonathan }
1015 1.1 jonathan if (encini == NULL && macini == NULL)
1016 1.1 jonathan return (EINVAL);
1017 1.1 jonathan
1018 1.40 bad if (encini && encini->cri_alg == CRYPTO_AES_CBC) {
1019 1.40 bad switch (encini->cri_klen) {
1020 1.40 bad case 128:
1021 1.40 bad case 192:
1022 1.40 bad case 256:
1023 1.40 bad break;
1024 1.40 bad default:
1025 1.40 bad return (EINVAL);
1026 1.40 bad }
1027 1.40 bad }
1028 1.40 bad
1029 1.1 jonathan if (sc->sc_sessions == NULL) {
1030 1.1 jonathan ses = sc->sc_sessions = (struct ubsec_session *)malloc(
1031 1.1 jonathan sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1032 1.1 jonathan if (ses == NULL)
1033 1.1 jonathan return (ENOMEM);
1034 1.1 jonathan sesn = 0;
1035 1.1 jonathan sc->sc_nsessions = 1;
1036 1.1 jonathan } else {
1037 1.1 jonathan for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
1038 1.1 jonathan if (sc->sc_sessions[sesn].ses_used == 0) {
1039 1.1 jonathan ses = &sc->sc_sessions[sesn];
1040 1.1 jonathan break;
1041 1.1 jonathan }
1042 1.1 jonathan }
1043 1.1 jonathan
1044 1.1 jonathan if (ses == NULL) {
1045 1.1 jonathan sesn = sc->sc_nsessions;
1046 1.1 jonathan ses = (struct ubsec_session *)malloc((sesn + 1) *
1047 1.1 jonathan sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
1048 1.1 jonathan if (ses == NULL)
1049 1.1 jonathan return (ENOMEM);
1050 1.20 tsutsui memcpy(ses, sc->sc_sessions, sesn *
1051 1.1 jonathan sizeof(struct ubsec_session));
1052 1.18 cegger memset(sc->sc_sessions, 0, sesn *
1053 1.1 jonathan sizeof(struct ubsec_session));
1054 1.1 jonathan free(sc->sc_sessions, M_DEVBUF);
1055 1.1 jonathan sc->sc_sessions = ses;
1056 1.1 jonathan ses = &sc->sc_sessions[sesn];
1057 1.1 jonathan sc->sc_nsessions++;
1058 1.1 jonathan }
1059 1.1 jonathan }
1060 1.1 jonathan
1061 1.18 cegger memset(ses, 0, sizeof(struct ubsec_session));
1062 1.1 jonathan ses->ses_used = 1;
1063 1.1 jonathan if (encini) {
1064 1.1 jonathan /* get an IV, network byte order */
1065 1.1 jonathan #ifdef __NetBSD__
1066 1.26 tls cprng_fast(ses->ses_iv, sizeof(ses->ses_iv));
1067 1.1 jonathan #else
1068 1.1 jonathan get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
1069 1.1 jonathan #endif
1070 1.1 jonathan
1071 1.1 jonathan /* Go ahead and compute key in ubsec's byte order */
1072 1.40 bad if (encini->cri_alg == CRYPTO_AES_CBC) {
1073 1.40 bad memcpy(ses->ses_key, encini->cri_key,
1074 1.40 bad encini->cri_klen / 8);
1075 1.40 bad }
1076 1.1 jonathan if (encini->cri_alg == CRYPTO_DES_CBC) {
1077 1.40 bad memcpy(&ses->ses_key[0], encini->cri_key, 8);
1078 1.40 bad memcpy(&ses->ses_key[2], encini->cri_key, 8);
1079 1.40 bad memcpy(&ses->ses_key[4], encini->cri_key, 8);
1080 1.1 jonathan } else
1081 1.40 bad memcpy(ses->ses_key, encini->cri_key, 24);
1082 1.1 jonathan
1083 1.40 bad SWAP32(ses->ses_key[0]);
1084 1.40 bad SWAP32(ses->ses_key[1]);
1085 1.40 bad SWAP32(ses->ses_key[2]);
1086 1.40 bad SWAP32(ses->ses_key[3]);
1087 1.40 bad SWAP32(ses->ses_key[4]);
1088 1.40 bad SWAP32(ses->ses_key[5]);
1089 1.1 jonathan }
1090 1.1 jonathan
1091 1.1 jonathan if (macini) {
1092 1.1 jonathan for (i = 0; i < macini->cri_klen / 8; i++)
1093 1.1 jonathan macini->cri_key[i] ^= HMAC_IPAD_VAL;
1094 1.1 jonathan
1095 1.15 tls if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1096 1.1 jonathan MD5Init(&md5ctx);
1097 1.1 jonathan MD5Update(&md5ctx, macini->cri_key,
1098 1.1 jonathan macini->cri_klen / 8);
1099 1.1 jonathan MD5Update(&md5ctx, hmac_ipad_buffer,
1100 1.1 jonathan HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1101 1.20 tsutsui memcpy(ses->ses_hminner, md5ctx.state,
1102 1.1 jonathan sizeof(md5ctx.state));
1103 1.1 jonathan } else {
1104 1.1 jonathan SHA1Init(&sha1ctx);
1105 1.1 jonathan SHA1Update(&sha1ctx, macini->cri_key,
1106 1.1 jonathan macini->cri_klen / 8);
1107 1.1 jonathan SHA1Update(&sha1ctx, hmac_ipad_buffer,
1108 1.1 jonathan HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1109 1.20 tsutsui memcpy(ses->ses_hminner, sha1ctx.state,
1110 1.1 jonathan sizeof(sha1ctx.state));
1111 1.1 jonathan }
1112 1.1 jonathan
1113 1.1 jonathan for (i = 0; i < macini->cri_klen / 8; i++)
1114 1.1 jonathan macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
1115 1.1 jonathan
1116 1.15 tls if (macini->cri_alg == CRYPTO_MD5_HMAC_96) {
1117 1.1 jonathan MD5Init(&md5ctx);
1118 1.1 jonathan MD5Update(&md5ctx, macini->cri_key,
1119 1.1 jonathan macini->cri_klen / 8);
1120 1.1 jonathan MD5Update(&md5ctx, hmac_opad_buffer,
1121 1.1 jonathan HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1122 1.20 tsutsui memcpy(ses->ses_hmouter, md5ctx.state,
1123 1.1 jonathan sizeof(md5ctx.state));
1124 1.1 jonathan } else {
1125 1.1 jonathan SHA1Init(&sha1ctx);
1126 1.1 jonathan SHA1Update(&sha1ctx, macini->cri_key,
1127 1.1 jonathan macini->cri_klen / 8);
1128 1.1 jonathan SHA1Update(&sha1ctx, hmac_opad_buffer,
1129 1.1 jonathan HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1130 1.20 tsutsui memcpy(ses->ses_hmouter, sha1ctx.state,
1131 1.1 jonathan sizeof(sha1ctx.state));
1132 1.1 jonathan }
1133 1.1 jonathan
1134 1.1 jonathan for (i = 0; i < macini->cri_klen / 8; i++)
1135 1.1 jonathan macini->cri_key[i] ^= HMAC_OPAD_VAL;
1136 1.1 jonathan }
1137 1.1 jonathan
1138 1.28 chs *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn);
1139 1.1 jonathan return (0);
1140 1.1 jonathan }
1141 1.1 jonathan
1142 1.1 jonathan /*
1143 1.1 jonathan * Deallocate a session.
1144 1.1 jonathan */
1145 1.1 jonathan static int
1146 1.1 jonathan ubsec_freesession(void *arg, u_int64_t tid)
1147 1.1 jonathan {
1148 1.1 jonathan struct ubsec_softc *sc;
1149 1.1 jonathan int session;
1150 1.1 jonathan u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
1151 1.1 jonathan
1152 1.1 jonathan sc = arg;
1153 1.1 jonathan KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/);
1154 1.1 jonathan
1155 1.1 jonathan session = UBSEC_SESSION(sid);
1156 1.1 jonathan if (session >= sc->sc_nsessions)
1157 1.1 jonathan return (EINVAL);
1158 1.1 jonathan
1159 1.18 cegger memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session]));
1160 1.1 jonathan return (0);
1161 1.1 jonathan }
1162 1.1 jonathan
1163 1.1 jonathan #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */
1164 1.1 jonathan static void
1165 1.1 jonathan ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1166 1.1 jonathan {
1167 1.1 jonathan struct ubsec_operand *op = arg;
1168 1.1 jonathan
1169 1.5 perry KASSERT(nsegs <= UBS_MAX_SCATTER
1170 1.1 jonathan /*, ("Too many DMA segments returned when mapping operand")*/);
1171 1.1 jonathan #ifdef UBSEC_DEBUG
1172 1.1 jonathan if (ubsec_debug)
1173 1.1 jonathan printf("ubsec_op_cb: mapsize %u nsegs %d\n",
1174 1.1 jonathan (u_int) mapsize, nsegs);
1175 1.1 jonathan #endif
1176 1.1 jonathan op->mapsize = mapsize;
1177 1.1 jonathan op->nsegs = nsegs;
1178 1.20 tsutsui memcpy(op->segs, seg, nsegs * sizeof (seg[0]));
1179 1.1 jonathan }
1180 1.1 jonathan #endif
1181 1.1 jonathan
1182 1.1 jonathan static int
1183 1.1 jonathan ubsec_process(void *arg, struct cryptop *crp, int hint)
1184 1.1 jonathan {
1185 1.1 jonathan struct ubsec_q *q = NULL;
1186 1.1 jonathan #ifdef __OpenBSD__
1187 1.1 jonathan int card;
1188 1.1 jonathan #endif
1189 1.29 tls int err = 0, i, j, nicealign;
1190 1.1 jonathan struct ubsec_softc *sc;
1191 1.1 jonathan struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1192 1.1 jonathan int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1193 1.1 jonathan int sskip, dskip, stheend, dtheend;
1194 1.1 jonathan int16_t coffset;
1195 1.40 bad struct ubsec_session *ses, key;
1196 1.1 jonathan struct ubsec_dma *dmap = NULL;
1197 1.40 bad u_int16_t flags = 0;
1198 1.40 bad int ivlen = 0, keylen = 0;
1199 1.1 jonathan
1200 1.2 jonathan sc = arg;
1201 1.2 jonathan KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/);
1202 1.2 jonathan
1203 1.1 jonathan if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1204 1.1 jonathan ubsecstats.hst_invalid++;
1205 1.1 jonathan return (EINVAL);
1206 1.1 jonathan }
1207 1.1 jonathan if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1208 1.1 jonathan ubsecstats.hst_badsession++;
1209 1.1 jonathan return (EINVAL);
1210 1.1 jonathan }
1211 1.1 jonathan
1212 1.29 tls mutex_spin_enter(&sc->sc_mtx);
1213 1.1 jonathan
1214 1.1 jonathan if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1215 1.1 jonathan ubsecstats.hst_queuefull++;
1216 1.1 jonathan sc->sc_needwakeup |= CRYPTO_SYMQ;
1217 1.29 tls mutex_spin_exit(&sc->sc_mtx);
1218 1.1 jonathan return(ERESTART);
1219 1.1 jonathan }
1220 1.1 jonathan
1221 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1222 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next);
1223 1.29 tls mutex_spin_exit(&sc->sc_mtx);
1224 1.1 jonathan
1225 1.1 jonathan dmap = q->q_dma; /* Save dma pointer */
1226 1.39 bad /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */
1227 1.39 bad memset(q, 0, offsetof(struct ubsec_q, q_src_map));
1228 1.40 bad memset(&key, 0, sizeof(key));
1229 1.1 jonathan
1230 1.1 jonathan q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1231 1.1 jonathan q->q_dma = dmap;
1232 1.1 jonathan ses = &sc->sc_sessions[q->q_sesn];
1233 1.1 jonathan
1234 1.1 jonathan if (crp->crp_flags & CRYPTO_F_IMBUF) {
1235 1.1 jonathan q->q_src_m = (struct mbuf *)crp->crp_buf;
1236 1.1 jonathan q->q_dst_m = (struct mbuf *)crp->crp_buf;
1237 1.1 jonathan } else if (crp->crp_flags & CRYPTO_F_IOV) {
1238 1.1 jonathan q->q_src_io = (struct uio *)crp->crp_buf;
1239 1.1 jonathan q->q_dst_io = (struct uio *)crp->crp_buf;
1240 1.1 jonathan } else {
1241 1.1 jonathan ubsecstats.hst_badflags++;
1242 1.1 jonathan err = EINVAL;
1243 1.1 jonathan goto errout; /* XXX we don't handle contiguous blocks! */
1244 1.1 jonathan }
1245 1.1 jonathan
1246 1.18 cegger memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr));
1247 1.1 jonathan
1248 1.1 jonathan dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1249 1.1 jonathan dmap->d_dma->d_mcr.mcr_flags = 0;
1250 1.1 jonathan q->q_crp = crp;
1251 1.1 jonathan
1252 1.1 jonathan crd1 = crp->crp_desc;
1253 1.1 jonathan if (crd1 == NULL) {
1254 1.1 jonathan ubsecstats.hst_nodesc++;
1255 1.1 jonathan err = EINVAL;
1256 1.1 jonathan goto errout;
1257 1.1 jonathan }
1258 1.1 jonathan crd2 = crd1->crd_next;
1259 1.1 jonathan
1260 1.1 jonathan if (crd2 == NULL) {
1261 1.15 tls if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
1262 1.15 tls crd1->crd_alg == CRYPTO_SHA1_HMAC_96) {
1263 1.1 jonathan maccrd = crd1;
1264 1.1 jonathan enccrd = NULL;
1265 1.1 jonathan } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1266 1.40 bad crd1->crd_alg == CRYPTO_3DES_CBC ||
1267 1.40 bad crd1->crd_alg == CRYPTO_AES_CBC) {
1268 1.1 jonathan maccrd = NULL;
1269 1.1 jonathan enccrd = crd1;
1270 1.1 jonathan } else {
1271 1.1 jonathan ubsecstats.hst_badalg++;
1272 1.1 jonathan err = EINVAL;
1273 1.1 jonathan goto errout;
1274 1.1 jonathan }
1275 1.1 jonathan } else {
1276 1.15 tls if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 ||
1277 1.15 tls crd1->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1278 1.1 jonathan (crd2->crd_alg == CRYPTO_DES_CBC ||
1279 1.40 bad crd2->crd_alg == CRYPTO_3DES_CBC ||
1280 1.40 bad crd2->crd_alg == CRYPTO_AES_CBC) &&
1281 1.1 jonathan ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1282 1.1 jonathan maccrd = crd1;
1283 1.1 jonathan enccrd = crd2;
1284 1.1 jonathan } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1285 1.40 bad crd1->crd_alg == CRYPTO_3DES_CBC ||
1286 1.40 bad crd1->crd_alg == CRYPTO_AES_CBC) &&
1287 1.15 tls (crd2->crd_alg == CRYPTO_MD5_HMAC_96 ||
1288 1.40 bad crd2->crd_alg == CRYPTO_SHA1_HMAC_96) &&
1289 1.1 jonathan (crd1->crd_flags & CRD_F_ENCRYPT)) {
1290 1.1 jonathan enccrd = crd1;
1291 1.1 jonathan maccrd = crd2;
1292 1.1 jonathan } else {
1293 1.1 jonathan /*
1294 1.1 jonathan * We cannot order the ubsec as requested
1295 1.1 jonathan */
1296 1.1 jonathan ubsecstats.hst_badalg++;
1297 1.1 jonathan err = EINVAL;
1298 1.1 jonathan goto errout;
1299 1.1 jonathan }
1300 1.1 jonathan }
1301 1.1 jonathan
1302 1.1 jonathan if (enccrd) {
1303 1.40 bad if (enccrd->crd_alg == CRYPTO_AES_CBC) {
1304 1.40 bad if ((sc->sc_flags & UBS_FLAGS_AES) == 0) {
1305 1.40 bad /*
1306 1.40 bad * We cannot order the ubsec as requested
1307 1.40 bad */
1308 1.40 bad ubsecstats.hst_badalg++;
1309 1.40 bad err = EINVAL;
1310 1.40 bad goto errout;
1311 1.40 bad }
1312 1.40 bad flags |= htole16(UBS_PKTCTX_ENC_AES);
1313 1.40 bad switch (enccrd->crd_klen) {
1314 1.40 bad case 128:
1315 1.40 bad case 192:
1316 1.40 bad case 256:
1317 1.40 bad keylen = enccrd->crd_klen / 8;
1318 1.40 bad break;
1319 1.40 bad default:
1320 1.40 bad err = EINVAL;
1321 1.40 bad goto errout;
1322 1.40 bad }
1323 1.40 bad ivlen = 16;
1324 1.40 bad } else {
1325 1.40 bad flags |= htole16(UBS_PKTCTX_ENC_3DES);
1326 1.40 bad ivlen = 8;
1327 1.40 bad keylen = 24;
1328 1.40 bad }
1329 1.40 bad
1330 1.1 jonathan encoffset = enccrd->crd_skip;
1331 1.1 jonathan
1332 1.1 jonathan if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1333 1.1 jonathan q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1334 1.1 jonathan
1335 1.1 jonathan if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1336 1.40 bad memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1337 1.1 jonathan else {
1338 1.40 bad for (i = 0; i < (ivlen / 4); i++)
1339 1.40 bad key.ses_iv[i] = ses->ses_iv[i];
1340 1.1 jonathan }
1341 1.1 jonathan
1342 1.1 jonathan if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1343 1.1 jonathan if (crp->crp_flags & CRYPTO_F_IMBUF)
1344 1.1 jonathan m_copyback(q->q_src_m,
1345 1.1 jonathan enccrd->crd_inject,
1346 1.40 bad ivlen, (void *)key.ses_iv);
1347 1.1 jonathan else if (crp->crp_flags & CRYPTO_F_IOV)
1348 1.1 jonathan cuio_copyback(q->q_src_io,
1349 1.1 jonathan enccrd->crd_inject,
1350 1.40 bad ivlen, (void *)key.ses_iv);
1351 1.1 jonathan }
1352 1.1 jonathan } else {
1353 1.40 bad flags |= htole16(UBS_PKTCTX_INBOUND);
1354 1.1 jonathan
1355 1.1 jonathan if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1356 1.40 bad memcpy(key.ses_iv, enccrd->crd_iv, ivlen);
1357 1.1 jonathan else if (crp->crp_flags & CRYPTO_F_IMBUF)
1358 1.1 jonathan m_copydata(q->q_src_m, enccrd->crd_inject,
1359 1.40 bad ivlen, (void *)key.ses_iv);
1360 1.1 jonathan else if (crp->crp_flags & CRYPTO_F_IOV)
1361 1.1 jonathan cuio_copydata(q->q_src_io,
1362 1.1 jonathan enccrd->crd_inject, 8,
1363 1.40 bad (void *)key.ses_iv);
1364 1.1 jonathan }
1365 1.1 jonathan
1366 1.40 bad for (i = 0; i < (keylen / 4); i++)
1367 1.40 bad key.ses_key[i] = ses->ses_key[i];
1368 1.40 bad for (i = 0; i < (ivlen / 4); i++)
1369 1.40 bad SWAP32(key.ses_iv[i]);
1370 1.1 jonathan }
1371 1.1 jonathan
1372 1.1 jonathan if (maccrd) {
1373 1.1 jonathan macoffset = maccrd->crd_skip;
1374 1.1 jonathan
1375 1.15 tls if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96)
1376 1.40 bad flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1377 1.1 jonathan else
1378 1.40 bad flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1379 1.1 jonathan
1380 1.1 jonathan for (i = 0; i < 5; i++) {
1381 1.40 bad key.ses_hminner[i] = ses->ses_hminner[i];
1382 1.40 bad key.ses_hmouter[i] = ses->ses_hmouter[i];
1383 1.1 jonathan
1384 1.40 bad HTOLE32(key.ses_hminner[i]);
1385 1.40 bad HTOLE32(key.ses_hmouter[i]);
1386 1.1 jonathan }
1387 1.1 jonathan }
1388 1.1 jonathan
1389 1.1 jonathan if (enccrd && maccrd) {
1390 1.1 jonathan /*
1391 1.1 jonathan * ubsec cannot handle packets where the end of encryption
1392 1.1 jonathan * and authentication are not the same, or where the
1393 1.1 jonathan * encrypted part begins before the authenticated part.
1394 1.1 jonathan */
1395 1.1 jonathan if ((encoffset + enccrd->crd_len) !=
1396 1.1 jonathan (macoffset + maccrd->crd_len)) {
1397 1.1 jonathan ubsecstats.hst_lenmismatch++;
1398 1.1 jonathan err = EINVAL;
1399 1.1 jonathan goto errout;
1400 1.1 jonathan }
1401 1.1 jonathan if (enccrd->crd_skip < maccrd->crd_skip) {
1402 1.1 jonathan ubsecstats.hst_skipmismatch++;
1403 1.1 jonathan err = EINVAL;
1404 1.1 jonathan goto errout;
1405 1.1 jonathan }
1406 1.1 jonathan sskip = maccrd->crd_skip;
1407 1.1 jonathan cpskip = dskip = enccrd->crd_skip;
1408 1.1 jonathan stheend = maccrd->crd_len;
1409 1.1 jonathan dtheend = enccrd->crd_len;
1410 1.1 jonathan coffset = enccrd->crd_skip - maccrd->crd_skip;
1411 1.1 jonathan cpoffset = cpskip + dtheend;
1412 1.1 jonathan #ifdef UBSEC_DEBUG
1413 1.1 jonathan if (ubsec_debug) {
1414 1.1 jonathan printf("mac: skip %d, len %d, inject %d\n",
1415 1.1 jonathan maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1416 1.1 jonathan printf("enc: skip %d, len %d, inject %d\n",
1417 1.1 jonathan enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1418 1.1 jonathan printf("src: skip %d, len %d\n", sskip, stheend);
1419 1.1 jonathan printf("dst: skip %d, len %d\n", dskip, dtheend);
1420 1.1 jonathan printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1421 1.1 jonathan coffset, stheend, cpskip, cpoffset);
1422 1.1 jonathan }
1423 1.1 jonathan #endif
1424 1.1 jonathan } else {
1425 1.1 jonathan cpskip = dskip = sskip = macoffset + encoffset;
1426 1.1 jonathan dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1427 1.1 jonathan cpoffset = cpskip + dtheend;
1428 1.1 jonathan coffset = 0;
1429 1.1 jonathan }
1430 1.1 jonathan
1431 1.39 bad if (q->q_src_map == NULL) {
1432 1.39 bad /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */
1433 1.39 bad if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER,
1434 1.39 bad 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) {
1435 1.39 bad err = ENOMEM;
1436 1.39 bad goto errout;
1437 1.39 bad }
1438 1.1 jonathan }
1439 1.1 jonathan if (crp->crp_flags & CRYPTO_F_IMBUF) {
1440 1.1 jonathan if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1441 1.1 jonathan q->q_src_m, BUS_DMA_NOWAIT) != 0) {
1442 1.1 jonathan ubsecstats.hst_noload++;
1443 1.1 jonathan err = ENOMEM;
1444 1.1 jonathan goto errout;
1445 1.1 jonathan }
1446 1.1 jonathan } else if (crp->crp_flags & CRYPTO_F_IOV) {
1447 1.1 jonathan if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1448 1.1 jonathan q->q_src_io, BUS_DMA_NOWAIT) != 0) {
1449 1.1 jonathan ubsecstats.hst_noload++;
1450 1.1 jonathan err = ENOMEM;
1451 1.1 jonathan goto errout;
1452 1.1 jonathan }
1453 1.1 jonathan }
1454 1.1 jonathan nicealign = ubsec_dmamap_aligned(q->q_src_map);
1455 1.1 jonathan
1456 1.1 jonathan dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1457 1.1 jonathan
1458 1.1 jonathan #ifdef UBSEC_DEBUG
1459 1.1 jonathan if (ubsec_debug)
1460 1.1 jonathan printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1461 1.1 jonathan #endif
1462 1.1 jonathan for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) {
1463 1.1 jonathan struct ubsec_pktbuf *pb;
1464 1.1 jonathan bus_size_t packl = q->q_src_map->dm_segs[i].ds_len;
1465 1.1 jonathan bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr;
1466 1.1 jonathan
1467 1.1 jonathan if (sskip >= packl) {
1468 1.1 jonathan sskip -= packl;
1469 1.1 jonathan continue;
1470 1.1 jonathan }
1471 1.1 jonathan
1472 1.1 jonathan packl -= sskip;
1473 1.1 jonathan packp += sskip;
1474 1.1 jonathan sskip = 0;
1475 1.1 jonathan
1476 1.1 jonathan if (packl > 0xfffc) {
1477 1.1 jonathan err = EIO;
1478 1.1 jonathan goto errout;
1479 1.1 jonathan }
1480 1.1 jonathan
1481 1.1 jonathan if (j == 0)
1482 1.1 jonathan pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1483 1.1 jonathan else
1484 1.1 jonathan pb = &dmap->d_dma->d_sbuf[j - 1];
1485 1.1 jonathan
1486 1.1 jonathan pb->pb_addr = htole32(packp);
1487 1.1 jonathan
1488 1.1 jonathan if (stheend) {
1489 1.1 jonathan if (packl > stheend) {
1490 1.1 jonathan pb->pb_len = htole32(stheend);
1491 1.1 jonathan stheend = 0;
1492 1.1 jonathan } else {
1493 1.1 jonathan pb->pb_len = htole32(packl);
1494 1.1 jonathan stheend -= packl;
1495 1.1 jonathan }
1496 1.1 jonathan } else
1497 1.1 jonathan pb->pb_len = htole32(packl);
1498 1.1 jonathan
1499 1.1 jonathan if ((i + 1) == q->q_src_map->dm_nsegs)
1500 1.1 jonathan pb->pb_next = 0;
1501 1.1 jonathan else
1502 1.1 jonathan pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1503 1.1 jonathan offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1504 1.1 jonathan j++;
1505 1.1 jonathan }
1506 1.1 jonathan
1507 1.1 jonathan if (enccrd == NULL && maccrd != NULL) {
1508 1.1 jonathan dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1509 1.1 jonathan dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1510 1.1 jonathan dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1511 1.1 jonathan offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1512 1.1 jonathan #ifdef UBSEC_DEBUG
1513 1.1 jonathan if (ubsec_debug)
1514 1.1 jonathan printf("opkt: %x %x %x\n",
1515 1.1 jonathan dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1516 1.1 jonathan dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1517 1.1 jonathan dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1518 1.1 jonathan
1519 1.1 jonathan #endif
1520 1.1 jonathan } else {
1521 1.1 jonathan if (crp->crp_flags & CRYPTO_F_IOV) {
1522 1.1 jonathan if (!nicealign) {
1523 1.1 jonathan ubsecstats.hst_iovmisaligned++;
1524 1.1 jonathan err = EINVAL;
1525 1.1 jonathan goto errout;
1526 1.1 jonathan }
1527 1.39 bad if (q->q_dst_map == NULL) {
1528 1.39 bad if (q->q_cached_dst_map == NULL) {
1529 1.39 bad /* XXX: ``what the heck's that'' 0xfff0? */
1530 1.39 bad if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1531 1.39 bad UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1532 1.39 bad &q->q_cached_dst_map) != 0) {
1533 1.39 bad ubsecstats.hst_nomap++;
1534 1.39 bad err = ENOMEM;
1535 1.39 bad goto errout;
1536 1.39 bad }
1537 1.39 bad }
1538 1.39 bad q->q_dst_map = q->q_cached_dst_map;
1539 1.1 jonathan }
1540 1.1 jonathan if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1541 1.1 jonathan q->q_dst_io, BUS_DMA_NOWAIT) != 0) {
1542 1.1 jonathan ubsecstats.hst_noload++;
1543 1.1 jonathan err = ENOMEM;
1544 1.1 jonathan goto errout;
1545 1.1 jonathan }
1546 1.1 jonathan } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1547 1.1 jonathan if (nicealign) {
1548 1.1 jonathan q->q_dst_m = q->q_src_m;
1549 1.1 jonathan q->q_dst_map = q->q_src_map;
1550 1.1 jonathan } else {
1551 1.1 jonathan int totlen, len;
1552 1.1 jonathan struct mbuf *m, *top, **mp;
1553 1.1 jonathan
1554 1.1 jonathan ubsecstats.hst_unaligned++;
1555 1.1 jonathan totlen = q->q_src_map->dm_mapsize;
1556 1.1 jonathan if (q->q_src_m->m_flags & M_PKTHDR) {
1557 1.1 jonathan len = MHLEN;
1558 1.1 jonathan MGETHDR(m, M_DONTWAIT, MT_DATA);
1559 1.1 jonathan /*XXX FIXME: m_dup_pkthdr */
1560 1.1 jonathan if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) {
1561 1.1 jonathan m_free(m);
1562 1.1 jonathan m = NULL;
1563 1.1 jonathan }
1564 1.1 jonathan } else {
1565 1.1 jonathan len = MLEN;
1566 1.1 jonathan MGET(m, M_DONTWAIT, MT_DATA);
1567 1.1 jonathan }
1568 1.1 jonathan if (m == NULL) {
1569 1.1 jonathan ubsecstats.hst_nombuf++;
1570 1.1 jonathan err = sc->sc_nqueue ? ERESTART : ENOMEM;
1571 1.1 jonathan goto errout;
1572 1.1 jonathan }
1573 1.1 jonathan if (len == MHLEN)
1574 1.1 jonathan /*XXX was M_DUP_PKTHDR*/
1575 1.1 jonathan M_COPY_PKTHDR(m, q->q_src_m);
1576 1.1 jonathan if (totlen >= MINCLSIZE) {
1577 1.1 jonathan MCLGET(m, M_DONTWAIT);
1578 1.1 jonathan if ((m->m_flags & M_EXT) == 0) {
1579 1.1 jonathan m_free(m);
1580 1.1 jonathan ubsecstats.hst_nomcl++;
1581 1.1 jonathan err = sc->sc_nqueue ? ERESTART : ENOMEM;
1582 1.1 jonathan goto errout;
1583 1.1 jonathan }
1584 1.1 jonathan len = MCLBYTES;
1585 1.1 jonathan }
1586 1.1 jonathan m->m_len = len;
1587 1.1 jonathan top = NULL;
1588 1.1 jonathan mp = ⊤
1589 1.1 jonathan
1590 1.1 jonathan while (totlen > 0) {
1591 1.1 jonathan if (top) {
1592 1.1 jonathan MGET(m, M_DONTWAIT, MT_DATA);
1593 1.1 jonathan if (m == NULL) {
1594 1.1 jonathan m_freem(top);
1595 1.1 jonathan ubsecstats.hst_nombuf++;
1596 1.1 jonathan err = sc->sc_nqueue ? ERESTART : ENOMEM;
1597 1.1 jonathan goto errout;
1598 1.1 jonathan }
1599 1.1 jonathan len = MLEN;
1600 1.1 jonathan }
1601 1.1 jonathan if (top && totlen >= MINCLSIZE) {
1602 1.1 jonathan MCLGET(m, M_DONTWAIT);
1603 1.1 jonathan if ((m->m_flags & M_EXT) == 0) {
1604 1.1 jonathan *mp = m;
1605 1.1 jonathan m_freem(top);
1606 1.1 jonathan ubsecstats.hst_nomcl++;
1607 1.1 jonathan err = sc->sc_nqueue ? ERESTART : ENOMEM;
1608 1.1 jonathan goto errout;
1609 1.1 jonathan }
1610 1.1 jonathan len = MCLBYTES;
1611 1.1 jonathan }
1612 1.1 jonathan m->m_len = len = min(totlen, len);
1613 1.1 jonathan totlen -= len;
1614 1.1 jonathan *mp = m;
1615 1.1 jonathan mp = &m->m_next;
1616 1.1 jonathan }
1617 1.1 jonathan q->q_dst_m = top;
1618 1.1 jonathan ubsec_mcopy(q->q_src_m, q->q_dst_m,
1619 1.1 jonathan cpskip, cpoffset);
1620 1.39 bad if (q->q_dst_map == NULL) {
1621 1.39 bad if (q->q_cached_dst_map == NULL) {
1622 1.39 bad /* XXX again, what the heck is that 0xfff0? */
1623 1.39 bad if (bus_dmamap_create(sc->sc_dmat, 0xfff0,
1624 1.39 bad UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT,
1625 1.39 bad &q->q_cached_dst_map) != 0) {
1626 1.39 bad ubsecstats.hst_nomap++;
1627 1.39 bad err = ENOMEM;
1628 1.39 bad goto errout;
1629 1.39 bad }
1630 1.39 bad }
1631 1.39 bad q->q_dst_map = q->q_cached_dst_map;
1632 1.1 jonathan }
1633 1.1 jonathan if (bus_dmamap_load_mbuf(sc->sc_dmat,
1634 1.1 jonathan q->q_dst_map, q->q_dst_m,
1635 1.1 jonathan BUS_DMA_NOWAIT) != 0) {
1636 1.1 jonathan ubsecstats.hst_noload++;
1637 1.1 jonathan err = ENOMEM;
1638 1.1 jonathan goto errout;
1639 1.1 jonathan }
1640 1.1 jonathan }
1641 1.1 jonathan } else {
1642 1.1 jonathan ubsecstats.hst_badflags++;
1643 1.1 jonathan err = EINVAL;
1644 1.1 jonathan goto errout;
1645 1.1 jonathan }
1646 1.1 jonathan
1647 1.1 jonathan #ifdef UBSEC_DEBUG
1648 1.1 jonathan if (ubsec_debug)
1649 1.1 jonathan printf("dst skip: %d\n", dskip);
1650 1.1 jonathan #endif
1651 1.1 jonathan for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) {
1652 1.1 jonathan struct ubsec_pktbuf *pb;
1653 1.1 jonathan bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len;
1654 1.1 jonathan bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr;
1655 1.1 jonathan
1656 1.1 jonathan if (dskip >= packl) {
1657 1.1 jonathan dskip -= packl;
1658 1.1 jonathan continue;
1659 1.1 jonathan }
1660 1.1 jonathan
1661 1.1 jonathan packl -= dskip;
1662 1.1 jonathan packp += dskip;
1663 1.1 jonathan dskip = 0;
1664 1.1 jonathan
1665 1.1 jonathan if (packl > 0xfffc) {
1666 1.1 jonathan err = EIO;
1667 1.1 jonathan goto errout;
1668 1.1 jonathan }
1669 1.1 jonathan
1670 1.1 jonathan if (j == 0)
1671 1.1 jonathan pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1672 1.1 jonathan else
1673 1.1 jonathan pb = &dmap->d_dma->d_dbuf[j - 1];
1674 1.1 jonathan
1675 1.1 jonathan pb->pb_addr = htole32(packp);
1676 1.1 jonathan
1677 1.1 jonathan if (dtheend) {
1678 1.1 jonathan if (packl > dtheend) {
1679 1.1 jonathan pb->pb_len = htole32(dtheend);
1680 1.1 jonathan dtheend = 0;
1681 1.1 jonathan } else {
1682 1.1 jonathan pb->pb_len = htole32(packl);
1683 1.1 jonathan dtheend -= packl;
1684 1.1 jonathan }
1685 1.1 jonathan } else
1686 1.1 jonathan pb->pb_len = htole32(packl);
1687 1.1 jonathan
1688 1.1 jonathan if ((i + 1) == q->q_dst_map->dm_nsegs) {
1689 1.1 jonathan if (maccrd)
1690 1.1 jonathan pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1691 1.1 jonathan offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1692 1.1 jonathan else
1693 1.1 jonathan pb->pb_next = 0;
1694 1.1 jonathan } else
1695 1.1 jonathan pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1696 1.1 jonathan offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1697 1.1 jonathan j++;
1698 1.1 jonathan }
1699 1.1 jonathan }
1700 1.1 jonathan
1701 1.1 jonathan dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1702 1.1 jonathan offsetof(struct ubsec_dmachunk, d_ctx));
1703 1.1 jonathan
1704 1.40 bad if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) {
1705 1.40 bad struct ubsec_pktctx_aes128 *aes128;
1706 1.40 bad struct ubsec_pktctx_aes192 *aes192;
1707 1.40 bad struct ubsec_pktctx_aes256 *aes256;
1708 1.40 bad struct ubsec_pktctx_hdr *ph;
1709 1.40 bad u_int8_t *ctx;
1710 1.40 bad
1711 1.40 bad ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1712 1.40 bad offsetof(struct ubsec_dmachunk, d_ctx);
1713 1.40 bad
1714 1.40 bad ph = (struct ubsec_pktctx_hdr *)ctx;
1715 1.40 bad ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES);
1716 1.40 bad ph->ph_flags = flags;
1717 1.40 bad ph->ph_offset = htole16(coffset >> 2);
1718 1.40 bad
1719 1.40 bad switch (enccrd->crd_klen) {
1720 1.40 bad case 128:
1721 1.40 bad aes128 = (struct ubsec_pktctx_aes128 *)ctx;
1722 1.40 bad ph->ph_len = htole16(sizeof(*aes128));
1723 1.40 bad ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128);
1724 1.40 bad for (i = 0; i < 4; i++)
1725 1.40 bad aes128->pc_aeskey[i] = key.ses_key[i];
1726 1.40 bad for (i = 0; i < 5; i++)
1727 1.40 bad aes128->pc_hminner[i] = key.ses_hminner[i];
1728 1.40 bad for (i = 0; i < 5; i++)
1729 1.40 bad aes128->pc_hmouter[i] = key.ses_hmouter[i];
1730 1.40 bad for (i = 0; i < 4; i++)
1731 1.40 bad aes128->pc_iv[i] = key.ses_iv[i];
1732 1.40 bad break;
1733 1.40 bad case 192:
1734 1.40 bad aes192 = (struct ubsec_pktctx_aes192 *)ctx;
1735 1.40 bad ph->ph_len = htole16(sizeof(*aes192));
1736 1.40 bad ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192);
1737 1.40 bad for (i = 0; i < 6; i++)
1738 1.40 bad aes192->pc_aeskey[i] = key.ses_key[i];
1739 1.40 bad for (i = 0; i < 5; i++)
1740 1.40 bad aes192->pc_hminner[i] = key.ses_hminner[i];
1741 1.40 bad for (i = 0; i < 5; i++)
1742 1.40 bad aes192->pc_hmouter[i] = key.ses_hmouter[i];
1743 1.40 bad for (i = 0; i < 4; i++)
1744 1.40 bad aes192->pc_iv[i] = key.ses_iv[i];
1745 1.40 bad break;
1746 1.40 bad case 256:
1747 1.40 bad aes256 = (struct ubsec_pktctx_aes256 *)ctx;
1748 1.40 bad ph->ph_len = htole16(sizeof(*aes256));
1749 1.40 bad ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256);
1750 1.40 bad for (i = 0; i < 8; i++)
1751 1.40 bad aes256->pc_aeskey[i] = key.ses_key[i];
1752 1.40 bad for (i = 0; i < 5; i++)
1753 1.40 bad aes256->pc_hminner[i] = key.ses_hminner[i];
1754 1.40 bad for (i = 0; i < 5; i++)
1755 1.40 bad aes256->pc_hmouter[i] = key.ses_hmouter[i];
1756 1.40 bad for (i = 0; i < 4; i++)
1757 1.40 bad aes256->pc_iv[i] = key.ses_iv[i];
1758 1.40 bad break;
1759 1.40 bad }
1760 1.40 bad } else if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1761 1.40 bad struct ubsec_pktctx_3des *ctx;
1762 1.40 bad struct ubsec_pktctx_hdr *ph;
1763 1.1 jonathan
1764 1.40 bad ctx = (struct ubsec_pktctx_3des *)
1765 1.40 bad ((u_int8_t *)(dmap->d_alloc.dma_vaddr) +
1766 1.1 jonathan offsetof(struct ubsec_dmachunk, d_ctx));
1767 1.5 perry
1768 1.40 bad ph = (struct ubsec_pktctx_hdr *)ctx;
1769 1.40 bad ph->ph_len = htole16(sizeof(*ctx));
1770 1.40 bad ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES);
1771 1.40 bad ph->ph_flags = flags;
1772 1.40 bad ph->ph_offset = htole16(coffset >> 2);
1773 1.40 bad
1774 1.1 jonathan for (i = 0; i < 6; i++)
1775 1.40 bad ctx->pc_deskey[i] = key.ses_key[i];
1776 1.1 jonathan for (i = 0; i < 5; i++)
1777 1.40 bad ctx->pc_hminner[i] = key.ses_hminner[i];
1778 1.1 jonathan for (i = 0; i < 5; i++)
1779 1.40 bad ctx->pc_hmouter[i] = key.ses_hmouter[i];
1780 1.40 bad for (i = 0; i < 2; i++)
1781 1.40 bad ctx->pc_iv[i] = key.ses_iv[i];
1782 1.40 bad } else {
1783 1.40 bad struct ubsec_pktctx *ctx = (struct ubsec_pktctx *)
1784 1.40 bad ((u_int8_t *)dmap->d_alloc.dma_vaddr +
1785 1.40 bad offsetof(struct ubsec_dmachunk, d_ctx));
1786 1.40 bad
1787 1.40 bad ctx->pc_flags = flags;
1788 1.40 bad ctx->pc_offset = htole16(coffset >> 2);
1789 1.40 bad for (i = 0; i < 6; i++)
1790 1.40 bad ctx->pc_deskey[i] = key.ses_key[i];
1791 1.40 bad for (i = 0; i < 5; i++)
1792 1.40 bad ctx->pc_hminner[i] = key.ses_hminner[i];
1793 1.40 bad for (i = 0; i < 5; i++)
1794 1.40 bad ctx->pc_hmouter[i] = key.ses_hmouter[i];
1795 1.40 bad for (i = 0; i < 2; i++)
1796 1.40 bad ctx->pc_iv[i] = key.ses_iv[i];
1797 1.40 bad }
1798 1.1 jonathan
1799 1.29 tls mutex_spin_enter(&sc->sc_mtx);
1800 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1801 1.1 jonathan sc->sc_nqueue++;
1802 1.1 jonathan ubsecstats.hst_ipackets++;
1803 1.1 jonathan ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize;
1804 1.1 jonathan if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch)
1805 1.1 jonathan ubsec_feed(sc);
1806 1.29 tls mutex_spin_exit(&sc->sc_mtx);
1807 1.1 jonathan return (0);
1808 1.1 jonathan
1809 1.1 jonathan errout:
1810 1.1 jonathan if (q != NULL) {
1811 1.1 jonathan if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1812 1.1 jonathan m_freem(q->q_dst_m);
1813 1.1 jonathan
1814 1.1 jonathan if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1815 1.1 jonathan bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1816 1.1 jonathan }
1817 1.1 jonathan if (q->q_src_map != NULL) {
1818 1.1 jonathan bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1819 1.1 jonathan }
1820 1.1 jonathan
1821 1.29 tls mutex_spin_enter(&sc->sc_mtx);
1822 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1823 1.29 tls mutex_spin_exit(&sc->sc_mtx);
1824 1.1 jonathan }
1825 1.1 jonathan #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */
1826 1.1 jonathan if (err == EINVAL)
1827 1.1 jonathan ubsecstats.hst_invalid++;
1828 1.1 jonathan else
1829 1.1 jonathan ubsecstats.hst_nomem++;
1830 1.5 perry #endif
1831 1.1 jonathan if (err != ERESTART) {
1832 1.1 jonathan crp->crp_etype = err;
1833 1.1 jonathan crypto_done(crp);
1834 1.1 jonathan } else {
1835 1.1 jonathan sc->sc_needwakeup |= CRYPTO_SYMQ;
1836 1.1 jonathan }
1837 1.1 jonathan return (err);
1838 1.1 jonathan }
1839 1.1 jonathan
1840 1.7 thorpej static void
1841 1.7 thorpej ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1842 1.1 jonathan {
1843 1.1 jonathan struct cryptop *crp = (struct cryptop *)q->q_crp;
1844 1.1 jonathan struct cryptodesc *crd;
1845 1.1 jonathan struct ubsec_dma *dmap = q->q_dma;
1846 1.1 jonathan
1847 1.1 jonathan ubsecstats.hst_opackets++;
1848 1.1 jonathan ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1849 1.1 jonathan
1850 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0,
1851 1.1 jonathan dmap->d_alloc.dma_map->dm_mapsize,
1852 1.1 jonathan BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1853 1.1 jonathan if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1854 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1855 1.1 jonathan 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1856 1.1 jonathan bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1857 1.1 jonathan }
1858 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_src_map,
1859 1.1 jonathan 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1860 1.1 jonathan bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1861 1.1 jonathan
1862 1.1 jonathan if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1863 1.1 jonathan m_freem(q->q_src_m);
1864 1.12 christos crp->crp_buf = (void *)q->q_dst_m;
1865 1.1 jonathan }
1866 1.1 jonathan
1867 1.1 jonathan /* copy out IV for future use */
1868 1.1 jonathan if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1869 1.1 jonathan for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1870 1.1 jonathan if (crd->crd_alg != CRYPTO_DES_CBC &&
1871 1.40 bad crd->crd_alg != CRYPTO_3DES_CBC &&
1872 1.40 bad crd->crd_alg != CRYPTO_AES_CBC)
1873 1.1 jonathan continue;
1874 1.1 jonathan if (crp->crp_flags & CRYPTO_F_IMBUF)
1875 1.1 jonathan m_copydata((struct mbuf *)crp->crp_buf,
1876 1.1 jonathan crd->crd_skip + crd->crd_len - 8, 8,
1877 1.12 christos (void *)sc->sc_sessions[q->q_sesn].ses_iv);
1878 1.1 jonathan else if (crp->crp_flags & CRYPTO_F_IOV) {
1879 1.1 jonathan cuio_copydata((struct uio *)crp->crp_buf,
1880 1.1 jonathan crd->crd_skip + crd->crd_len - 8, 8,
1881 1.12 christos (void *)sc->sc_sessions[q->q_sesn].ses_iv);
1882 1.1 jonathan }
1883 1.1 jonathan break;
1884 1.1 jonathan }
1885 1.1 jonathan }
1886 1.1 jonathan
1887 1.1 jonathan for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1888 1.15 tls if (crd->crd_alg != CRYPTO_MD5_HMAC_96 &&
1889 1.15 tls crd->crd_alg != CRYPTO_SHA1_HMAC_96)
1890 1.1 jonathan continue;
1891 1.1 jonathan if (crp->crp_flags & CRYPTO_F_IMBUF)
1892 1.1 jonathan m_copyback((struct mbuf *)crp->crp_buf,
1893 1.1 jonathan crd->crd_inject, 12,
1894 1.12 christos (void *)dmap->d_dma->d_macbuf);
1895 1.1 jonathan else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1896 1.12 christos bcopy((void *)dmap->d_dma->d_macbuf,
1897 1.1 jonathan crp->crp_mac, 12);
1898 1.1 jonathan break;
1899 1.1 jonathan }
1900 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1901 1.1 jonathan crypto_done(crp);
1902 1.1 jonathan }
1903 1.1 jonathan
1904 1.1 jonathan static void
1905 1.1 jonathan ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1906 1.1 jonathan {
1907 1.1 jonathan int i, j, dlen, slen;
1908 1.12 christos char *dptr, *sptr;
1909 1.1 jonathan
1910 1.1 jonathan j = 0;
1911 1.1 jonathan sptr = srcm->m_data;
1912 1.1 jonathan slen = srcm->m_len;
1913 1.1 jonathan dptr = dstm->m_data;
1914 1.1 jonathan dlen = dstm->m_len;
1915 1.1 jonathan
1916 1.1 jonathan while (1) {
1917 1.1 jonathan for (i = 0; i < min(slen, dlen); i++) {
1918 1.1 jonathan if (j < hoffset || j >= toffset)
1919 1.1 jonathan *dptr++ = *sptr++;
1920 1.1 jonathan slen--;
1921 1.1 jonathan dlen--;
1922 1.1 jonathan j++;
1923 1.1 jonathan }
1924 1.1 jonathan if (slen == 0) {
1925 1.1 jonathan srcm = srcm->m_next;
1926 1.1 jonathan if (srcm == NULL)
1927 1.1 jonathan return;
1928 1.1 jonathan sptr = srcm->m_data;
1929 1.1 jonathan slen = srcm->m_len;
1930 1.1 jonathan }
1931 1.1 jonathan if (dlen == 0) {
1932 1.1 jonathan dstm = dstm->m_next;
1933 1.1 jonathan if (dstm == NULL)
1934 1.1 jonathan return;
1935 1.1 jonathan dptr = dstm->m_data;
1936 1.1 jonathan dlen = dstm->m_len;
1937 1.1 jonathan }
1938 1.1 jonathan }
1939 1.1 jonathan }
1940 1.1 jonathan
1941 1.1 jonathan /*
1942 1.1 jonathan * feed the key generator, must be called at splnet() or higher.
1943 1.1 jonathan */
1944 1.1 jonathan static void
1945 1.1 jonathan ubsec_feed2(struct ubsec_softc *sc)
1946 1.1 jonathan {
1947 1.1 jonathan struct ubsec_q2 *q;
1948 1.1 jonathan
1949 1.1 jonathan while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1950 1.1 jonathan if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1951 1.1 jonathan break;
1952 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_queue2);
1953 1.1 jonathan
1954 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
1955 1.1 jonathan q->q_mcr.dma_map->dm_mapsize,
1956 1.1 jonathan BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1957 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1958 1.1 jonathan q->q_ctx.dma_map->dm_mapsize,
1959 1.1 jonathan BUS_DMASYNC_PREWRITE);
1960 1.1 jonathan
1961 1.1 jonathan WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1962 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_queue2);
1963 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next);
1964 1.1 jonathan --sc->sc_nqueue2;
1965 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1966 1.1 jonathan }
1967 1.1 jonathan }
1968 1.1 jonathan
1969 1.1 jonathan /*
1970 1.34 bad * feed the RNG (used instead of ubsec_feed2() on 5827+ devices)
1971 1.34 bad */
1972 1.34 bad void
1973 1.34 bad ubsec_feed4(struct ubsec_softc *sc)
1974 1.34 bad {
1975 1.34 bad struct ubsec_q2 *q;
1976 1.34 bad
1977 1.34 bad while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) {
1978 1.34 bad if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL)
1979 1.34 bad break;
1980 1.34 bad q = SIMPLEQ_FIRST(&sc->sc_queue4);
1981 1.34 bad
1982 1.34 bad bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0,
1983 1.34 bad q->q_mcr.dma_map->dm_mapsize,
1984 1.34 bad BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1985 1.34 bad bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
1986 1.34 bad q->q_ctx.dma_map->dm_mapsize,
1987 1.34 bad BUS_DMASYNC_PREWRITE);
1988 1.34 bad
1989 1.34 bad WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr);
1990 1.34 bad SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next);
1991 1.34 bad --sc->sc_nqueue4;
1992 1.34 bad SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next);
1993 1.34 bad }
1994 1.34 bad }
1995 1.34 bad
1996 1.34 bad /*
1997 1.1 jonathan * Callback for handling random numbers
1998 1.1 jonathan */
1999 1.1 jonathan static void
2000 1.1 jonathan ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
2001 1.1 jonathan {
2002 1.1 jonathan struct cryptkop *krp;
2003 1.1 jonathan struct ubsec_ctx_keyop *ctx;
2004 1.1 jonathan
2005 1.1 jonathan ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
2006 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0,
2007 1.1 jonathan q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2008 1.1 jonathan
2009 1.1 jonathan switch (q->q_type) {
2010 1.1 jonathan #ifndef UBSEC_NO_RNG
2011 1.1 jonathan case UBS_CTXOP_RNGSHA1:
2012 1.1 jonathan case UBS_CTXOP_RNGBYPASS: {
2013 1.1 jonathan struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
2014 1.1 jonathan u_int32_t *p;
2015 1.1 jonathan int i;
2016 1.1 jonathan
2017 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
2018 1.1 jonathan rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2019 1.1 jonathan p = (u_int32_t *)rng->rng_buf.dma_vaddr;
2020 1.1 jonathan #ifndef __NetBSD__
2021 1.1 jonathan for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++)
2022 1.1 jonathan add_true_randomness(letoh32(*p));
2023 1.1 jonathan #else
2024 1.29 tls i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t);
2025 1.29 tls rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY);
2026 1.29 tls sc->sc_rng_need -= i;
2027 1.34 bad #endif
2028 1.29 tls rng->rng_used = 0;
2029 1.1 jonathan #ifdef __OpenBSD__
2030 1.1 jonathan timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2031 1.1 jonathan #else
2032 1.29 tls if (sc->sc_rng_need > 0) {
2033 1.30 bad callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2034 1.29 tls }
2035 1.1 jonathan #endif
2036 1.1 jonathan break;
2037 1.1 jonathan }
2038 1.1 jonathan #endif
2039 1.1 jonathan case UBS_CTXOP_MODEXP: {
2040 1.1 jonathan struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2041 1.1 jonathan u_int rlen, clen;
2042 1.1 jonathan
2043 1.1 jonathan krp = me->me_krp;
2044 1.1 jonathan rlen = (me->me_modbits + 7) / 8;
2045 1.1 jonathan clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
2046 1.1 jonathan
2047 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2048 1.1 jonathan 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2049 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2050 1.1 jonathan 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2051 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2052 1.1 jonathan 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2053 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2054 1.1 jonathan 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2055 1.1 jonathan
2056 1.1 jonathan if (clen < rlen)
2057 1.1 jonathan krp->krp_status = E2BIG;
2058 1.1 jonathan else {
2059 1.1 jonathan if (sc->sc_flags & UBS_FLAGS_HWNORM) {
2060 1.18 cegger memset(krp->krp_param[krp->krp_iparams].crp_p, 0,
2061 1.1 jonathan (krp->krp_param[krp->krp_iparams].crp_nbits
2062 1.1 jonathan + 7) / 8);
2063 1.1 jonathan bcopy(me->me_C.dma_vaddr,
2064 1.1 jonathan krp->krp_param[krp->krp_iparams].crp_p,
2065 1.1 jonathan (me->me_modbits + 7) / 8);
2066 1.1 jonathan } else
2067 1.1 jonathan ubsec_kshift_l(me->me_shiftbits,
2068 1.1 jonathan me->me_C.dma_vaddr, me->me_normbits,
2069 1.1 jonathan krp->krp_param[krp->krp_iparams].crp_p,
2070 1.1 jonathan krp->krp_param[krp->krp_iparams].crp_nbits);
2071 1.1 jonathan }
2072 1.1 jonathan
2073 1.1 jonathan crypto_kdone(krp);
2074 1.1 jonathan
2075 1.1 jonathan /* bzero all potentially sensitive data */
2076 1.18 cegger memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2077 1.18 cegger memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2078 1.18 cegger memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2079 1.18 cegger memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2080 1.1 jonathan
2081 1.1 jonathan /* Can't free here, so put us on the free list. */
2082 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
2083 1.1 jonathan break;
2084 1.1 jonathan }
2085 1.1 jonathan case UBS_CTXOP_RSAPRIV: {
2086 1.1 jonathan struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2087 1.1 jonathan u_int len;
2088 1.1 jonathan
2089 1.1 jonathan krp = rp->rpr_krp;
2090 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0,
2091 1.1 jonathan rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2092 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0,
2093 1.1 jonathan rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
2094 1.1 jonathan
2095 1.1 jonathan len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
2096 1.1 jonathan bcopy(rp->rpr_msgout.dma_vaddr,
2097 1.1 jonathan krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
2098 1.1 jonathan
2099 1.1 jonathan crypto_kdone(krp);
2100 1.1 jonathan
2101 1.18 cegger memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size);
2102 1.18 cegger memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size);
2103 1.18 cegger memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size);
2104 1.1 jonathan
2105 1.1 jonathan /* Can't free here, so put us on the free list. */
2106 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
2107 1.1 jonathan break;
2108 1.1 jonathan }
2109 1.1 jonathan default:
2110 1.28 chs printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev),
2111 1.1 jonathan letoh16(ctx->ctx_op));
2112 1.1 jonathan break;
2113 1.1 jonathan }
2114 1.1 jonathan }
2115 1.1 jonathan
2116 1.1 jonathan #ifndef UBSEC_NO_RNG
2117 1.29 tls
2118 1.29 tls static void
2119 1.29 tls ubsec_rng_get(size_t bytes, void *vsc)
2120 1.29 tls {
2121 1.29 tls struct ubsec_softc *sc = vsc;
2122 1.29 tls
2123 1.29 tls mutex_spin_enter(&sc->sc_mtx);
2124 1.29 tls sc->sc_rng_need = bytes;
2125 1.29 tls ubsec_rng_locked(sc);
2126 1.29 tls mutex_spin_exit(&sc->sc_mtx);
2127 1.29 tls
2128 1.29 tls }
2129 1.29 tls
2130 1.1 jonathan static void
2131 1.1 jonathan ubsec_rng(void *vsc)
2132 1.1 jonathan {
2133 1.1 jonathan struct ubsec_softc *sc = vsc;
2134 1.29 tls mutex_spin_enter(&sc->sc_mtx);
2135 1.29 tls ubsec_rng_locked(sc);
2136 1.29 tls mutex_spin_exit(&sc->sc_mtx);
2137 1.29 tls }
2138 1.29 tls
2139 1.29 tls static void
2140 1.29 tls ubsec_rng_locked(void *vsc)
2141 1.29 tls {
2142 1.29 tls struct ubsec_softc *sc = vsc;
2143 1.1 jonathan struct ubsec_q2_rng *rng = &sc->sc_rng;
2144 1.1 jonathan struct ubsec_mcr *mcr;
2145 1.1 jonathan struct ubsec_ctx_rngbypass *ctx;
2146 1.34 bad int *nqueue;
2147 1.1 jonathan
2148 1.31 bad /* Caller is responsible to lock and release sc_mtx. */
2149 1.31 bad KASSERT(mutex_owned(&sc->sc_mtx));
2150 1.31 bad
2151 1.1 jonathan if (rng->rng_used) {
2152 1.1 jonathan return;
2153 1.1 jonathan }
2154 1.29 tls
2155 1.29 tls if (sc->sc_rng_need < 1) {
2156 1.29 tls callout_stop(&sc->sc_rngto);
2157 1.29 tls return;
2158 1.29 tls }
2159 1.29 tls
2160 1.34 bad if (sc->sc_flags & UBS_FLAGS_RNG4)
2161 1.34 bad nqueue = &sc->sc_nqueue4;
2162 1.34 bad else
2163 1.34 bad nqueue = &sc->sc_nqueue2;
2164 1.34 bad
2165 1.34 bad (*nqueue)++;
2166 1.34 bad if (*nqueue >= UBS_MAX_NQUEUE)
2167 1.34 bad goto out;
2168 1.1 jonathan
2169 1.1 jonathan mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
2170 1.1 jonathan ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
2171 1.1 jonathan
2172 1.1 jonathan mcr->mcr_pkts = htole16(1);
2173 1.1 jonathan mcr->mcr_flags = 0;
2174 1.1 jonathan mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
2175 1.1 jonathan mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
2176 1.1 jonathan mcr->mcr_ipktbuf.pb_len = 0;
2177 1.1 jonathan mcr->mcr_reserved = mcr->mcr_pktlen = 0;
2178 1.1 jonathan mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
2179 1.1 jonathan mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
2180 1.1 jonathan UBS_PKTBUF_LEN);
2181 1.1 jonathan mcr->mcr_opktbuf.pb_next = 0;
2182 1.1 jonathan
2183 1.1 jonathan ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
2184 1.1 jonathan ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1);
2185 1.1 jonathan rng->rng_q.q_type = UBS_CTXOP_RNGSHA1;
2186 1.1 jonathan
2187 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0,
2188 1.1 jonathan rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2189 1.1 jonathan
2190 1.34 bad if (sc->sc_flags & UBS_FLAGS_RNG4) {
2191 1.34 bad SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next);
2192 1.34 bad ubsec_feed4(sc);
2193 1.34 bad } else {
2194 1.34 bad SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
2195 1.34 bad ubsec_feed2(sc);
2196 1.34 bad }
2197 1.1 jonathan rng->rng_used = 1;
2198 1.1 jonathan ubsecstats.hst_rng++;
2199 1.1 jonathan
2200 1.1 jonathan return;
2201 1.1 jonathan
2202 1.1 jonathan out:
2203 1.1 jonathan /*
2204 1.1 jonathan * Something weird happened, generate our own call back.
2205 1.1 jonathan */
2206 1.34 bad (*nqueue)--;
2207 1.1 jonathan #ifdef __OpenBSD__
2208 1.1 jonathan timeout_add(&sc->sc_rngto, sc->sc_rnghz);
2209 1.1 jonathan #else
2210 1.30 bad callout_schedule(&sc->sc_rngto, sc->sc_rnghz);
2211 1.1 jonathan #endif
2212 1.1 jonathan }
2213 1.1 jonathan #endif /* UBSEC_NO_RNG */
2214 1.1 jonathan
2215 1.1 jonathan static int
2216 1.1 jonathan ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size,
2217 1.1 jonathan struct ubsec_dma_alloc *dma,int mapflags)
2218 1.1 jonathan {
2219 1.1 jonathan int r;
2220 1.1 jonathan
2221 1.1 jonathan if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
2222 1.1 jonathan &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
2223 1.1 jonathan goto fail_0;
2224 1.1 jonathan
2225 1.1 jonathan if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
2226 1.1 jonathan size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
2227 1.1 jonathan goto fail_1;
2228 1.1 jonathan
2229 1.1 jonathan if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2230 1.1 jonathan BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
2231 1.1 jonathan goto fail_2;
2232 1.1 jonathan
2233 1.1 jonathan if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
2234 1.1 jonathan size, NULL, BUS_DMA_NOWAIT)) != 0)
2235 1.1 jonathan goto fail_3;
2236 1.1 jonathan
2237 1.1 jonathan dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
2238 1.1 jonathan dma->dma_size = size;
2239 1.1 jonathan return (0);
2240 1.1 jonathan
2241 1.1 jonathan fail_3:
2242 1.1 jonathan bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2243 1.1 jonathan fail_2:
2244 1.1 jonathan bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
2245 1.1 jonathan fail_1:
2246 1.1 jonathan bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2247 1.1 jonathan fail_0:
2248 1.1 jonathan dma->dma_map = NULL;
2249 1.1 jonathan return (r);
2250 1.1 jonathan }
2251 1.1 jonathan
2252 1.1 jonathan static void
2253 1.1 jonathan ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
2254 1.1 jonathan {
2255 1.1 jonathan bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
2256 1.1 jonathan bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
2257 1.1 jonathan bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2258 1.1 jonathan bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2259 1.1 jonathan }
2260 1.1 jonathan
2261 1.1 jonathan /*
2262 1.1 jonathan * Resets the board. Values in the regesters are left as is
2263 1.1 jonathan * from the reset (i.e. initial values are assigned elsewhere).
2264 1.1 jonathan */
2265 1.1 jonathan static void
2266 1.1 jonathan ubsec_reset_board(struct ubsec_softc *sc)
2267 1.1 jonathan {
2268 1.34 bad volatile u_int32_t ctrl;
2269 1.34 bad
2270 1.34 bad ctrl = READ_REG(sc, BS_CTRL);
2271 1.34 bad ctrl |= BS_CTRL_RESET;
2272 1.34 bad WRITE_REG(sc, BS_CTRL, ctrl);
2273 1.34 bad
2274 1.34 bad /*
2275 1.34 bad * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
2276 1.34 bad */
2277 1.34 bad DELAY(10);
2278 1.1 jonathan
2279 1.34 bad /* Enable RNG and interrupts on newer devices */
2280 1.34 bad if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2281 1.34 bad #ifndef UBSEC_NO_RNG
2282 1.34 bad WRITE_REG(sc, BS_CFG, BS_CFG_RNG);
2283 1.34 bad #endif
2284 1.34 bad WRITE_REG(sc, BS_INT, BS_INT_DMAINT);
2285 1.34 bad }
2286 1.1 jonathan }
2287 1.1 jonathan
2288 1.1 jonathan /*
2289 1.1 jonathan * Init Broadcom registers
2290 1.1 jonathan */
2291 1.1 jonathan static void
2292 1.1 jonathan ubsec_init_board(struct ubsec_softc *sc)
2293 1.1 jonathan {
2294 1.1 jonathan u_int32_t ctrl;
2295 1.1 jonathan
2296 1.1 jonathan ctrl = READ_REG(sc, BS_CTRL);
2297 1.1 jonathan ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
2298 1.1 jonathan ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
2299 1.1 jonathan
2300 1.1 jonathan /*
2301 1.1 jonathan * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)).
2302 1.1 jonathan * anyone got hw docs?
2303 1.1 jonathan */
2304 1.1 jonathan if (sc->sc_flags & UBS_FLAGS_KEY)
2305 1.1 jonathan ctrl |= BS_CTRL_MCR2INT;
2306 1.1 jonathan else
2307 1.1 jonathan ctrl &= ~BS_CTRL_MCR2INT;
2308 1.1 jonathan
2309 1.1 jonathan if (sc->sc_flags & UBS_FLAGS_HWNORM)
2310 1.1 jonathan ctrl &= ~BS_CTRL_SWNORM;
2311 1.1 jonathan
2312 1.34 bad if (sc->sc_flags & UBS_FLAGS_MULTIMCR) {
2313 1.34 bad ctrl |= BS_CTRL_BSIZE240;
2314 1.34 bad ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */
2315 1.34 bad
2316 1.34 bad if (sc->sc_flags & UBS_FLAGS_RNG4)
2317 1.34 bad ctrl |= BS_CTRL_MCR4INT;
2318 1.34 bad else
2319 1.34 bad ctrl &= ~BS_CTRL_MCR4INT;
2320 1.34 bad }
2321 1.34 bad
2322 1.1 jonathan WRITE_REG(sc, BS_CTRL, ctrl);
2323 1.1 jonathan }
2324 1.1 jonathan
2325 1.1 jonathan /*
2326 1.1 jonathan * Init Broadcom PCI registers
2327 1.1 jonathan */
2328 1.1 jonathan static void
2329 1.7 thorpej ubsec_init_pciregs(struct pci_attach_args *pa)
2330 1.1 jonathan {
2331 1.1 jonathan pci_chipset_tag_t pc = pa->pa_pc;
2332 1.1 jonathan u_int32_t misc;
2333 1.1 jonathan
2334 1.1 jonathan /*
2335 1.1 jonathan * This will set the cache line size to 1, this will
2336 1.1 jonathan * force the BCM58xx chip just to do burst read/writes.
2337 1.1 jonathan * Cache line read/writes are to slow
2338 1.1 jonathan */
2339 1.1 jonathan misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG);
2340 1.1 jonathan misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT))
2341 1.1 jonathan | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT);
2342 1.1 jonathan pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc);
2343 1.1 jonathan }
2344 1.1 jonathan
2345 1.1 jonathan /*
2346 1.1 jonathan * Clean up after a chip crash.
2347 1.1 jonathan * It is assumed that the caller in splnet()
2348 1.1 jonathan */
2349 1.1 jonathan static void
2350 1.1 jonathan ubsec_cleanchip(struct ubsec_softc *sc)
2351 1.1 jonathan {
2352 1.1 jonathan struct ubsec_q *q;
2353 1.1 jonathan
2354 1.1 jonathan while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2355 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_qchip);
2356 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next);
2357 1.1 jonathan ubsec_free_q(sc, q);
2358 1.1 jonathan }
2359 1.1 jonathan sc->sc_nqchip = 0;
2360 1.1 jonathan }
2361 1.1 jonathan
2362 1.1 jonathan /*
2363 1.1 jonathan * free a ubsec_q
2364 1.1 jonathan * It is assumed that the caller is within splnet()
2365 1.1 jonathan */
2366 1.1 jonathan static int
2367 1.1 jonathan ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2368 1.1 jonathan {
2369 1.1 jonathan struct ubsec_q *q2;
2370 1.1 jonathan struct cryptop *crp;
2371 1.1 jonathan int npkts;
2372 1.1 jonathan int i;
2373 1.1 jonathan
2374 1.1 jonathan npkts = q->q_nstacked_mcrs;
2375 1.1 jonathan
2376 1.1 jonathan for (i = 0; i < npkts; i++) {
2377 1.1 jonathan if(q->q_stacked_mcr[i]) {
2378 1.1 jonathan q2 = q->q_stacked_mcr[i];
2379 1.1 jonathan
2380 1.5 perry if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2381 1.1 jonathan m_freem(q2->q_dst_m);
2382 1.1 jonathan
2383 1.1 jonathan crp = (struct cryptop *)q2->q_crp;
2384 1.5 perry
2385 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2386 1.5 perry
2387 1.1 jonathan crp->crp_etype = EFAULT;
2388 1.1 jonathan crypto_done(crp);
2389 1.1 jonathan } else {
2390 1.1 jonathan break;
2391 1.1 jonathan }
2392 1.1 jonathan }
2393 1.1 jonathan
2394 1.1 jonathan /*
2395 1.1 jonathan * Free header MCR
2396 1.1 jonathan */
2397 1.1 jonathan if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2398 1.1 jonathan m_freem(q->q_dst_m);
2399 1.1 jonathan
2400 1.1 jonathan crp = (struct cryptop *)q->q_crp;
2401 1.5 perry
2402 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2403 1.5 perry
2404 1.1 jonathan crp->crp_etype = EFAULT;
2405 1.1 jonathan crypto_done(crp);
2406 1.1 jonathan return(0);
2407 1.1 jonathan }
2408 1.1 jonathan
2409 1.1 jonathan /*
2410 1.1 jonathan * Routine to reset the chip and clean up.
2411 1.1 jonathan * It is assumed that the caller is in splnet()
2412 1.1 jonathan */
2413 1.1 jonathan static void
2414 1.1 jonathan ubsec_totalreset(struct ubsec_softc *sc)
2415 1.1 jonathan {
2416 1.1 jonathan ubsec_reset_board(sc);
2417 1.1 jonathan ubsec_init_board(sc);
2418 1.1 jonathan ubsec_cleanchip(sc);
2419 1.1 jonathan }
2420 1.1 jonathan
2421 1.1 jonathan static int
2422 1.1 jonathan ubsec_dmamap_aligned(bus_dmamap_t map)
2423 1.1 jonathan {
2424 1.1 jonathan int i;
2425 1.1 jonathan
2426 1.1 jonathan for (i = 0; i < map->dm_nsegs; i++) {
2427 1.1 jonathan if (map->dm_segs[i].ds_addr & 3)
2428 1.1 jonathan return (0);
2429 1.1 jonathan if ((i != (map->dm_nsegs - 1)) &&
2430 1.1 jonathan (map->dm_segs[i].ds_len & 3))
2431 1.1 jonathan return (0);
2432 1.1 jonathan }
2433 1.1 jonathan return (1);
2434 1.1 jonathan }
2435 1.1 jonathan
2436 1.1 jonathan #ifdef __OpenBSD__
2437 1.1 jonathan struct ubsec_softc *
2438 1.7 thorpej ubsec_kfind(struct cryptkop *krp)
2439 1.1 jonathan {
2440 1.1 jonathan struct ubsec_softc *sc;
2441 1.1 jonathan int i;
2442 1.1 jonathan
2443 1.1 jonathan for (i = 0; i < ubsec_cd.cd_ndevs; i++) {
2444 1.1 jonathan sc = ubsec_cd.cd_devs[i];
2445 1.1 jonathan if (sc == NULL)
2446 1.1 jonathan continue;
2447 1.1 jonathan if (sc->sc_cid == krp->krp_hid)
2448 1.1 jonathan return (sc);
2449 1.1 jonathan }
2450 1.1 jonathan return (NULL);
2451 1.1 jonathan }
2452 1.1 jonathan #endif
2453 1.1 jonathan
2454 1.1 jonathan static void
2455 1.1 jonathan ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2456 1.1 jonathan {
2457 1.1 jonathan switch (q->q_type) {
2458 1.1 jonathan case UBS_CTXOP_MODEXP: {
2459 1.1 jonathan struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2460 1.1 jonathan
2461 1.1 jonathan ubsec_dma_free(sc, &me->me_q.q_mcr);
2462 1.1 jonathan ubsec_dma_free(sc, &me->me_q.q_ctx);
2463 1.1 jonathan ubsec_dma_free(sc, &me->me_M);
2464 1.1 jonathan ubsec_dma_free(sc, &me->me_E);
2465 1.1 jonathan ubsec_dma_free(sc, &me->me_C);
2466 1.1 jonathan ubsec_dma_free(sc, &me->me_epb);
2467 1.1 jonathan free(me, M_DEVBUF);
2468 1.1 jonathan break;
2469 1.1 jonathan }
2470 1.1 jonathan case UBS_CTXOP_RSAPRIV: {
2471 1.1 jonathan struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2472 1.1 jonathan
2473 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2474 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2475 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_msgin);
2476 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_msgout);
2477 1.1 jonathan free(rp, M_DEVBUF);
2478 1.1 jonathan break;
2479 1.1 jonathan }
2480 1.1 jonathan default:
2481 1.28 chs printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev),
2482 1.1 jonathan q->q_type);
2483 1.1 jonathan break;
2484 1.1 jonathan }
2485 1.1 jonathan }
2486 1.1 jonathan
2487 1.1 jonathan static int
2488 1.1 jonathan ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2489 1.1 jonathan {
2490 1.1 jonathan struct ubsec_softc *sc;
2491 1.1 jonathan int r;
2492 1.1 jonathan
2493 1.1 jonathan if (krp == NULL || krp->krp_callback == NULL)
2494 1.1 jonathan return (EINVAL);
2495 1.1 jonathan #ifdef __OpenBSD__
2496 1.1 jonathan if ((sc = ubsec_kfind(krp)) == NULL)
2497 1.1 jonathan return (EINVAL);
2498 1.1 jonathan #else
2499 1.1 jonathan sc = arg;
2500 1.1 jonathan KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/);
2501 1.1 jonathan #endif
2502 1.1 jonathan
2503 1.1 jonathan while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2504 1.1 jonathan struct ubsec_q2 *q;
2505 1.1 jonathan
2506 1.1 jonathan q = SIMPLEQ_FIRST(&sc->sc_q2free);
2507 1.1 jonathan SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next);
2508 1.1 jonathan ubsec_kfree(sc, q);
2509 1.1 jonathan }
2510 1.1 jonathan
2511 1.1 jonathan switch (krp->krp_op) {
2512 1.1 jonathan case CRK_MOD_EXP:
2513 1.1 jonathan if (sc->sc_flags & UBS_FLAGS_HWNORM)
2514 1.1 jonathan r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2515 1.1 jonathan else
2516 1.1 jonathan r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2517 1.1 jonathan break;
2518 1.1 jonathan case CRK_MOD_EXP_CRT:
2519 1.1 jonathan r = ubsec_kprocess_rsapriv(sc, krp, hint);
2520 1.1 jonathan break;
2521 1.1 jonathan default:
2522 1.1 jonathan printf("%s: kprocess: invalid op 0x%x\n",
2523 1.28 chs device_xname(sc->sc_dev), krp->krp_op);
2524 1.1 jonathan krp->krp_status = EOPNOTSUPP;
2525 1.1 jonathan crypto_kdone(krp);
2526 1.1 jonathan r = 0;
2527 1.1 jonathan }
2528 1.1 jonathan return (r);
2529 1.1 jonathan }
2530 1.1 jonathan
2531 1.1 jonathan /*
2532 1.1 jonathan * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2533 1.1 jonathan */
2534 1.1 jonathan static int
2535 1.1 jonathan ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp,
2536 1.11 christos int hint)
2537 1.1 jonathan {
2538 1.1 jonathan struct ubsec_q2_modexp *me;
2539 1.1 jonathan struct ubsec_mcr *mcr;
2540 1.1 jonathan struct ubsec_ctx_modexp *ctx;
2541 1.1 jonathan struct ubsec_pktbuf *epb;
2542 1.29 tls int err = 0;
2543 1.1 jonathan u_int nbits, normbits, mbits, shiftbits, ebits;
2544 1.1 jonathan
2545 1.1 jonathan me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2546 1.1 jonathan if (me == NULL) {
2547 1.1 jonathan err = ENOMEM;
2548 1.1 jonathan goto errout;
2549 1.1 jonathan }
2550 1.18 cegger memset(me, 0, sizeof *me);
2551 1.1 jonathan me->me_krp = krp;
2552 1.1 jonathan me->me_q.q_type = UBS_CTXOP_MODEXP;
2553 1.1 jonathan
2554 1.1 jonathan nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2555 1.1 jonathan if (nbits <= 512)
2556 1.1 jonathan normbits = 512;
2557 1.1 jonathan else if (nbits <= 768)
2558 1.1 jonathan normbits = 768;
2559 1.1 jonathan else if (nbits <= 1024)
2560 1.1 jonathan normbits = 1024;
2561 1.1 jonathan else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2562 1.1 jonathan normbits = 1536;
2563 1.1 jonathan else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2564 1.1 jonathan normbits = 2048;
2565 1.1 jonathan else {
2566 1.1 jonathan err = E2BIG;
2567 1.1 jonathan goto errout;
2568 1.1 jonathan }
2569 1.1 jonathan
2570 1.1 jonathan shiftbits = normbits - nbits;
2571 1.1 jonathan
2572 1.1 jonathan me->me_modbits = nbits;
2573 1.1 jonathan me->me_shiftbits = shiftbits;
2574 1.1 jonathan me->me_normbits = normbits;
2575 1.1 jonathan
2576 1.1 jonathan /* Sanity check: result bits must be >= true modulus bits. */
2577 1.1 jonathan if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2578 1.1 jonathan err = ERANGE;
2579 1.1 jonathan goto errout;
2580 1.1 jonathan }
2581 1.1 jonathan
2582 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2583 1.1 jonathan &me->me_q.q_mcr, 0)) {
2584 1.1 jonathan err = ENOMEM;
2585 1.1 jonathan goto errout;
2586 1.1 jonathan }
2587 1.1 jonathan mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2588 1.1 jonathan
2589 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2590 1.1 jonathan &me->me_q.q_ctx, 0)) {
2591 1.1 jonathan err = ENOMEM;
2592 1.1 jonathan goto errout;
2593 1.1 jonathan }
2594 1.1 jonathan
2595 1.1 jonathan mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2596 1.1 jonathan if (mbits > nbits) {
2597 1.1 jonathan err = E2BIG;
2598 1.1 jonathan goto errout;
2599 1.1 jonathan }
2600 1.1 jonathan if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2601 1.1 jonathan err = ENOMEM;
2602 1.1 jonathan goto errout;
2603 1.1 jonathan }
2604 1.1 jonathan ubsec_kshift_r(shiftbits,
2605 1.1 jonathan krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2606 1.1 jonathan me->me_M.dma_vaddr, normbits);
2607 1.1 jonathan
2608 1.1 jonathan if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2609 1.1 jonathan err = ENOMEM;
2610 1.1 jonathan goto errout;
2611 1.1 jonathan }
2612 1.18 cegger memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2613 1.1 jonathan
2614 1.1 jonathan ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2615 1.1 jonathan if (ebits > nbits) {
2616 1.1 jonathan err = E2BIG;
2617 1.1 jonathan goto errout;
2618 1.1 jonathan }
2619 1.1 jonathan if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2620 1.1 jonathan err = ENOMEM;
2621 1.1 jonathan goto errout;
2622 1.1 jonathan }
2623 1.1 jonathan ubsec_kshift_r(shiftbits,
2624 1.1 jonathan krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2625 1.1 jonathan me->me_E.dma_vaddr, normbits);
2626 1.1 jonathan
2627 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2628 1.1 jonathan &me->me_epb, 0)) {
2629 1.1 jonathan err = ENOMEM;
2630 1.1 jonathan goto errout;
2631 1.1 jonathan }
2632 1.1 jonathan epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2633 1.1 jonathan epb->pb_addr = htole32(me->me_E.dma_paddr);
2634 1.1 jonathan epb->pb_next = 0;
2635 1.1 jonathan epb->pb_len = htole32(normbits / 8);
2636 1.1 jonathan
2637 1.1 jonathan #ifdef UBSEC_DEBUG
2638 1.1 jonathan if (ubsec_debug) {
2639 1.1 jonathan printf("Epb ");
2640 1.1 jonathan ubsec_dump_pb(epb);
2641 1.1 jonathan }
2642 1.1 jonathan #endif
2643 1.1 jonathan
2644 1.1 jonathan mcr->mcr_pkts = htole16(1);
2645 1.1 jonathan mcr->mcr_flags = 0;
2646 1.1 jonathan mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2647 1.1 jonathan mcr->mcr_reserved = 0;
2648 1.1 jonathan mcr->mcr_pktlen = 0;
2649 1.1 jonathan
2650 1.1 jonathan mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2651 1.1 jonathan mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2652 1.1 jonathan mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2653 1.1 jonathan
2654 1.1 jonathan mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2655 1.1 jonathan mcr->mcr_opktbuf.pb_next = 0;
2656 1.1 jonathan mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2657 1.1 jonathan
2658 1.1 jonathan #ifdef DIAGNOSTIC
2659 1.1 jonathan /* Misaligned output buffer will hang the chip. */
2660 1.1 jonathan if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2661 1.1 jonathan panic("%s: modexp invalid addr 0x%x",
2662 1.28 chs device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr));
2663 1.1 jonathan if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2664 1.1 jonathan panic("%s: modexp invalid len 0x%x",
2665 1.28 chs device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len));
2666 1.1 jonathan #endif
2667 1.1 jonathan
2668 1.1 jonathan ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2669 1.18 cegger memset(ctx, 0, sizeof(*ctx));
2670 1.1 jonathan ubsec_kshift_r(shiftbits,
2671 1.1 jonathan krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2672 1.1 jonathan ctx->me_N, normbits);
2673 1.1 jonathan ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2674 1.1 jonathan ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2675 1.1 jonathan ctx->me_E_len = htole16(nbits);
2676 1.1 jonathan ctx->me_N_len = htole16(nbits);
2677 1.1 jonathan
2678 1.1 jonathan #ifdef UBSEC_DEBUG
2679 1.1 jonathan if (ubsec_debug) {
2680 1.1 jonathan ubsec_dump_mcr(mcr);
2681 1.1 jonathan ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2682 1.1 jonathan }
2683 1.1 jonathan #endif
2684 1.1 jonathan
2685 1.1 jonathan /*
2686 1.1 jonathan * ubsec_feed2 will sync mcr and ctx, we just need to sync
2687 1.1 jonathan * everything else.
2688 1.1 jonathan */
2689 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2690 1.1 jonathan 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2691 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2692 1.1 jonathan 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2693 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2694 1.1 jonathan 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2695 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2696 1.1 jonathan 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2697 1.1 jonathan
2698 1.1 jonathan /* Enqueue and we're done... */
2699 1.29 tls mutex_spin_enter(&sc->sc_mtx);
2700 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2701 1.1 jonathan ubsec_feed2(sc);
2702 1.1 jonathan ubsecstats.hst_modexp++;
2703 1.29 tls mutex_spin_exit(&sc->sc_mtx);
2704 1.1 jonathan
2705 1.1 jonathan return (0);
2706 1.1 jonathan
2707 1.1 jonathan errout:
2708 1.1 jonathan if (me != NULL) {
2709 1.1 jonathan if (me->me_q.q_mcr.dma_map != NULL)
2710 1.1 jonathan ubsec_dma_free(sc, &me->me_q.q_mcr);
2711 1.1 jonathan if (me->me_q.q_ctx.dma_map != NULL) {
2712 1.18 cegger memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2713 1.1 jonathan ubsec_dma_free(sc, &me->me_q.q_ctx);
2714 1.1 jonathan }
2715 1.1 jonathan if (me->me_M.dma_map != NULL) {
2716 1.18 cegger memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2717 1.1 jonathan ubsec_dma_free(sc, &me->me_M);
2718 1.1 jonathan }
2719 1.1 jonathan if (me->me_E.dma_map != NULL) {
2720 1.18 cegger memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2721 1.1 jonathan ubsec_dma_free(sc, &me->me_E);
2722 1.1 jonathan }
2723 1.1 jonathan if (me->me_C.dma_map != NULL) {
2724 1.18 cegger memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2725 1.1 jonathan ubsec_dma_free(sc, &me->me_C);
2726 1.1 jonathan }
2727 1.1 jonathan if (me->me_epb.dma_map != NULL)
2728 1.1 jonathan ubsec_dma_free(sc, &me->me_epb);
2729 1.1 jonathan free(me, M_DEVBUF);
2730 1.1 jonathan }
2731 1.1 jonathan krp->krp_status = err;
2732 1.1 jonathan crypto_kdone(krp);
2733 1.1 jonathan return (0);
2734 1.1 jonathan }
2735 1.1 jonathan
2736 1.1 jonathan /*
2737 1.1 jonathan * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2738 1.1 jonathan */
2739 1.1 jonathan static int
2740 1.1 jonathan ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp,
2741 1.11 christos int hint)
2742 1.1 jonathan {
2743 1.1 jonathan struct ubsec_q2_modexp *me;
2744 1.1 jonathan struct ubsec_mcr *mcr;
2745 1.1 jonathan struct ubsec_ctx_modexp *ctx;
2746 1.1 jonathan struct ubsec_pktbuf *epb;
2747 1.29 tls int err = 0;
2748 1.1 jonathan u_int nbits, normbits, mbits, shiftbits, ebits;
2749 1.1 jonathan
2750 1.1 jonathan me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2751 1.1 jonathan if (me == NULL) {
2752 1.1 jonathan err = ENOMEM;
2753 1.1 jonathan goto errout;
2754 1.1 jonathan }
2755 1.18 cegger memset(me, 0, sizeof *me);
2756 1.1 jonathan me->me_krp = krp;
2757 1.1 jonathan me->me_q.q_type = UBS_CTXOP_MODEXP;
2758 1.1 jonathan
2759 1.1 jonathan nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2760 1.1 jonathan if (nbits <= 512)
2761 1.1 jonathan normbits = 512;
2762 1.1 jonathan else if (nbits <= 768)
2763 1.1 jonathan normbits = 768;
2764 1.1 jonathan else if (nbits <= 1024)
2765 1.1 jonathan normbits = 1024;
2766 1.1 jonathan else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2767 1.1 jonathan normbits = 1536;
2768 1.1 jonathan else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2769 1.1 jonathan normbits = 2048;
2770 1.1 jonathan else {
2771 1.1 jonathan err = E2BIG;
2772 1.1 jonathan goto errout;
2773 1.1 jonathan }
2774 1.1 jonathan
2775 1.1 jonathan shiftbits = normbits - nbits;
2776 1.1 jonathan
2777 1.1 jonathan /* XXX ??? */
2778 1.1 jonathan me->me_modbits = nbits;
2779 1.1 jonathan me->me_shiftbits = shiftbits;
2780 1.1 jonathan me->me_normbits = normbits;
2781 1.1 jonathan
2782 1.1 jonathan /* Sanity check: result bits must be >= true modulus bits. */
2783 1.1 jonathan if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2784 1.1 jonathan err = ERANGE;
2785 1.1 jonathan goto errout;
2786 1.1 jonathan }
2787 1.1 jonathan
2788 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2789 1.1 jonathan &me->me_q.q_mcr, 0)) {
2790 1.1 jonathan err = ENOMEM;
2791 1.1 jonathan goto errout;
2792 1.1 jonathan }
2793 1.1 jonathan mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2794 1.1 jonathan
2795 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2796 1.1 jonathan &me->me_q.q_ctx, 0)) {
2797 1.1 jonathan err = ENOMEM;
2798 1.1 jonathan goto errout;
2799 1.1 jonathan }
2800 1.1 jonathan
2801 1.1 jonathan mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2802 1.1 jonathan if (mbits > nbits) {
2803 1.1 jonathan err = E2BIG;
2804 1.1 jonathan goto errout;
2805 1.1 jonathan }
2806 1.1 jonathan if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2807 1.1 jonathan err = ENOMEM;
2808 1.1 jonathan goto errout;
2809 1.1 jonathan }
2810 1.18 cegger memset(me->me_M.dma_vaddr, 0, normbits / 8);
2811 1.1 jonathan bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2812 1.1 jonathan me->me_M.dma_vaddr, (mbits + 7) / 8);
2813 1.1 jonathan
2814 1.1 jonathan if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2815 1.1 jonathan err = ENOMEM;
2816 1.1 jonathan goto errout;
2817 1.1 jonathan }
2818 1.18 cegger memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2819 1.1 jonathan
2820 1.1 jonathan ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2821 1.1 jonathan if (ebits > nbits) {
2822 1.1 jonathan err = E2BIG;
2823 1.1 jonathan goto errout;
2824 1.1 jonathan }
2825 1.1 jonathan if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2826 1.1 jonathan err = ENOMEM;
2827 1.1 jonathan goto errout;
2828 1.1 jonathan }
2829 1.18 cegger memset(me->me_E.dma_vaddr, 0, normbits / 8);
2830 1.1 jonathan bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2831 1.1 jonathan me->me_E.dma_vaddr, (ebits + 7) / 8);
2832 1.1 jonathan
2833 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2834 1.1 jonathan &me->me_epb, 0)) {
2835 1.1 jonathan err = ENOMEM;
2836 1.1 jonathan goto errout;
2837 1.1 jonathan }
2838 1.1 jonathan epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2839 1.1 jonathan epb->pb_addr = htole32(me->me_E.dma_paddr);
2840 1.1 jonathan epb->pb_next = 0;
2841 1.1 jonathan epb->pb_len = htole32((ebits + 7) / 8);
2842 1.1 jonathan
2843 1.1 jonathan #ifdef UBSEC_DEBUG
2844 1.1 jonathan if (ubsec_debug) {
2845 1.1 jonathan printf("Epb ");
2846 1.1 jonathan ubsec_dump_pb(epb);
2847 1.1 jonathan }
2848 1.1 jonathan #endif
2849 1.1 jonathan
2850 1.1 jonathan mcr->mcr_pkts = htole16(1);
2851 1.1 jonathan mcr->mcr_flags = 0;
2852 1.1 jonathan mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2853 1.1 jonathan mcr->mcr_reserved = 0;
2854 1.1 jonathan mcr->mcr_pktlen = 0;
2855 1.1 jonathan
2856 1.1 jonathan mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2857 1.1 jonathan mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2858 1.1 jonathan mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2859 1.1 jonathan
2860 1.1 jonathan mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2861 1.1 jonathan mcr->mcr_opktbuf.pb_next = 0;
2862 1.1 jonathan mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2863 1.1 jonathan
2864 1.1 jonathan #ifdef DIAGNOSTIC
2865 1.1 jonathan /* Misaligned output buffer will hang the chip. */
2866 1.1 jonathan if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2867 1.1 jonathan panic("%s: modexp invalid addr 0x%x",
2868 1.28 chs device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr));
2869 1.1 jonathan if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2870 1.1 jonathan panic("%s: modexp invalid len 0x%x",
2871 1.28 chs device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len));
2872 1.1 jonathan #endif
2873 1.1 jonathan
2874 1.1 jonathan ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2875 1.18 cegger memset(ctx, 0, sizeof(*ctx));
2876 1.20 tsutsui memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p,
2877 1.1 jonathan (nbits + 7) / 8);
2878 1.1 jonathan ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2879 1.1 jonathan ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2880 1.1 jonathan ctx->me_E_len = htole16(ebits);
2881 1.1 jonathan ctx->me_N_len = htole16(nbits);
2882 1.1 jonathan
2883 1.1 jonathan #ifdef UBSEC_DEBUG
2884 1.1 jonathan if (ubsec_debug) {
2885 1.1 jonathan ubsec_dump_mcr(mcr);
2886 1.1 jonathan ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2887 1.1 jonathan }
2888 1.1 jonathan #endif
2889 1.1 jonathan
2890 1.1 jonathan /*
2891 1.1 jonathan * ubsec_feed2 will sync mcr and ctx, we just need to sync
2892 1.1 jonathan * everything else.
2893 1.1 jonathan */
2894 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map,
2895 1.1 jonathan 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2896 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map,
2897 1.1 jonathan 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2898 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map,
2899 1.1 jonathan 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
2900 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map,
2901 1.1 jonathan 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
2902 1.1 jonathan
2903 1.1 jonathan /* Enqueue and we're done... */
2904 1.29 tls mutex_spin_enter(&sc->sc_mtx);
2905 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2906 1.1 jonathan ubsec_feed2(sc);
2907 1.29 tls mutex_spin_exit(&sc->sc_mtx);
2908 1.1 jonathan
2909 1.1 jonathan return (0);
2910 1.1 jonathan
2911 1.1 jonathan errout:
2912 1.1 jonathan if (me != NULL) {
2913 1.1 jonathan if (me->me_q.q_mcr.dma_map != NULL)
2914 1.1 jonathan ubsec_dma_free(sc, &me->me_q.q_mcr);
2915 1.1 jonathan if (me->me_q.q_ctx.dma_map != NULL) {
2916 1.18 cegger memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size);
2917 1.1 jonathan ubsec_dma_free(sc, &me->me_q.q_ctx);
2918 1.1 jonathan }
2919 1.1 jonathan if (me->me_M.dma_map != NULL) {
2920 1.18 cegger memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size);
2921 1.1 jonathan ubsec_dma_free(sc, &me->me_M);
2922 1.1 jonathan }
2923 1.1 jonathan if (me->me_E.dma_map != NULL) {
2924 1.18 cegger memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size);
2925 1.1 jonathan ubsec_dma_free(sc, &me->me_E);
2926 1.1 jonathan }
2927 1.1 jonathan if (me->me_C.dma_map != NULL) {
2928 1.18 cegger memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size);
2929 1.1 jonathan ubsec_dma_free(sc, &me->me_C);
2930 1.1 jonathan }
2931 1.1 jonathan if (me->me_epb.dma_map != NULL)
2932 1.1 jonathan ubsec_dma_free(sc, &me->me_epb);
2933 1.1 jonathan free(me, M_DEVBUF);
2934 1.1 jonathan }
2935 1.1 jonathan krp->krp_status = err;
2936 1.1 jonathan crypto_kdone(krp);
2937 1.1 jonathan return (0);
2938 1.1 jonathan }
2939 1.1 jonathan
2940 1.1 jonathan static int
2941 1.1 jonathan ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp,
2942 1.11 christos int hint)
2943 1.1 jonathan {
2944 1.1 jonathan struct ubsec_q2_rsapriv *rp = NULL;
2945 1.1 jonathan struct ubsec_mcr *mcr;
2946 1.1 jonathan struct ubsec_ctx_rsapriv *ctx;
2947 1.29 tls int err = 0;
2948 1.1 jonathan u_int padlen, msglen;
2949 1.1 jonathan
2950 1.1 jonathan msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2951 1.1 jonathan padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2952 1.1 jonathan if (msglen > padlen)
2953 1.1 jonathan padlen = msglen;
2954 1.1 jonathan
2955 1.1 jonathan if (padlen <= 256)
2956 1.1 jonathan padlen = 256;
2957 1.1 jonathan else if (padlen <= 384)
2958 1.1 jonathan padlen = 384;
2959 1.1 jonathan else if (padlen <= 512)
2960 1.1 jonathan padlen = 512;
2961 1.1 jonathan else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2962 1.1 jonathan padlen = 768;
2963 1.1 jonathan else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2964 1.1 jonathan padlen = 1024;
2965 1.1 jonathan else {
2966 1.1 jonathan err = E2BIG;
2967 1.1 jonathan goto errout;
2968 1.1 jonathan }
2969 1.1 jonathan
2970 1.1 jonathan if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2971 1.1 jonathan err = E2BIG;
2972 1.1 jonathan goto errout;
2973 1.1 jonathan }
2974 1.1 jonathan
2975 1.1 jonathan if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2976 1.1 jonathan err = E2BIG;
2977 1.1 jonathan goto errout;
2978 1.1 jonathan }
2979 1.1 jonathan
2980 1.1 jonathan if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2981 1.1 jonathan err = E2BIG;
2982 1.1 jonathan goto errout;
2983 1.1 jonathan }
2984 1.1 jonathan
2985 1.17 cegger rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO);
2986 1.1 jonathan if (rp == NULL)
2987 1.1 jonathan return (ENOMEM);
2988 1.1 jonathan rp->rpr_krp = krp;
2989 1.1 jonathan rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2990 1.1 jonathan
2991 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2992 1.1 jonathan &rp->rpr_q.q_mcr, 0)) {
2993 1.1 jonathan err = ENOMEM;
2994 1.1 jonathan goto errout;
2995 1.1 jonathan }
2996 1.1 jonathan mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2997 1.1 jonathan
2998 1.1 jonathan if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2999 1.1 jonathan &rp->rpr_q.q_ctx, 0)) {
3000 1.1 jonathan err = ENOMEM;
3001 1.1 jonathan goto errout;
3002 1.1 jonathan }
3003 1.1 jonathan ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
3004 1.18 cegger memset(ctx, 0, sizeof *ctx);
3005 1.1 jonathan
3006 1.1 jonathan /* Copy in p */
3007 1.1 jonathan bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
3008 1.1 jonathan &ctx->rpr_buf[0 * (padlen / 8)],
3009 1.1 jonathan (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
3010 1.1 jonathan
3011 1.1 jonathan /* Copy in q */
3012 1.1 jonathan bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
3013 1.1 jonathan &ctx->rpr_buf[1 * (padlen / 8)],
3014 1.1 jonathan (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
3015 1.1 jonathan
3016 1.1 jonathan /* Copy in dp */
3017 1.1 jonathan bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
3018 1.1 jonathan &ctx->rpr_buf[2 * (padlen / 8)],
3019 1.1 jonathan (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
3020 1.1 jonathan
3021 1.1 jonathan /* Copy in dq */
3022 1.1 jonathan bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
3023 1.1 jonathan &ctx->rpr_buf[3 * (padlen / 8)],
3024 1.1 jonathan (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
3025 1.1 jonathan
3026 1.1 jonathan /* Copy in pinv */
3027 1.1 jonathan bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
3028 1.1 jonathan &ctx->rpr_buf[4 * (padlen / 8)],
3029 1.1 jonathan (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
3030 1.1 jonathan
3031 1.1 jonathan msglen = padlen * 2;
3032 1.1 jonathan
3033 1.1 jonathan /* Copy in input message (aligned buffer/length). */
3034 1.1 jonathan if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
3035 1.1 jonathan /* Is this likely? */
3036 1.1 jonathan err = E2BIG;
3037 1.1 jonathan goto errout;
3038 1.1 jonathan }
3039 1.1 jonathan if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
3040 1.1 jonathan err = ENOMEM;
3041 1.1 jonathan goto errout;
3042 1.1 jonathan }
3043 1.18 cegger memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8);
3044 1.1 jonathan bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
3045 1.1 jonathan rp->rpr_msgin.dma_vaddr,
3046 1.1 jonathan (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
3047 1.1 jonathan
3048 1.1 jonathan /* Prepare space for output message (aligned buffer/length). */
3049 1.1 jonathan if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
3050 1.1 jonathan /* Is this likely? */
3051 1.1 jonathan err = E2BIG;
3052 1.1 jonathan goto errout;
3053 1.1 jonathan }
3054 1.1 jonathan if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
3055 1.1 jonathan err = ENOMEM;
3056 1.1 jonathan goto errout;
3057 1.1 jonathan }
3058 1.18 cegger memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8);
3059 1.1 jonathan
3060 1.1 jonathan mcr->mcr_pkts = htole16(1);
3061 1.1 jonathan mcr->mcr_flags = 0;
3062 1.1 jonathan mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
3063 1.1 jonathan mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
3064 1.1 jonathan mcr->mcr_ipktbuf.pb_next = 0;
3065 1.1 jonathan mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
3066 1.1 jonathan mcr->mcr_reserved = 0;
3067 1.1 jonathan mcr->mcr_pktlen = htole16(msglen);
3068 1.1 jonathan mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
3069 1.1 jonathan mcr->mcr_opktbuf.pb_next = 0;
3070 1.1 jonathan mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
3071 1.1 jonathan
3072 1.1 jonathan #ifdef DIAGNOSTIC
3073 1.1 jonathan if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
3074 1.3 thorpej panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)",
3075 1.28 chs device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr,
3076 1.3 thorpej (u_long) rp->rpr_msgin.dma_size);
3077 1.1 jonathan }
3078 1.1 jonathan if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
3079 1.3 thorpej panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)",
3080 1.28 chs device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr,
3081 1.3 thorpej (u_long) rp->rpr_msgout.dma_size);
3082 1.1 jonathan }
3083 1.1 jonathan #endif
3084 1.1 jonathan
3085 1.1 jonathan ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
3086 1.1 jonathan ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
3087 1.1 jonathan ctx->rpr_q_len = htole16(padlen);
3088 1.1 jonathan ctx->rpr_p_len = htole16(padlen);
3089 1.1 jonathan
3090 1.1 jonathan /*
3091 1.1 jonathan * ubsec_feed2 will sync mcr and ctx, we just need to sync
3092 1.1 jonathan * everything else.
3093 1.1 jonathan */
3094 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map,
3095 1.1 jonathan 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
3096 1.1 jonathan bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map,
3097 1.1 jonathan 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD);
3098 1.1 jonathan
3099 1.1 jonathan /* Enqueue and we're done... */
3100 1.29 tls mutex_spin_enter(&sc->sc_mtx);
3101 1.1 jonathan SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
3102 1.1 jonathan ubsec_feed2(sc);
3103 1.1 jonathan ubsecstats.hst_modexpcrt++;
3104 1.29 tls mutex_spin_exit(&sc->sc_mtx);
3105 1.1 jonathan return (0);
3106 1.1 jonathan
3107 1.1 jonathan errout:
3108 1.1 jonathan if (rp != NULL) {
3109 1.1 jonathan if (rp->rpr_q.q_mcr.dma_map != NULL)
3110 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
3111 1.1 jonathan if (rp->rpr_msgin.dma_map != NULL) {
3112 1.18 cegger memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size);
3113 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_msgin);
3114 1.1 jonathan }
3115 1.1 jonathan if (rp->rpr_msgout.dma_map != NULL) {
3116 1.18 cegger memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size);
3117 1.1 jonathan ubsec_dma_free(sc, &rp->rpr_msgout);
3118 1.1 jonathan }
3119 1.1 jonathan free(rp, M_DEVBUF);
3120 1.1 jonathan }
3121 1.1 jonathan krp->krp_status = err;
3122 1.1 jonathan crypto_kdone(krp);
3123 1.1 jonathan return (0);
3124 1.1 jonathan }
3125 1.1 jonathan
3126 1.1 jonathan #ifdef UBSEC_DEBUG
3127 1.1 jonathan static void
3128 1.1 jonathan ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
3129 1.1 jonathan {
3130 1.1 jonathan printf("addr 0x%x (0x%x) next 0x%x\n",
3131 1.1 jonathan pb->pb_addr, pb->pb_len, pb->pb_next);
3132 1.1 jonathan }
3133 1.1 jonathan
3134 1.1 jonathan static void
3135 1.1 jonathan ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c)
3136 1.1 jonathan {
3137 1.1 jonathan printf("CTX (0x%x):\n", c->ctx_len);
3138 1.1 jonathan switch (letoh16(c->ctx_op)) {
3139 1.1 jonathan case UBS_CTXOP_RNGBYPASS:
3140 1.1 jonathan case UBS_CTXOP_RNGSHA1:
3141 1.1 jonathan break;
3142 1.1 jonathan case UBS_CTXOP_MODEXP:
3143 1.1 jonathan {
3144 1.1 jonathan struct ubsec_ctx_modexp *cx = (void *)c;
3145 1.1 jonathan int i, len;
3146 1.1 jonathan
3147 1.1 jonathan printf(" Elen %u, Nlen %u\n",
3148 1.1 jonathan letoh16(cx->me_E_len), letoh16(cx->me_N_len));
3149 1.1 jonathan len = (cx->me_N_len + 7)/8;
3150 1.1 jonathan for (i = 0; i < len; i++)
3151 1.1 jonathan printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
3152 1.1 jonathan printf("\n");
3153 1.1 jonathan break;
3154 1.1 jonathan }
3155 1.1 jonathan default:
3156 1.1 jonathan printf("unknown context: %x\n", c->ctx_op);
3157 1.1 jonathan }
3158 1.1 jonathan printf("END CTX\n");
3159 1.1 jonathan }
3160 1.1 jonathan
3161 1.1 jonathan static void
3162 1.1 jonathan ubsec_dump_mcr(struct ubsec_mcr *mcr)
3163 1.1 jonathan {
3164 1.1 jonathan volatile struct ubsec_mcr_add *ma;
3165 1.1 jonathan int i;
3166 1.1 jonathan
3167 1.1 jonathan printf("MCR:\n");
3168 1.1 jonathan printf(" pkts: %u, flags 0x%x\n",
3169 1.1 jonathan letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
3170 1.1 jonathan ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
3171 1.1 jonathan for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
3172 1.1 jonathan printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
3173 1.1 jonathan letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
3174 1.1 jonathan letoh16(ma->mcr_reserved));
3175 1.1 jonathan printf(" %d: ipkt ", i);
3176 1.1 jonathan ubsec_dump_pb(&ma->mcr_ipktbuf);
3177 1.1 jonathan printf(" %d: opkt ", i);
3178 1.1 jonathan ubsec_dump_pb(&ma->mcr_opktbuf);
3179 1.1 jonathan ma++;
3180 1.1 jonathan }
3181 1.1 jonathan printf("END MCR\n");
3182 1.1 jonathan }
3183 1.1 jonathan #endif /* UBSEC_DEBUG */
3184 1.1 jonathan
3185 1.1 jonathan /*
3186 1.1 jonathan * Return the number of significant bits of a big number.
3187 1.1 jonathan */
3188 1.1 jonathan static int
3189 1.1 jonathan ubsec_ksigbits(struct crparam *cr)
3190 1.1 jonathan {
3191 1.1 jonathan u_int plen = (cr->crp_nbits + 7) / 8;
3192 1.1 jonathan int i, sig = plen * 8;
3193 1.1 jonathan u_int8_t c, *p = cr->crp_p;
3194 1.1 jonathan
3195 1.1 jonathan for (i = plen - 1; i >= 0; i--) {
3196 1.1 jonathan c = p[i];
3197 1.1 jonathan if (c != 0) {
3198 1.1 jonathan while ((c & 0x80) == 0) {
3199 1.1 jonathan sig--;
3200 1.1 jonathan c <<= 1;
3201 1.1 jonathan }
3202 1.1 jonathan break;
3203 1.1 jonathan }
3204 1.1 jonathan sig -= 8;
3205 1.1 jonathan }
3206 1.1 jonathan return (sig);
3207 1.1 jonathan }
3208 1.1 jonathan
3209 1.1 jonathan static void
3210 1.7 thorpej ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits,
3211 1.7 thorpej u_int8_t *dst, u_int dstbits)
3212 1.1 jonathan {
3213 1.1 jonathan u_int slen, dlen;
3214 1.1 jonathan int i, si, di, n;
3215 1.1 jonathan
3216 1.1 jonathan slen = (srcbits + 7) / 8;
3217 1.1 jonathan dlen = (dstbits + 7) / 8;
3218 1.1 jonathan
3219 1.1 jonathan for (i = 0; i < slen; i++)
3220 1.1 jonathan dst[i] = src[i];
3221 1.1 jonathan for (i = 0; i < dlen - slen; i++)
3222 1.1 jonathan dst[slen + i] = 0;
3223 1.1 jonathan
3224 1.1 jonathan n = shiftbits / 8;
3225 1.1 jonathan if (n != 0) {
3226 1.1 jonathan si = dlen - n - 1;
3227 1.1 jonathan di = dlen - 1;
3228 1.1 jonathan while (si >= 0)
3229 1.1 jonathan dst[di--] = dst[si--];
3230 1.1 jonathan while (di >= 0)
3231 1.1 jonathan dst[di--] = 0;
3232 1.1 jonathan }
3233 1.1 jonathan
3234 1.1 jonathan n = shiftbits % 8;
3235 1.1 jonathan if (n != 0) {
3236 1.1 jonathan for (i = dlen - 1; i > 0; i--)
3237 1.1 jonathan dst[i] = (dst[i] << n) |
3238 1.1 jonathan (dst[i - 1] >> (8 - n));
3239 1.1 jonathan dst[0] = dst[0] << n;
3240 1.1 jonathan }
3241 1.1 jonathan }
3242 1.1 jonathan
3243 1.1 jonathan static void
3244 1.7 thorpej ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits,
3245 1.7 thorpej u_int8_t *dst, u_int dstbits)
3246 1.1 jonathan {
3247 1.1 jonathan int slen, dlen, i, n;
3248 1.1 jonathan
3249 1.1 jonathan slen = (srcbits + 7) / 8;
3250 1.1 jonathan dlen = (dstbits + 7) / 8;
3251 1.1 jonathan
3252 1.1 jonathan n = shiftbits / 8;
3253 1.1 jonathan for (i = 0; i < slen; i++)
3254 1.1 jonathan dst[i] = src[i + n];
3255 1.1 jonathan for (i = 0; i < dlen - slen; i++)
3256 1.1 jonathan dst[slen + i] = 0;
3257 1.1 jonathan
3258 1.1 jonathan n = shiftbits % 8;
3259 1.1 jonathan if (n != 0) {
3260 1.1 jonathan for (i = 0; i < (dlen - 1); i++)
3261 1.1 jonathan dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
3262 1.1 jonathan dst[dlen - 1] = dst[dlen - 1] >> n;
3263 1.1 jonathan }
3264 1.1 jonathan }
3265