ubsec.c revision 1.38
1 /* $NetBSD: ubsec.c,v 1.38 2014/03/29 19:28:25 christos Exp $ */ 2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.127 2003/06/04 14:04:58 jason Exp $ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Effort sponsored in part by the Defense Advanced Research Projects 32 * Agency (DARPA) and Air Force Research Laboratory, Air Force 33 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 34 * 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.38 2014/03/29 19:28:25 christos Exp $"); 39 40 #undef UBSEC_DEBUG 41 42 /* 43 * uBsec 5[56]01, bcm580xx, bcm582x hardware crypto accelerator 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/endian.h> 50 #ifdef __NetBSD__ 51 #define UBSEC_NO_RNG /* hangs on attach */ 52 #define letoh16 htole16 53 #define letoh32 htole32 54 #endif 55 #include <sys/errno.h> 56 #include <sys/malloc.h> 57 #include <sys/kernel.h> 58 #include <sys/mbuf.h> 59 #include <sys/device.h> 60 #include <sys/module.h> 61 #include <sys/queue.h> 62 #include <sys/sysctl.h> 63 64 #include <opencrypto/cryptodev.h> 65 #include <opencrypto/xform.h> 66 #ifdef __OpenBSD__ 67 #include <dev/rndvar.h> 68 #include <sys/md5k.h> 69 #else 70 #include <sys/cprng.h> 71 #include <sys/md5.h> 72 #endif 73 #include <sys/sha1.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pcidevs.h> 78 79 #include <dev/pci/ubsecreg.h> 80 #include <dev/pci/ubsecvar.h> 81 82 /* 83 * Prototypes and count for the pci_device structure 84 */ 85 static int ubsec_probe(device_t, cfdata_t, void *); 86 static void ubsec_attach(device_t, device_t, void *); 87 static int ubsec_detach(device_t, int); 88 static int ubsec_sysctl_init(void); 89 static void ubsec_reset_board(struct ubsec_softc *); 90 static void ubsec_init_board(struct ubsec_softc *); 91 static void ubsec_init_pciregs(struct pci_attach_args *pa); 92 static void ubsec_cleanchip(struct ubsec_softc *); 93 static void ubsec_totalreset(struct ubsec_softc *); 94 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *); 95 96 #ifdef __OpenBSD__ 97 struct cfattach ubsec_ca = { 98 sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 99 }; 100 101 struct cfdriver ubsec_cd = { 102 0, "ubsec", DV_DULL 103 }; 104 #else 105 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 106 ubsec_detach, NULL); 107 extern struct cfdriver ubsec_cd; 108 #endif 109 110 /* patchable */ 111 #ifdef UBSEC_DEBUG 112 extern int ubsec_debug; 113 int ubsec_debug=1; 114 #endif 115 116 static int ubsec_intr(void *); 117 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *); 118 static int ubsec_freesession(void*, u_int64_t); 119 static int ubsec_process(void*, struct cryptop *, int hint); 120 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 121 static void ubsec_feed(struct ubsec_softc *); 122 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 123 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 124 static void ubsec_feed2(struct ubsec_softc *); 125 static void ubsec_feed4(struct ubsec_softc *); 126 #ifndef UBSEC_NO_RNG 127 static void ubsec_rng(void *); 128 static void ubsec_rng_locked(void *); 129 static void ubsec_rng_get(size_t, void *); 130 #endif /* UBSEC_NO_RNG */ 131 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 132 struct ubsec_dma_alloc *, int); 133 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 134 static int ubsec_dmamap_aligned(bus_dmamap_t); 135 136 static int ubsec_kprocess(void*, struct cryptkop *, int); 137 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, 138 struct cryptkop *, int); 139 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, 140 struct cryptkop *, int); 141 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, 142 struct cryptkop *, int); 143 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 144 static int ubsec_ksigbits(struct crparam *); 145 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 146 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 147 148 #ifdef UBSEC_DEBUG 149 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 150 static void ubsec_dump_mcr(struct ubsec_mcr *); 151 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *); 152 #endif 153 154 #define READ_REG(sc,r) \ 155 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 156 157 #define WRITE_REG(sc,reg,val) \ 158 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 159 160 #define SWAP32(x) (x) = htole32(ntohl((x))) 161 #ifndef HTOLE32 162 #define HTOLE32(x) (x) = htole32(x) 163 #endif 164 165 struct ubsec_stats ubsecstats; 166 167 static struct sysctllog *ubsec_sysctllog; 168 169 /* 170 * ubsec_maxbatch controls the number of crypto ops to voluntarily 171 * collect into one submission to the hardware. This batching happens 172 * when ops are dispatched from the crypto subsystem with a hint that 173 * more are to follow immediately. These ops must also not be marked 174 * with a ``no delay'' flag. 175 */ 176 static int ubsec_maxbatch = 1; 177 178 /* 179 * ubsec_maxaggr controls the number of crypto ops to submit to the 180 * hardware as a unit. This aggregation reduces the number of interrupts 181 * to the host at the expense of increased latency (for all but the last 182 * operation). For network traffic setting this to one yields the highest 183 * performance but at the expense of more interrupt processing. 184 */ 185 static int ubsec_maxaggr = 1; 186 187 static const struct ubsec_product { 188 pci_vendor_id_t ubsec_vendor; 189 pci_product_id_t ubsec_product; 190 int ubsec_flags; 191 int ubsec_statmask; 192 int ubsec_maxaggr; 193 const char *ubsec_name; 194 } ubsec_products[] = { 195 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501, 196 0, 197 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 198 UBS_MIN_AGGR, 199 "Bluesteel 5501" 200 }, 201 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601, 202 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 203 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 204 UBS_MIN_AGGR, 205 "Bluesteel 5601" 206 }, 207 208 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801, 209 0, 210 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 211 UBS_MIN_AGGR, 212 "Broadcom BCM5801" 213 }, 214 215 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802, 216 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 217 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 218 UBS_MIN_AGGR, 219 "Broadcom BCM5802" 220 }, 221 222 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805, 223 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 224 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 225 UBS_MIN_AGGR, 226 "Broadcom BCM5805" 227 }, 228 229 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820, 230 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 231 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 232 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 233 UBS_MIN_AGGR, 234 "Broadcom BCM5820" 235 }, 236 237 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821, 238 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 239 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 240 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 241 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 242 UBS_MIN_AGGR, 243 "Broadcom BCM5821" 244 }, 245 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K, 246 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 247 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 248 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 249 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 250 UBS_MIN_AGGR, 251 "Sun Crypto Accelerator 1000" 252 }, 253 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821, 254 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 255 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 256 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 257 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 258 UBS_MIN_AGGR, 259 "Broadcom BCM5821 (Sun)" 260 }, 261 262 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822, 263 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 264 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 265 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 266 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 267 UBS_MIN_AGGR, 268 "Broadcom BCM5822" 269 }, 270 271 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823, 272 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 273 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 274 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 275 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 276 UBS_MIN_AGGR, 277 "Broadcom BCM5823" 278 }, 279 280 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825, 281 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 282 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 283 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 284 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 285 UBS_MIN_AGGR, 286 "Broadcom BCM5825" 287 }, 288 289 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860, 290 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 291 UBS_FLAGS_LONGCTX | 292 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 293 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY, 294 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 295 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 296 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 297 UBS_MAX_AGGR, 298 "Broadcom BCM5860" 299 }, 300 301 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861, 302 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 303 UBS_FLAGS_LONGCTX | 304 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 305 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY, 306 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 307 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 308 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 309 UBS_MAX_AGGR, 310 "Broadcom BCM5861" 311 }, 312 313 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862, 314 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 315 UBS_FLAGS_LONGCTX | 316 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 317 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY, 318 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 319 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 320 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 321 UBS_MAX_AGGR, 322 "Broadcom BCM5862" 323 }, 324 325 { 0, 0, 326 0, 327 0, 328 0, 329 NULL 330 } 331 }; 332 333 static const struct ubsec_product * 334 ubsec_lookup(const struct pci_attach_args *pa) 335 { 336 const struct ubsec_product *up; 337 338 for (up = ubsec_products; up->ubsec_name != NULL; up++) { 339 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor && 340 PCI_PRODUCT(pa->pa_id) == up->ubsec_product) 341 return (up); 342 } 343 return (NULL); 344 } 345 346 static int 347 ubsec_probe(device_t parent, cfdata_t match, void *aux) 348 { 349 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 350 351 if (ubsec_lookup(pa) != NULL) 352 return (1); 353 354 return (0); 355 } 356 357 static void 358 ubsec_attach(device_t parent, device_t self, void *aux) 359 { 360 struct ubsec_softc *sc = device_private(self); 361 struct pci_attach_args *pa = aux; 362 const struct ubsec_product *up; 363 pci_chipset_tag_t pc = pa->pa_pc; 364 pci_intr_handle_t ih; 365 const char *intrstr = NULL; 366 pcireg_t memtype; 367 struct ubsec_dma *dmap; 368 u_int32_t cmd, i; 369 char intrbuf[PCI_INTRSTR_LEN]; 370 371 sc->sc_dev = self; 372 sc->sc_pct = pc; 373 374 up = ubsec_lookup(pa); 375 if (up == NULL) { 376 printf("\n"); 377 panic("ubsec_attach: impossible"); 378 } 379 380 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1); 381 382 SIMPLEQ_INIT(&sc->sc_queue); 383 SIMPLEQ_INIT(&sc->sc_qchip); 384 SIMPLEQ_INIT(&sc->sc_queue2); 385 SIMPLEQ_INIT(&sc->sc_qchip2); 386 SIMPLEQ_INIT(&sc->sc_queue4); 387 SIMPLEQ_INIT(&sc->sc_qchip4); 388 SIMPLEQ_INIT(&sc->sc_q2free); 389 390 sc->sc_flags = up->ubsec_flags; 391 sc->sc_statmask = up->ubsec_statmask; 392 sc->sc_maxaggr = up->ubsec_maxaggr; 393 394 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 395 cmd |= PCI_COMMAND_MASTER_ENABLE; 396 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 397 398 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR); 399 if (pci_mapreg_map(pa, BS_BAR, memtype, 0, 400 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) { 401 aprint_error_dev(self, "can't find mem space"); 402 return; 403 } 404 405 sc->sc_dmat = pa->pa_dmat; 406 407 if (pci_intr_map(pa, &ih)) { 408 aprint_error_dev(self, "couldn't map interrupt\n"); 409 return; 410 } 411 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 412 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc); 413 if (sc->sc_ih == NULL) { 414 aprint_error_dev(self, "couldn't establish interrupt"); 415 if (intrstr != NULL) 416 aprint_error(" at %s", intrstr); 417 aprint_error("\n"); 418 return; 419 } 420 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 421 422 sc->sc_cid = crypto_get_driverid(0); 423 if (sc->sc_cid < 0) { 424 aprint_error_dev(self, "couldn't get crypto driver id\n"); 425 pci_intr_disestablish(pc, sc->sc_ih); 426 return; 427 } 428 429 sc->sc_rng_need = RND_POOLBITS / NBBY; 430 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 431 432 SIMPLEQ_INIT(&sc->sc_freequeue); 433 dmap = sc->sc_dmaa; 434 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 435 struct ubsec_q *q; 436 437 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 438 M_DEVBUF, M_NOWAIT); 439 if (q == NULL) { 440 aprint_error_dev(self, "can't allocate queue buffers\n"); 441 break; 442 } 443 444 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 445 &dmap->d_alloc, 0)) { 446 aprint_error_dev(self, "can't allocate dma buffers\n"); 447 free(q, M_DEVBUF); 448 break; 449 } 450 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 451 452 q->q_dma = dmap; 453 sc->sc_queuea[i] = q; 454 455 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 456 } 457 458 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 459 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 460 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 461 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 462 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 463 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 464 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 465 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 466 467 /* 468 * Reset Broadcom chip 469 */ 470 ubsec_reset_board(sc); 471 472 /* 473 * Init Broadcom specific PCI settings 474 */ 475 ubsec_init_pciregs(pa); 476 477 /* 478 * Init Broadcom chip 479 */ 480 ubsec_init_board(sc); 481 482 #ifndef UBSEC_NO_RNG 483 if (sc->sc_flags & UBS_FLAGS_RNG) { 484 if (sc->sc_flags & UBS_FLAGS_RNG4) 485 sc->sc_statmask |= BS_STAT_MCR4_DONE; 486 else 487 sc->sc_statmask |= BS_STAT_MCR2_DONE; 488 489 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 490 &sc->sc_rng.rng_q.q_mcr, 0)) 491 goto skip_rng; 492 493 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 494 &sc->sc_rng.rng_q.q_ctx, 0)) { 495 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 496 goto skip_rng; 497 } 498 499 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 500 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 501 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 502 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 503 goto skip_rng; 504 } 505 506 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc); 507 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev), 508 RND_TYPE_RNG, 509 RND_FLAG_NO_ESTIMATE|RND_FLAG_HASCB); 510 if (hz >= 100) 511 sc->sc_rnghz = hz / 100; 512 else 513 sc->sc_rnghz = 1; 514 #ifdef __OpenBSD__ 515 timeout_set(&sc->sc_rngto, ubsec_rng, sc); 516 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 517 #else 518 callout_init(&sc->sc_rngto, 0); 519 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc); 520 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 521 #endif 522 skip_rng: 523 if (sc->sc_rnghz) 524 aprint_normal_dev(self, "random number generator enabled\n"); 525 else 526 aprint_error_dev(self, "WARNING: random number generator " 527 "disabled\n"); 528 } 529 #endif /* UBSEC_NO_RNG */ 530 531 if (sc->sc_flags & UBS_FLAGS_KEY) { 532 sc->sc_statmask |= BS_STAT_MCR2_DONE; 533 534 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 535 ubsec_kprocess, sc); 536 #if 0 537 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 538 ubsec_kprocess, sc); 539 #endif 540 } 541 } 542 543 static int 544 ubsec_detach(device_t self, int flags) 545 { 546 struct ubsec_softc *sc = device_private(self); 547 struct ubsec_q *q, *qtmp; 548 volatile u_int32_t ctrl; 549 550 /* disable interrupts */ 551 /* XXX wait/abort current ops? where is DMAERR enabled? */ 552 ctrl = READ_REG(sc, BS_CTRL); 553 554 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR); 555 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 556 ctrl &= ~BS_CTRL_MCR4INT; 557 558 WRITE_REG(sc, BS_CTRL, ctrl); 559 560 #ifndef UBSEC_NO_RNG 561 if (sc->sc_flags & UBS_FLAGS_RNG) { 562 callout_halt(&sc->sc_rngto, NULL); 563 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 564 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 565 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 566 rnd_detach_source(&sc->sc_rnd_source); 567 } 568 #endif /* UBSEC_NO_RNG */ 569 570 crypto_unregister_all(sc->sc_cid); 571 572 mutex_spin_enter(&sc->sc_mtx); 573 574 ubsec_totalreset(sc); /* XXX leaves the chip running */ 575 576 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) { 577 ubsec_dma_free(sc, &q->q_dma->d_alloc); 578 free(q, M_DEVBUF); 579 } 580 581 mutex_spin_exit(&sc->sc_mtx); 582 583 if (sc->sc_ih != NULL) { 584 pci_intr_disestablish(sc->sc_pct, sc->sc_ih); 585 sc->sc_ih = NULL; 586 } 587 588 if (sc->sc_memsize != 0) { 589 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize); 590 sc->sc_memsize = 0; 591 } 592 593 return 0; 594 } 595 596 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto"); 597 598 #ifdef _MODULE 599 #include "ioconf.c" 600 #endif 601 602 static int 603 ubsec_modcmd(modcmd_t cmd, void *data) 604 { 605 int error = 0; 606 607 switch (cmd) { 608 case MODULE_CMD_INIT: 609 #ifdef _MODULE 610 error = config_init_component(cfdriver_ioconf_ubsec, 611 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 612 #endif 613 if (error == 0) 614 error = ubsec_sysctl_init(); 615 return error; 616 case MODULE_CMD_FINI: 617 if (ubsec_sysctllog != NULL) 618 sysctl_teardown(&ubsec_sysctllog); 619 #ifdef _MODULE 620 error = config_fini_component(cfdriver_ioconf_ubsec, 621 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 622 #endif 623 return error; 624 default: 625 return ENOTTY; 626 } 627 } 628 629 static int 630 ubsec_sysctl_init(void) 631 { 632 const struct sysctlnode *node = NULL; 633 634 ubsec_sysctllog = NULL; 635 636 sysctl_createv(&ubsec_sysctllog, 0, NULL, &node, 637 CTLFLAG_PERMANENT, 638 CTLTYPE_NODE, "ubsec", 639 SYSCTL_DESCR("ubsec opetions"), 640 NULL, 0, NULL, 0, 641 CTL_HW, CTL_CREATE, CTL_EOL); 642 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 643 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 644 CTLTYPE_INT, "maxbatch", 645 SYSCTL_DESCR("max ops to batch w/o interrupt"), 646 NULL, 0, &ubsec_maxbatch, 0, 647 CTL_CREATE, CTL_EOL); 648 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 649 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 650 CTLTYPE_INT, "maxaggr", 651 SYSCTL_DESCR("max ops to aggregate under one interrupt"), 652 NULL, 0, &ubsec_maxaggr, 0, 653 CTL_CREATE, CTL_EOL); 654 655 return 0; 656 } 657 658 /* 659 * UBSEC Interrupt routine 660 */ 661 static int 662 ubsec_intr(void *arg) 663 { 664 struct ubsec_softc *sc = arg; 665 volatile u_int32_t stat; 666 struct ubsec_q *q; 667 struct ubsec_dma *dmap; 668 int flags; 669 int npkts = 0, i; 670 671 mutex_spin_enter(&sc->sc_mtx); 672 stat = READ_REG(sc, BS_STAT); 673 stat &= sc->sc_statmask; 674 if (stat == 0) { 675 mutex_spin_exit(&sc->sc_mtx); 676 return (0); 677 } 678 679 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 680 681 /* 682 * Check to see if we have any packets waiting for us 683 */ 684 if ((stat & BS_STAT_MCR1_DONE)) { 685 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 686 q = SIMPLEQ_FIRST(&sc->sc_qchip); 687 dmap = q->q_dma; 688 689 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 690 break; 691 692 q = SIMPLEQ_FIRST(&sc->sc_qchip); 693 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 694 695 npkts = q->q_nstacked_mcrs; 696 sc->sc_nqchip -= 1+npkts; 697 /* 698 * search for further sc_qchip ubsec_q's that share 699 * the same MCR, and complete them too, they must be 700 * at the top. 701 */ 702 for (i = 0; i < npkts; i++) { 703 if(q->q_stacked_mcr[i]) 704 ubsec_callback(sc, q->q_stacked_mcr[i]); 705 else 706 break; 707 } 708 ubsec_callback(sc, q); 709 } 710 711 /* 712 * Don't send any more packet to chip if there has been 713 * a DMAERR. 714 */ 715 if (!(stat & BS_STAT_DMAERR)) 716 ubsec_feed(sc); 717 } 718 719 /* 720 * Check to see if we have any key setups/rng's waiting for us 721 */ 722 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 723 (stat & BS_STAT_MCR2_DONE)) { 724 struct ubsec_q2 *q2; 725 struct ubsec_mcr *mcr; 726 727 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 728 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 729 730 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 731 0, q2->q_mcr.dma_map->dm_mapsize, 732 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 733 734 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 735 736 /* A bug in new devices requires to swap this field */ 737 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 738 flags = htole16(mcr->mcr_flags); 739 else 740 flags = mcr->mcr_flags; 741 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 742 bus_dmamap_sync(sc->sc_dmat, 743 q2->q_mcr.dma_map, 0, 744 q2->q_mcr.dma_map->dm_mapsize, 745 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 746 break; 747 } 748 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 749 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next); 750 ubsec_callback2(sc, q2); 751 /* 752 * Don't send any more packet to chip if there has been 753 * a DMAERR. 754 */ 755 if (!(stat & BS_STAT_DMAERR)) 756 ubsec_feed2(sc); 757 } 758 } 759 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) { 760 struct ubsec_q2 *q2; 761 struct ubsec_mcr *mcr; 762 763 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) { 764 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4); 765 766 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 767 0, q2->q_mcr.dma_map->dm_mapsize, 768 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 769 770 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 771 772 /* A bug in new devices requires to swap this field */ 773 flags = htole16(mcr->mcr_flags); 774 775 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 776 bus_dmamap_sync(sc->sc_dmat, 777 q2->q_mcr.dma_map, 0, 778 q2->q_mcr.dma_map->dm_mapsize, 779 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 780 break; 781 } 782 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next); 783 ubsec_callback2(sc, q2); 784 /* 785 * Don't send any more packet to chip if there has been 786 * a DMAERR. 787 */ 788 if (!(stat & BS_STAT_DMAERR)) 789 ubsec_feed4(sc); 790 } 791 } 792 793 /* 794 * Check to see if we got any DMA Error 795 */ 796 if (stat & BS_STAT_DMAERR) { 797 #ifdef UBSEC_DEBUG 798 if (ubsec_debug) { 799 volatile u_int32_t a = READ_REG(sc, BS_ERR); 800 801 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev), 802 (a & BS_ERR_READ) ? "read" : "write", 803 a & BS_ERR_ADDR); 804 } 805 #endif /* UBSEC_DEBUG */ 806 ubsecstats.hst_dmaerr++; 807 ubsec_totalreset(sc); 808 ubsec_feed(sc); 809 } 810 811 if (sc->sc_needwakeup) { /* XXX check high watermark */ 812 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 813 #ifdef UBSEC_DEBUG 814 if (ubsec_debug) 815 printf("%s: wakeup crypto (%x)\n", device_xname(sc->sc_dev), 816 sc->sc_needwakeup); 817 #endif /* UBSEC_DEBUG */ 818 sc->sc_needwakeup &= ~wkeup; 819 crypto_unblock(sc->sc_cid, wkeup); 820 } 821 mutex_spin_exit(&sc->sc_mtx); 822 return (1); 823 } 824 825 /* 826 * ubsec_feed() - aggregate and post requests to chip 827 * OpenBSD comments: 828 * It is assumed that the caller set splnet() 829 */ 830 static void 831 ubsec_feed(struct ubsec_softc *sc) 832 { 833 struct ubsec_q *q, *q2; 834 int npkts, i; 835 void *v; 836 u_int32_t stat; 837 #ifdef UBSEC_DEBUG 838 static int max; 839 #endif /* UBSEC_DEBUG */ 840 841 npkts = sc->sc_nqueue; 842 if (npkts > ubsecstats.hst_maxqueue) 843 ubsecstats.hst_maxqueue = npkts; 844 if (npkts < 2) 845 goto feed1; 846 847 /* 848 * Decide how many ops to combine in a single MCR. We cannot 849 * aggregate more than UBS_MAX_AGGR because this is the number 850 * of slots defined in the data structure. Otherwise we clamp 851 * based on the tunable parameter ubsec_maxaggr. Note that 852 * aggregation can happen in two ways: either by batching ops 853 * from above or because the h/w backs up and throttles us. 854 * Aggregating ops reduces the number of interrupts to the host 855 * but also (potentially) increases the latency for processing 856 * completed ops as we only get an interrupt when all aggregated 857 * ops have completed. 858 */ 859 if (npkts > sc->sc_maxaggr) 860 npkts = sc->sc_maxaggr; 861 if (npkts > ubsec_maxaggr) 862 npkts = ubsec_maxaggr; 863 if (npkts > ubsecstats.hst_maxbatch) 864 ubsecstats.hst_maxbatch = npkts; 865 if (npkts < 2) 866 goto feed1; 867 ubsecstats.hst_totbatch += npkts-1; 868 869 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 870 if (stat & BS_STAT_DMAERR) { 871 ubsec_totalreset(sc); 872 ubsecstats.hst_dmaerr++; 873 } else { 874 ubsecstats.hst_mcr1full++; 875 } 876 return; 877 } 878 879 #ifdef UBSEC_DEBUG 880 if (ubsec_debug) 881 printf("merging %d records\n", npkts); 882 /* XXX temporary aggregation statistics reporting code */ 883 if (max < npkts) { 884 max = npkts; 885 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), max); 886 } 887 #endif /* UBSEC_DEBUG */ 888 889 q = SIMPLEQ_FIRST(&sc->sc_queue); 890 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 891 --sc->sc_nqueue; 892 893 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 894 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 895 if (q->q_dst_map != NULL) 896 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 897 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 898 899 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 900 901 for (i = 0; i < q->q_nstacked_mcrs; i++) { 902 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 903 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 904 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 905 if (q2->q_dst_map != NULL) 906 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 907 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 908 q2= SIMPLEQ_FIRST(&sc->sc_queue); 909 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next); 910 --sc->sc_nqueue; 911 912 v = ((void *)&q2->q_dma->d_dma->d_mcr); 913 v = (char*)v + (sizeof(struct ubsec_mcr) - 914 sizeof(struct ubsec_mcr_add)); 915 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, sizeof(struct ubsec_mcr_add)); 916 q->q_stacked_mcr[i] = q2; 917 } 918 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 919 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 920 sc->sc_nqchip += npkts; 921 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 922 ubsecstats.hst_maxqchip = sc->sc_nqchip; 923 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 924 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 925 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 926 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 927 offsetof(struct ubsec_dmachunk, d_mcr)); 928 return; 929 930 feed1: 931 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 932 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 933 if (stat & BS_STAT_DMAERR) { 934 ubsec_totalreset(sc); 935 ubsecstats.hst_dmaerr++; 936 } else { 937 ubsecstats.hst_mcr1full++; 938 } 939 break; 940 } 941 942 q = SIMPLEQ_FIRST(&sc->sc_queue); 943 944 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 945 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 946 if (q->q_dst_map != NULL) 947 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 948 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 949 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 950 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 951 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 952 953 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 954 offsetof(struct ubsec_dmachunk, d_mcr)); 955 #ifdef UBSEC_DEBUG 956 if (ubsec_debug) 957 printf("feed: q->chip %p %08x stat %08x\n", 958 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr, 959 stat); 960 #endif /* UBSEC_DEBUG */ 961 q = SIMPLEQ_FIRST(&sc->sc_queue); 962 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 963 --sc->sc_nqueue; 964 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 965 sc->sc_nqchip++; 966 } 967 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 968 ubsecstats.hst_maxqchip = sc->sc_nqchip; 969 } 970 971 /* 972 * Allocate a new 'session' and return an encoded session id. 'sidp' 973 * contains our registration id, and should contain an encoded session 974 * id on successful allocation. 975 */ 976 static int 977 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 978 { 979 struct cryptoini *c, *encini = NULL, *macini = NULL; 980 struct ubsec_softc *sc; 981 struct ubsec_session *ses = NULL; 982 MD5_CTX md5ctx; 983 SHA1_CTX sha1ctx; 984 int i, sesn; 985 986 sc = arg; 987 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/); 988 989 if (sidp == NULL || cri == NULL || sc == NULL) 990 return (EINVAL); 991 992 for (c = cri; c != NULL; c = c->cri_next) { 993 if (c->cri_alg == CRYPTO_MD5_HMAC_96 || 994 c->cri_alg == CRYPTO_SHA1_HMAC_96) { 995 if (macini) 996 return (EINVAL); 997 macini = c; 998 } else if (c->cri_alg == CRYPTO_DES_CBC || 999 c->cri_alg == CRYPTO_3DES_CBC) { 1000 if (encini) 1001 return (EINVAL); 1002 encini = c; 1003 } else 1004 return (EINVAL); 1005 } 1006 if (encini == NULL && macini == NULL) 1007 return (EINVAL); 1008 1009 if (sc->sc_sessions == NULL) { 1010 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 1011 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1012 if (ses == NULL) 1013 return (ENOMEM); 1014 sesn = 0; 1015 sc->sc_nsessions = 1; 1016 } else { 1017 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 1018 if (sc->sc_sessions[sesn].ses_used == 0) { 1019 ses = &sc->sc_sessions[sesn]; 1020 break; 1021 } 1022 } 1023 1024 if (ses == NULL) { 1025 sesn = sc->sc_nsessions; 1026 ses = (struct ubsec_session *)malloc((sesn + 1) * 1027 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1028 if (ses == NULL) 1029 return (ENOMEM); 1030 memcpy(ses, sc->sc_sessions, sesn * 1031 sizeof(struct ubsec_session)); 1032 memset(sc->sc_sessions, 0, sesn * 1033 sizeof(struct ubsec_session)); 1034 free(sc->sc_sessions, M_DEVBUF); 1035 sc->sc_sessions = ses; 1036 ses = &sc->sc_sessions[sesn]; 1037 sc->sc_nsessions++; 1038 } 1039 } 1040 1041 memset(ses, 0, sizeof(struct ubsec_session)); 1042 ses->ses_used = 1; 1043 if (encini) { 1044 /* get an IV, network byte order */ 1045 #ifdef __NetBSD__ 1046 cprng_fast(ses->ses_iv, sizeof(ses->ses_iv)); 1047 #else 1048 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv)); 1049 #endif 1050 1051 /* Go ahead and compute key in ubsec's byte order */ 1052 if (encini->cri_alg == CRYPTO_DES_CBC) { 1053 memcpy(&ses->ses_deskey[0], encini->cri_key, 8); 1054 memcpy(&ses->ses_deskey[2], encini->cri_key, 8); 1055 memcpy(&ses->ses_deskey[4], encini->cri_key, 8); 1056 } else 1057 memcpy(ses->ses_deskey, encini->cri_key, 24); 1058 1059 SWAP32(ses->ses_deskey[0]); 1060 SWAP32(ses->ses_deskey[1]); 1061 SWAP32(ses->ses_deskey[2]); 1062 SWAP32(ses->ses_deskey[3]); 1063 SWAP32(ses->ses_deskey[4]); 1064 SWAP32(ses->ses_deskey[5]); 1065 } 1066 1067 if (macini) { 1068 for (i = 0; i < macini->cri_klen / 8; i++) 1069 macini->cri_key[i] ^= HMAC_IPAD_VAL; 1070 1071 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1072 MD5Init(&md5ctx); 1073 MD5Update(&md5ctx, macini->cri_key, 1074 macini->cri_klen / 8); 1075 MD5Update(&md5ctx, hmac_ipad_buffer, 1076 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1077 memcpy(ses->ses_hminner, md5ctx.state, 1078 sizeof(md5ctx.state)); 1079 } else { 1080 SHA1Init(&sha1ctx); 1081 SHA1Update(&sha1ctx, macini->cri_key, 1082 macini->cri_klen / 8); 1083 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1084 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1085 memcpy(ses->ses_hminner, sha1ctx.state, 1086 sizeof(sha1ctx.state)); 1087 } 1088 1089 for (i = 0; i < macini->cri_klen / 8; i++) 1090 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1091 1092 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1093 MD5Init(&md5ctx); 1094 MD5Update(&md5ctx, macini->cri_key, 1095 macini->cri_klen / 8); 1096 MD5Update(&md5ctx, hmac_opad_buffer, 1097 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1098 memcpy(ses->ses_hmouter, md5ctx.state, 1099 sizeof(md5ctx.state)); 1100 } else { 1101 SHA1Init(&sha1ctx); 1102 SHA1Update(&sha1ctx, macini->cri_key, 1103 macini->cri_klen / 8); 1104 SHA1Update(&sha1ctx, hmac_opad_buffer, 1105 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1106 memcpy(ses->ses_hmouter, sha1ctx.state, 1107 sizeof(sha1ctx.state)); 1108 } 1109 1110 for (i = 0; i < macini->cri_klen / 8; i++) 1111 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1112 } 1113 1114 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn); 1115 return (0); 1116 } 1117 1118 /* 1119 * Deallocate a session. 1120 */ 1121 static int 1122 ubsec_freesession(void *arg, u_int64_t tid) 1123 { 1124 struct ubsec_softc *sc; 1125 int session; 1126 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1127 1128 sc = arg; 1129 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/); 1130 1131 session = UBSEC_SESSION(sid); 1132 if (session >= sc->sc_nsessions) 1133 return (EINVAL); 1134 1135 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 1136 return (0); 1137 } 1138 1139 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */ 1140 static void 1141 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1142 { 1143 struct ubsec_operand *op = arg; 1144 1145 KASSERT(nsegs <= UBS_MAX_SCATTER 1146 /*, ("Too many DMA segments returned when mapping operand")*/); 1147 #ifdef UBSEC_DEBUG 1148 if (ubsec_debug) 1149 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 1150 (u_int) mapsize, nsegs); 1151 #endif 1152 op->mapsize = mapsize; 1153 op->nsegs = nsegs; 1154 memcpy(op->segs, seg, nsegs * sizeof (seg[0])); 1155 } 1156 #endif 1157 1158 static int 1159 ubsec_process(void *arg, struct cryptop *crp, int hint) 1160 { 1161 struct ubsec_q *q = NULL; 1162 #ifdef __OpenBSD__ 1163 int card; 1164 #endif 1165 int err = 0, i, j, nicealign; 1166 struct ubsec_softc *sc; 1167 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1168 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1169 int sskip, dskip, stheend, dtheend; 1170 int16_t coffset; 1171 struct ubsec_session *ses; 1172 struct ubsec_pktctx ctx; 1173 struct ubsec_dma *dmap = NULL; 1174 1175 sc = arg; 1176 KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/); 1177 1178 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1179 ubsecstats.hst_invalid++; 1180 return (EINVAL); 1181 } 1182 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1183 ubsecstats.hst_badsession++; 1184 return (EINVAL); 1185 } 1186 1187 mutex_spin_enter(&sc->sc_mtx); 1188 1189 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1190 ubsecstats.hst_queuefull++; 1191 sc->sc_needwakeup |= CRYPTO_SYMQ; 1192 mutex_spin_exit(&sc->sc_mtx); 1193 return(ERESTART); 1194 } 1195 1196 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1197 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next); 1198 mutex_spin_exit(&sc->sc_mtx); 1199 1200 dmap = q->q_dma; /* Save dma pointer */ 1201 memset(q, 0, sizeof(struct ubsec_q)); 1202 memset(&ctx, 0, sizeof(ctx)); 1203 1204 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1205 q->q_dma = dmap; 1206 ses = &sc->sc_sessions[q->q_sesn]; 1207 1208 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1209 q->q_src_m = (struct mbuf *)crp->crp_buf; 1210 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1211 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1212 q->q_src_io = (struct uio *)crp->crp_buf; 1213 q->q_dst_io = (struct uio *)crp->crp_buf; 1214 } else { 1215 ubsecstats.hst_badflags++; 1216 err = EINVAL; 1217 goto errout; /* XXX we don't handle contiguous blocks! */ 1218 } 1219 1220 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr)); 1221 1222 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1223 dmap->d_dma->d_mcr.mcr_flags = 0; 1224 q->q_crp = crp; 1225 1226 crd1 = crp->crp_desc; 1227 if (crd1 == NULL) { 1228 ubsecstats.hst_nodesc++; 1229 err = EINVAL; 1230 goto errout; 1231 } 1232 crd2 = crd1->crd_next; 1233 1234 if (crd2 == NULL) { 1235 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1236 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) { 1237 maccrd = crd1; 1238 enccrd = NULL; 1239 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1240 crd1->crd_alg == CRYPTO_3DES_CBC) { 1241 maccrd = NULL; 1242 enccrd = crd1; 1243 } else { 1244 ubsecstats.hst_badalg++; 1245 err = EINVAL; 1246 goto errout; 1247 } 1248 } else { 1249 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1250 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) && 1251 (crd2->crd_alg == CRYPTO_DES_CBC || 1252 crd2->crd_alg == CRYPTO_3DES_CBC) && 1253 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1254 maccrd = crd1; 1255 enccrd = crd2; 1256 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1257 crd1->crd_alg == CRYPTO_3DES_CBC) && 1258 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 1259 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) && 1260 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1261 enccrd = crd1; 1262 maccrd = crd2; 1263 } else { 1264 /* 1265 * We cannot order the ubsec as requested 1266 */ 1267 ubsecstats.hst_badalg++; 1268 err = EINVAL; 1269 goto errout; 1270 } 1271 } 1272 1273 if (enccrd) { 1274 encoffset = enccrd->crd_skip; 1275 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1276 1277 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1278 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1279 1280 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1281 memcpy(ctx.pc_iv, enccrd->crd_iv, 8); 1282 else { 1283 ctx.pc_iv[0] = ses->ses_iv[0]; 1284 ctx.pc_iv[1] = ses->ses_iv[1]; 1285 } 1286 1287 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1288 if (crp->crp_flags & CRYPTO_F_IMBUF) 1289 m_copyback(q->q_src_m, 1290 enccrd->crd_inject, 1291 8, (void *)ctx.pc_iv); 1292 else if (crp->crp_flags & CRYPTO_F_IOV) 1293 cuio_copyback(q->q_src_io, 1294 enccrd->crd_inject, 1295 8, (void *)ctx.pc_iv); 1296 } 1297 } else { 1298 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1299 1300 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1301 memcpy(ctx.pc_iv, enccrd->crd_iv, 8); 1302 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1303 m_copydata(q->q_src_m, enccrd->crd_inject, 1304 8, (void *)ctx.pc_iv); 1305 else if (crp->crp_flags & CRYPTO_F_IOV) 1306 cuio_copydata(q->q_src_io, 1307 enccrd->crd_inject, 8, 1308 (void *)ctx.pc_iv); 1309 } 1310 1311 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1312 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1313 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1314 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1315 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1316 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1317 SWAP32(ctx.pc_iv[0]); 1318 SWAP32(ctx.pc_iv[1]); 1319 } 1320 1321 if (maccrd) { 1322 macoffset = maccrd->crd_skip; 1323 1324 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96) 1325 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1326 else 1327 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1328 1329 for (i = 0; i < 5; i++) { 1330 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1331 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1332 1333 HTOLE32(ctx.pc_hminner[i]); 1334 HTOLE32(ctx.pc_hmouter[i]); 1335 } 1336 } 1337 1338 if (enccrd && maccrd) { 1339 /* 1340 * ubsec cannot handle packets where the end of encryption 1341 * and authentication are not the same, or where the 1342 * encrypted part begins before the authenticated part. 1343 */ 1344 if ((encoffset + enccrd->crd_len) != 1345 (macoffset + maccrd->crd_len)) { 1346 ubsecstats.hst_lenmismatch++; 1347 err = EINVAL; 1348 goto errout; 1349 } 1350 if (enccrd->crd_skip < maccrd->crd_skip) { 1351 ubsecstats.hst_skipmismatch++; 1352 err = EINVAL; 1353 goto errout; 1354 } 1355 sskip = maccrd->crd_skip; 1356 cpskip = dskip = enccrd->crd_skip; 1357 stheend = maccrd->crd_len; 1358 dtheend = enccrd->crd_len; 1359 coffset = enccrd->crd_skip - maccrd->crd_skip; 1360 cpoffset = cpskip + dtheend; 1361 #ifdef UBSEC_DEBUG 1362 if (ubsec_debug) { 1363 printf("mac: skip %d, len %d, inject %d\n", 1364 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1365 printf("enc: skip %d, len %d, inject %d\n", 1366 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1367 printf("src: skip %d, len %d\n", sskip, stheend); 1368 printf("dst: skip %d, len %d\n", dskip, dtheend); 1369 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1370 coffset, stheend, cpskip, cpoffset); 1371 } 1372 #endif 1373 } else { 1374 cpskip = dskip = sskip = macoffset + encoffset; 1375 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1376 cpoffset = cpskip + dtheend; 1377 coffset = 0; 1378 } 1379 ctx.pc_offset = htole16(coffset >> 2); 1380 1381 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */ 1382 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER, 1383 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) { 1384 err = ENOMEM; 1385 goto errout; 1386 } 1387 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1388 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1389 q->q_src_m, BUS_DMA_NOWAIT) != 0) { 1390 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1391 q->q_src_map = NULL; 1392 ubsecstats.hst_noload++; 1393 err = ENOMEM; 1394 goto errout; 1395 } 1396 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1397 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1398 q->q_src_io, BUS_DMA_NOWAIT) != 0) { 1399 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1400 q->q_src_map = NULL; 1401 ubsecstats.hst_noload++; 1402 err = ENOMEM; 1403 goto errout; 1404 } 1405 } 1406 nicealign = ubsec_dmamap_aligned(q->q_src_map); 1407 1408 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1409 1410 #ifdef UBSEC_DEBUG 1411 if (ubsec_debug) 1412 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1413 #endif 1414 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) { 1415 struct ubsec_pktbuf *pb; 1416 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len; 1417 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr; 1418 1419 if (sskip >= packl) { 1420 sskip -= packl; 1421 continue; 1422 } 1423 1424 packl -= sskip; 1425 packp += sskip; 1426 sskip = 0; 1427 1428 if (packl > 0xfffc) { 1429 err = EIO; 1430 goto errout; 1431 } 1432 1433 if (j == 0) 1434 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1435 else 1436 pb = &dmap->d_dma->d_sbuf[j - 1]; 1437 1438 pb->pb_addr = htole32(packp); 1439 1440 if (stheend) { 1441 if (packl > stheend) { 1442 pb->pb_len = htole32(stheend); 1443 stheend = 0; 1444 } else { 1445 pb->pb_len = htole32(packl); 1446 stheend -= packl; 1447 } 1448 } else 1449 pb->pb_len = htole32(packl); 1450 1451 if ((i + 1) == q->q_src_map->dm_nsegs) 1452 pb->pb_next = 0; 1453 else 1454 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1455 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1456 j++; 1457 } 1458 1459 if (enccrd == NULL && maccrd != NULL) { 1460 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1461 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1462 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1463 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1464 #ifdef UBSEC_DEBUG 1465 if (ubsec_debug) 1466 printf("opkt: %x %x %x\n", 1467 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1468 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1469 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1470 1471 #endif 1472 } else { 1473 if (crp->crp_flags & CRYPTO_F_IOV) { 1474 if (!nicealign) { 1475 ubsecstats.hst_iovmisaligned++; 1476 err = EINVAL; 1477 goto errout; 1478 } 1479 /* XXX: ``what the heck's that'' 0xfff0? */ 1480 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1481 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1482 &q->q_dst_map) != 0) { 1483 ubsecstats.hst_nomap++; 1484 err = ENOMEM; 1485 goto errout; 1486 } 1487 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1488 q->q_dst_io, BUS_DMA_NOWAIT) != 0) { 1489 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1490 q->q_dst_map = NULL; 1491 ubsecstats.hst_noload++; 1492 err = ENOMEM; 1493 goto errout; 1494 } 1495 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1496 if (nicealign) { 1497 q->q_dst_m = q->q_src_m; 1498 q->q_dst_map = q->q_src_map; 1499 } else { 1500 int totlen, len; 1501 struct mbuf *m, *top, **mp; 1502 1503 ubsecstats.hst_unaligned++; 1504 totlen = q->q_src_map->dm_mapsize; 1505 if (q->q_src_m->m_flags & M_PKTHDR) { 1506 len = MHLEN; 1507 MGETHDR(m, M_DONTWAIT, MT_DATA); 1508 /*XXX FIXME: m_dup_pkthdr */ 1509 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) { 1510 m_free(m); 1511 m = NULL; 1512 } 1513 } else { 1514 len = MLEN; 1515 MGET(m, M_DONTWAIT, MT_DATA); 1516 } 1517 if (m == NULL) { 1518 ubsecstats.hst_nombuf++; 1519 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1520 goto errout; 1521 } 1522 if (len == MHLEN) 1523 /*XXX was M_DUP_PKTHDR*/ 1524 M_COPY_PKTHDR(m, q->q_src_m); 1525 if (totlen >= MINCLSIZE) { 1526 MCLGET(m, M_DONTWAIT); 1527 if ((m->m_flags & M_EXT) == 0) { 1528 m_free(m); 1529 ubsecstats.hst_nomcl++; 1530 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1531 goto errout; 1532 } 1533 len = MCLBYTES; 1534 } 1535 m->m_len = len; 1536 top = NULL; 1537 mp = ⊤ 1538 1539 while (totlen > 0) { 1540 if (top) { 1541 MGET(m, M_DONTWAIT, MT_DATA); 1542 if (m == NULL) { 1543 m_freem(top); 1544 ubsecstats.hst_nombuf++; 1545 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1546 goto errout; 1547 } 1548 len = MLEN; 1549 } 1550 if (top && totlen >= MINCLSIZE) { 1551 MCLGET(m, M_DONTWAIT); 1552 if ((m->m_flags & M_EXT) == 0) { 1553 *mp = m; 1554 m_freem(top); 1555 ubsecstats.hst_nomcl++; 1556 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1557 goto errout; 1558 } 1559 len = MCLBYTES; 1560 } 1561 m->m_len = len = min(totlen, len); 1562 totlen -= len; 1563 *mp = m; 1564 mp = &m->m_next; 1565 } 1566 q->q_dst_m = top; 1567 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1568 cpskip, cpoffset); 1569 /* XXX again, what the heck is that 0xfff0? */ 1570 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1571 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1572 &q->q_dst_map) != 0) { 1573 ubsecstats.hst_nomap++; 1574 err = ENOMEM; 1575 goto errout; 1576 } 1577 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1578 q->q_dst_map, q->q_dst_m, 1579 BUS_DMA_NOWAIT) != 0) { 1580 bus_dmamap_destroy(sc->sc_dmat, 1581 q->q_dst_map); 1582 q->q_dst_map = NULL; 1583 ubsecstats.hst_noload++; 1584 err = ENOMEM; 1585 goto errout; 1586 } 1587 } 1588 } else { 1589 ubsecstats.hst_badflags++; 1590 err = EINVAL; 1591 goto errout; 1592 } 1593 1594 #ifdef UBSEC_DEBUG 1595 if (ubsec_debug) 1596 printf("dst skip: %d\n", dskip); 1597 #endif 1598 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) { 1599 struct ubsec_pktbuf *pb; 1600 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len; 1601 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr; 1602 1603 if (dskip >= packl) { 1604 dskip -= packl; 1605 continue; 1606 } 1607 1608 packl -= dskip; 1609 packp += dskip; 1610 dskip = 0; 1611 1612 if (packl > 0xfffc) { 1613 err = EIO; 1614 goto errout; 1615 } 1616 1617 if (j == 0) 1618 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1619 else 1620 pb = &dmap->d_dma->d_dbuf[j - 1]; 1621 1622 pb->pb_addr = htole32(packp); 1623 1624 if (dtheend) { 1625 if (packl > dtheend) { 1626 pb->pb_len = htole32(dtheend); 1627 dtheend = 0; 1628 } else { 1629 pb->pb_len = htole32(packl); 1630 dtheend -= packl; 1631 } 1632 } else 1633 pb->pb_len = htole32(packl); 1634 1635 if ((i + 1) == q->q_dst_map->dm_nsegs) { 1636 if (maccrd) 1637 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1638 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1639 else 1640 pb->pb_next = 0; 1641 } else 1642 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1643 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1644 j++; 1645 } 1646 } 1647 1648 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1649 offsetof(struct ubsec_dmachunk, d_ctx)); 1650 1651 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1652 struct ubsec_pktctx_long *ctxl; 1653 1654 ctxl = (struct ubsec_pktctx_long *)((char *)dmap->d_alloc.dma_vaddr + 1655 offsetof(struct ubsec_dmachunk, d_ctx)); 1656 1657 /* transform small context into long context */ 1658 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1659 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1660 ctxl->pc_flags = ctx.pc_flags; 1661 ctxl->pc_offset = ctx.pc_offset; 1662 for (i = 0; i < 6; i++) 1663 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1664 for (i = 0; i < 5; i++) 1665 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1666 for (i = 0; i < 5; i++) 1667 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1668 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1669 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1670 } else 1671 memcpy((char *)dmap->d_alloc.dma_vaddr + 1672 offsetof(struct ubsec_dmachunk, d_ctx), &ctx, 1673 sizeof(struct ubsec_pktctx)); 1674 1675 mutex_spin_enter(&sc->sc_mtx); 1676 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1677 sc->sc_nqueue++; 1678 ubsecstats.hst_ipackets++; 1679 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize; 1680 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1681 ubsec_feed(sc); 1682 mutex_spin_exit(&sc->sc_mtx); 1683 return (0); 1684 1685 errout: 1686 if (q != NULL) { 1687 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1688 m_freem(q->q_dst_m); 1689 1690 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1691 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1692 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1693 } 1694 if (q->q_src_map != NULL) { 1695 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1696 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1697 } 1698 1699 mutex_spin_enter(&sc->sc_mtx); 1700 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1701 mutex_spin_exit(&sc->sc_mtx); 1702 } 1703 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */ 1704 if (err == EINVAL) 1705 ubsecstats.hst_invalid++; 1706 else 1707 ubsecstats.hst_nomem++; 1708 #endif 1709 if (err != ERESTART) { 1710 crp->crp_etype = err; 1711 crypto_done(crp); 1712 } else { 1713 sc->sc_needwakeup |= CRYPTO_SYMQ; 1714 } 1715 return (err); 1716 } 1717 1718 static void 1719 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1720 { 1721 struct cryptop *crp = (struct cryptop *)q->q_crp; 1722 struct cryptodesc *crd; 1723 struct ubsec_dma *dmap = q->q_dma; 1724 1725 ubsecstats.hst_opackets++; 1726 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1727 1728 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0, 1729 dmap->d_alloc.dma_map->dm_mapsize, 1730 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1731 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1732 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1733 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1734 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1735 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1736 } 1737 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 1738 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1739 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1740 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1741 1742 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1743 m_freem(q->q_src_m); 1744 crp->crp_buf = (void *)q->q_dst_m; 1745 } 1746 1747 /* copy out IV for future use */ 1748 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1749 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1750 if (crd->crd_alg != CRYPTO_DES_CBC && 1751 crd->crd_alg != CRYPTO_3DES_CBC) 1752 continue; 1753 if (crp->crp_flags & CRYPTO_F_IMBUF) 1754 m_copydata((struct mbuf *)crp->crp_buf, 1755 crd->crd_skip + crd->crd_len - 8, 8, 1756 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1757 else if (crp->crp_flags & CRYPTO_F_IOV) { 1758 cuio_copydata((struct uio *)crp->crp_buf, 1759 crd->crd_skip + crd->crd_len - 8, 8, 1760 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1761 } 1762 break; 1763 } 1764 } 1765 1766 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1767 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 && 1768 crd->crd_alg != CRYPTO_SHA1_HMAC_96) 1769 continue; 1770 if (crp->crp_flags & CRYPTO_F_IMBUF) 1771 m_copyback((struct mbuf *)crp->crp_buf, 1772 crd->crd_inject, 12, 1773 (void *)dmap->d_dma->d_macbuf); 1774 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1775 bcopy((void *)dmap->d_dma->d_macbuf, 1776 crp->crp_mac, 12); 1777 break; 1778 } 1779 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1780 crypto_done(crp); 1781 } 1782 1783 static void 1784 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1785 { 1786 int i, j, dlen, slen; 1787 char *dptr, *sptr; 1788 1789 j = 0; 1790 sptr = srcm->m_data; 1791 slen = srcm->m_len; 1792 dptr = dstm->m_data; 1793 dlen = dstm->m_len; 1794 1795 while (1) { 1796 for (i = 0; i < min(slen, dlen); i++) { 1797 if (j < hoffset || j >= toffset) 1798 *dptr++ = *sptr++; 1799 slen--; 1800 dlen--; 1801 j++; 1802 } 1803 if (slen == 0) { 1804 srcm = srcm->m_next; 1805 if (srcm == NULL) 1806 return; 1807 sptr = srcm->m_data; 1808 slen = srcm->m_len; 1809 } 1810 if (dlen == 0) { 1811 dstm = dstm->m_next; 1812 if (dstm == NULL) 1813 return; 1814 dptr = dstm->m_data; 1815 dlen = dstm->m_len; 1816 } 1817 } 1818 } 1819 1820 /* 1821 * feed the key generator, must be called at splnet() or higher. 1822 */ 1823 static void 1824 ubsec_feed2(struct ubsec_softc *sc) 1825 { 1826 struct ubsec_q2 *q; 1827 1828 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1829 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1830 break; 1831 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1832 1833 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1834 q->q_mcr.dma_map->dm_mapsize, 1835 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1836 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1837 q->q_ctx.dma_map->dm_mapsize, 1838 BUS_DMASYNC_PREWRITE); 1839 1840 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1841 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1842 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next); 1843 --sc->sc_nqueue2; 1844 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1845 } 1846 } 1847 1848 /* 1849 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices) 1850 */ 1851 void 1852 ubsec_feed4(struct ubsec_softc *sc) 1853 { 1854 struct ubsec_q2 *q; 1855 1856 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) { 1857 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL) 1858 break; 1859 q = SIMPLEQ_FIRST(&sc->sc_queue4); 1860 1861 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1862 q->q_mcr.dma_map->dm_mapsize, 1863 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1864 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1865 q->q_ctx.dma_map->dm_mapsize, 1866 BUS_DMASYNC_PREWRITE); 1867 1868 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr); 1869 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next); 1870 --sc->sc_nqueue4; 1871 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next); 1872 } 1873 } 1874 1875 /* 1876 * Callback for handling random numbers 1877 */ 1878 static void 1879 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1880 { 1881 struct cryptkop *krp; 1882 struct ubsec_ctx_keyop *ctx; 1883 1884 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1885 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1886 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1887 1888 switch (q->q_type) { 1889 #ifndef UBSEC_NO_RNG 1890 case UBS_CTXOP_RNGSHA1: 1891 case UBS_CTXOP_RNGBYPASS: { 1892 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1893 u_int32_t *p; 1894 int i; 1895 1896 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 1897 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1898 p = (u_int32_t *)rng->rng_buf.dma_vaddr; 1899 #ifndef __NetBSD__ 1900 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++) 1901 add_true_randomness(letoh32(*p)); 1902 #else 1903 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t); 1904 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY); 1905 sc->sc_rng_need -= i; 1906 #endif 1907 rng->rng_used = 0; 1908 #ifdef __OpenBSD__ 1909 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 1910 #else 1911 if (sc->sc_rng_need > 0) { 1912 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 1913 } 1914 #endif 1915 break; 1916 } 1917 #endif 1918 case UBS_CTXOP_MODEXP: { 1919 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1920 u_int rlen, clen; 1921 1922 krp = me->me_krp; 1923 rlen = (me->me_modbits + 7) / 8; 1924 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1925 1926 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 1927 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1928 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 1929 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1930 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 1931 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1932 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 1933 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1934 1935 if (clen < rlen) 1936 krp->krp_status = E2BIG; 1937 else { 1938 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1939 memset(krp->krp_param[krp->krp_iparams].crp_p, 0, 1940 (krp->krp_param[krp->krp_iparams].crp_nbits 1941 + 7) / 8); 1942 bcopy(me->me_C.dma_vaddr, 1943 krp->krp_param[krp->krp_iparams].crp_p, 1944 (me->me_modbits + 7) / 8); 1945 } else 1946 ubsec_kshift_l(me->me_shiftbits, 1947 me->me_C.dma_vaddr, me->me_normbits, 1948 krp->krp_param[krp->krp_iparams].crp_p, 1949 krp->krp_param[krp->krp_iparams].crp_nbits); 1950 } 1951 1952 crypto_kdone(krp); 1953 1954 /* bzero all potentially sensitive data */ 1955 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 1956 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 1957 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 1958 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 1959 1960 /* Can't free here, so put us on the free list. */ 1961 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1962 break; 1963 } 1964 case UBS_CTXOP_RSAPRIV: { 1965 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1966 u_int len; 1967 1968 krp = rp->rpr_krp; 1969 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0, 1970 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1971 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0, 1972 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1973 1974 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1975 bcopy(rp->rpr_msgout.dma_vaddr, 1976 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1977 1978 crypto_kdone(krp); 1979 1980 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 1981 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 1982 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size); 1983 1984 /* Can't free here, so put us on the free list. */ 1985 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1986 break; 1987 } 1988 default: 1989 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev), 1990 letoh16(ctx->ctx_op)); 1991 break; 1992 } 1993 } 1994 1995 #ifndef UBSEC_NO_RNG 1996 1997 static void 1998 ubsec_rng_get(size_t bytes, void *vsc) 1999 { 2000 struct ubsec_softc *sc = vsc; 2001 2002 mutex_spin_enter(&sc->sc_mtx); 2003 sc->sc_rng_need = bytes; 2004 ubsec_rng_locked(sc); 2005 mutex_spin_exit(&sc->sc_mtx); 2006 2007 } 2008 2009 static void 2010 ubsec_rng(void *vsc) 2011 { 2012 struct ubsec_softc *sc = vsc; 2013 mutex_spin_enter(&sc->sc_mtx); 2014 ubsec_rng_locked(sc); 2015 mutex_spin_exit(&sc->sc_mtx); 2016 } 2017 2018 static void 2019 ubsec_rng_locked(void *vsc) 2020 { 2021 struct ubsec_softc *sc = vsc; 2022 struct ubsec_q2_rng *rng = &sc->sc_rng; 2023 struct ubsec_mcr *mcr; 2024 struct ubsec_ctx_rngbypass *ctx; 2025 int *nqueue; 2026 2027 /* Caller is responsible to lock and release sc_mtx. */ 2028 KASSERT(mutex_owned(&sc->sc_mtx)); 2029 2030 if (rng->rng_used) { 2031 return; 2032 } 2033 2034 if (sc->sc_rng_need < 1) { 2035 callout_stop(&sc->sc_rngto); 2036 return; 2037 } 2038 2039 if (sc->sc_flags & UBS_FLAGS_RNG4) 2040 nqueue = &sc->sc_nqueue4; 2041 else 2042 nqueue = &sc->sc_nqueue2; 2043 2044 (*nqueue)++; 2045 if (*nqueue >= UBS_MAX_NQUEUE) 2046 goto out; 2047 2048 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 2049 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 2050 2051 mcr->mcr_pkts = htole16(1); 2052 mcr->mcr_flags = 0; 2053 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 2054 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 2055 mcr->mcr_ipktbuf.pb_len = 0; 2056 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 2057 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 2058 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 2059 UBS_PKTBUF_LEN); 2060 mcr->mcr_opktbuf.pb_next = 0; 2061 2062 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 2063 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1); 2064 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1; 2065 2066 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2067 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2068 2069 if (sc->sc_flags & UBS_FLAGS_RNG4) { 2070 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next); 2071 ubsec_feed4(sc); 2072 } else { 2073 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 2074 ubsec_feed2(sc); 2075 } 2076 rng->rng_used = 1; 2077 ubsecstats.hst_rng++; 2078 2079 return; 2080 2081 out: 2082 /* 2083 * Something weird happened, generate our own call back. 2084 */ 2085 (*nqueue)--; 2086 #ifdef __OpenBSD__ 2087 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2088 #else 2089 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2090 #endif 2091 } 2092 #endif /* UBSEC_NO_RNG */ 2093 2094 static int 2095 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size, 2096 struct ubsec_dma_alloc *dma,int mapflags) 2097 { 2098 int r; 2099 2100 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, 2101 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0) 2102 goto fail_0; 2103 2104 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, 2105 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0) 2106 goto fail_1; 2107 2108 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2109 BUS_DMA_NOWAIT, &dma->dma_map)) != 0) 2110 goto fail_2; 2111 2112 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 2113 size, NULL, BUS_DMA_NOWAIT)) != 0) 2114 goto fail_3; 2115 2116 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2117 dma->dma_size = size; 2118 return (0); 2119 2120 fail_3: 2121 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2122 fail_2: 2123 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size); 2124 fail_1: 2125 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2126 fail_0: 2127 dma->dma_map = NULL; 2128 return (r); 2129 } 2130 2131 static void 2132 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 2133 { 2134 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 2135 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size); 2136 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2137 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2138 } 2139 2140 /* 2141 * Resets the board. Values in the regesters are left as is 2142 * from the reset (i.e. initial values are assigned elsewhere). 2143 */ 2144 static void 2145 ubsec_reset_board(struct ubsec_softc *sc) 2146 { 2147 volatile u_int32_t ctrl; 2148 2149 ctrl = READ_REG(sc, BS_CTRL); 2150 ctrl |= BS_CTRL_RESET; 2151 WRITE_REG(sc, BS_CTRL, ctrl); 2152 2153 /* 2154 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2155 */ 2156 DELAY(10); 2157 2158 /* Enable RNG and interrupts on newer devices */ 2159 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2160 #ifndef UBSEC_NO_RNG 2161 WRITE_REG(sc, BS_CFG, BS_CFG_RNG); 2162 #endif 2163 WRITE_REG(sc, BS_INT, BS_INT_DMAINT); 2164 } 2165 } 2166 2167 /* 2168 * Init Broadcom registers 2169 */ 2170 static void 2171 ubsec_init_board(struct ubsec_softc *sc) 2172 { 2173 u_int32_t ctrl; 2174 2175 ctrl = READ_REG(sc, BS_CTRL); 2176 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2177 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2178 2179 /* 2180 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)). 2181 * anyone got hw docs? 2182 */ 2183 if (sc->sc_flags & UBS_FLAGS_KEY) 2184 ctrl |= BS_CTRL_MCR2INT; 2185 else 2186 ctrl &= ~BS_CTRL_MCR2INT; 2187 2188 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2189 ctrl &= ~BS_CTRL_SWNORM; 2190 2191 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2192 ctrl |= BS_CTRL_BSIZE240; 2193 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */ 2194 2195 if (sc->sc_flags & UBS_FLAGS_RNG4) 2196 ctrl |= BS_CTRL_MCR4INT; 2197 else 2198 ctrl &= ~BS_CTRL_MCR4INT; 2199 } 2200 2201 WRITE_REG(sc, BS_CTRL, ctrl); 2202 } 2203 2204 /* 2205 * Init Broadcom PCI registers 2206 */ 2207 static void 2208 ubsec_init_pciregs(struct pci_attach_args *pa) 2209 { 2210 pci_chipset_tag_t pc = pa->pa_pc; 2211 u_int32_t misc; 2212 2213 /* 2214 * This will set the cache line size to 1, this will 2215 * force the BCM58xx chip just to do burst read/writes. 2216 * Cache line read/writes are to slow 2217 */ 2218 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG); 2219 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT)) 2220 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT); 2221 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc); 2222 } 2223 2224 /* 2225 * Clean up after a chip crash. 2226 * It is assumed that the caller in splnet() 2227 */ 2228 static void 2229 ubsec_cleanchip(struct ubsec_softc *sc) 2230 { 2231 struct ubsec_q *q; 2232 2233 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2234 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2235 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 2236 ubsec_free_q(sc, q); 2237 } 2238 sc->sc_nqchip = 0; 2239 } 2240 2241 /* 2242 * free a ubsec_q 2243 * It is assumed that the caller is within splnet() 2244 */ 2245 static int 2246 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2247 { 2248 struct ubsec_q *q2; 2249 struct cryptop *crp; 2250 int npkts; 2251 int i; 2252 2253 npkts = q->q_nstacked_mcrs; 2254 2255 for (i = 0; i < npkts; i++) { 2256 if(q->q_stacked_mcr[i]) { 2257 q2 = q->q_stacked_mcr[i]; 2258 2259 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2260 m_freem(q2->q_dst_m); 2261 2262 crp = (struct cryptop *)q2->q_crp; 2263 2264 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2265 2266 crp->crp_etype = EFAULT; 2267 crypto_done(crp); 2268 } else { 2269 break; 2270 } 2271 } 2272 2273 /* 2274 * Free header MCR 2275 */ 2276 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2277 m_freem(q->q_dst_m); 2278 2279 crp = (struct cryptop *)q->q_crp; 2280 2281 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2282 2283 crp->crp_etype = EFAULT; 2284 crypto_done(crp); 2285 return(0); 2286 } 2287 2288 /* 2289 * Routine to reset the chip and clean up. 2290 * It is assumed that the caller is in splnet() 2291 */ 2292 static void 2293 ubsec_totalreset(struct ubsec_softc *sc) 2294 { 2295 ubsec_reset_board(sc); 2296 ubsec_init_board(sc); 2297 ubsec_cleanchip(sc); 2298 } 2299 2300 static int 2301 ubsec_dmamap_aligned(bus_dmamap_t map) 2302 { 2303 int i; 2304 2305 for (i = 0; i < map->dm_nsegs; i++) { 2306 if (map->dm_segs[i].ds_addr & 3) 2307 return (0); 2308 if ((i != (map->dm_nsegs - 1)) && 2309 (map->dm_segs[i].ds_len & 3)) 2310 return (0); 2311 } 2312 return (1); 2313 } 2314 2315 #ifdef __OpenBSD__ 2316 struct ubsec_softc * 2317 ubsec_kfind(struct cryptkop *krp) 2318 { 2319 struct ubsec_softc *sc; 2320 int i; 2321 2322 for (i = 0; i < ubsec_cd.cd_ndevs; i++) { 2323 sc = ubsec_cd.cd_devs[i]; 2324 if (sc == NULL) 2325 continue; 2326 if (sc->sc_cid == krp->krp_hid) 2327 return (sc); 2328 } 2329 return (NULL); 2330 } 2331 #endif 2332 2333 static void 2334 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2335 { 2336 switch (q->q_type) { 2337 case UBS_CTXOP_MODEXP: { 2338 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2339 2340 ubsec_dma_free(sc, &me->me_q.q_mcr); 2341 ubsec_dma_free(sc, &me->me_q.q_ctx); 2342 ubsec_dma_free(sc, &me->me_M); 2343 ubsec_dma_free(sc, &me->me_E); 2344 ubsec_dma_free(sc, &me->me_C); 2345 ubsec_dma_free(sc, &me->me_epb); 2346 free(me, M_DEVBUF); 2347 break; 2348 } 2349 case UBS_CTXOP_RSAPRIV: { 2350 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2351 2352 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2353 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2354 ubsec_dma_free(sc, &rp->rpr_msgin); 2355 ubsec_dma_free(sc, &rp->rpr_msgout); 2356 free(rp, M_DEVBUF); 2357 break; 2358 } 2359 default: 2360 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev), 2361 q->q_type); 2362 break; 2363 } 2364 } 2365 2366 static int 2367 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2368 { 2369 struct ubsec_softc *sc; 2370 int r; 2371 2372 if (krp == NULL || krp->krp_callback == NULL) 2373 return (EINVAL); 2374 #ifdef __OpenBSD__ 2375 if ((sc = ubsec_kfind(krp)) == NULL) 2376 return (EINVAL); 2377 #else 2378 sc = arg; 2379 KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/); 2380 #endif 2381 2382 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2383 struct ubsec_q2 *q; 2384 2385 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2386 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next); 2387 ubsec_kfree(sc, q); 2388 } 2389 2390 switch (krp->krp_op) { 2391 case CRK_MOD_EXP: 2392 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2393 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2394 else 2395 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2396 break; 2397 case CRK_MOD_EXP_CRT: 2398 r = ubsec_kprocess_rsapriv(sc, krp, hint); 2399 break; 2400 default: 2401 printf("%s: kprocess: invalid op 0x%x\n", 2402 device_xname(sc->sc_dev), krp->krp_op); 2403 krp->krp_status = EOPNOTSUPP; 2404 crypto_kdone(krp); 2405 r = 0; 2406 } 2407 return (r); 2408 } 2409 2410 /* 2411 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2412 */ 2413 static int 2414 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, 2415 int hint) 2416 { 2417 struct ubsec_q2_modexp *me; 2418 struct ubsec_mcr *mcr; 2419 struct ubsec_ctx_modexp *ctx; 2420 struct ubsec_pktbuf *epb; 2421 int err = 0; 2422 u_int nbits, normbits, mbits, shiftbits, ebits; 2423 2424 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2425 if (me == NULL) { 2426 err = ENOMEM; 2427 goto errout; 2428 } 2429 memset(me, 0, sizeof *me); 2430 me->me_krp = krp; 2431 me->me_q.q_type = UBS_CTXOP_MODEXP; 2432 2433 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2434 if (nbits <= 512) 2435 normbits = 512; 2436 else if (nbits <= 768) 2437 normbits = 768; 2438 else if (nbits <= 1024) 2439 normbits = 1024; 2440 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2441 normbits = 1536; 2442 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2443 normbits = 2048; 2444 else { 2445 err = E2BIG; 2446 goto errout; 2447 } 2448 2449 shiftbits = normbits - nbits; 2450 2451 me->me_modbits = nbits; 2452 me->me_shiftbits = shiftbits; 2453 me->me_normbits = normbits; 2454 2455 /* Sanity check: result bits must be >= true modulus bits. */ 2456 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2457 err = ERANGE; 2458 goto errout; 2459 } 2460 2461 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2462 &me->me_q.q_mcr, 0)) { 2463 err = ENOMEM; 2464 goto errout; 2465 } 2466 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2467 2468 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2469 &me->me_q.q_ctx, 0)) { 2470 err = ENOMEM; 2471 goto errout; 2472 } 2473 2474 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2475 if (mbits > nbits) { 2476 err = E2BIG; 2477 goto errout; 2478 } 2479 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2480 err = ENOMEM; 2481 goto errout; 2482 } 2483 ubsec_kshift_r(shiftbits, 2484 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2485 me->me_M.dma_vaddr, normbits); 2486 2487 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2488 err = ENOMEM; 2489 goto errout; 2490 } 2491 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2492 2493 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2494 if (ebits > nbits) { 2495 err = E2BIG; 2496 goto errout; 2497 } 2498 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2499 err = ENOMEM; 2500 goto errout; 2501 } 2502 ubsec_kshift_r(shiftbits, 2503 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2504 me->me_E.dma_vaddr, normbits); 2505 2506 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2507 &me->me_epb, 0)) { 2508 err = ENOMEM; 2509 goto errout; 2510 } 2511 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2512 epb->pb_addr = htole32(me->me_E.dma_paddr); 2513 epb->pb_next = 0; 2514 epb->pb_len = htole32(normbits / 8); 2515 2516 #ifdef UBSEC_DEBUG 2517 if (ubsec_debug) { 2518 printf("Epb "); 2519 ubsec_dump_pb(epb); 2520 } 2521 #endif 2522 2523 mcr->mcr_pkts = htole16(1); 2524 mcr->mcr_flags = 0; 2525 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2526 mcr->mcr_reserved = 0; 2527 mcr->mcr_pktlen = 0; 2528 2529 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2530 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2531 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2532 2533 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2534 mcr->mcr_opktbuf.pb_next = 0; 2535 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2536 2537 #ifdef DIAGNOSTIC 2538 /* Misaligned output buffer will hang the chip. */ 2539 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2540 panic("%s: modexp invalid addr 0x%x", 2541 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr)); 2542 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2543 panic("%s: modexp invalid len 0x%x", 2544 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len)); 2545 #endif 2546 2547 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2548 memset(ctx, 0, sizeof(*ctx)); 2549 ubsec_kshift_r(shiftbits, 2550 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2551 ctx->me_N, normbits); 2552 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2553 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2554 ctx->me_E_len = htole16(nbits); 2555 ctx->me_N_len = htole16(nbits); 2556 2557 #ifdef UBSEC_DEBUG 2558 if (ubsec_debug) { 2559 ubsec_dump_mcr(mcr); 2560 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2561 } 2562 #endif 2563 2564 /* 2565 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2566 * everything else. 2567 */ 2568 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2569 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2570 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2571 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2572 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2573 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2574 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2575 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2576 2577 /* Enqueue and we're done... */ 2578 mutex_spin_enter(&sc->sc_mtx); 2579 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2580 ubsec_feed2(sc); 2581 ubsecstats.hst_modexp++; 2582 mutex_spin_exit(&sc->sc_mtx); 2583 2584 return (0); 2585 2586 errout: 2587 if (me != NULL) { 2588 if (me->me_q.q_mcr.dma_map != NULL) 2589 ubsec_dma_free(sc, &me->me_q.q_mcr); 2590 if (me->me_q.q_ctx.dma_map != NULL) { 2591 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2592 ubsec_dma_free(sc, &me->me_q.q_ctx); 2593 } 2594 if (me->me_M.dma_map != NULL) { 2595 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2596 ubsec_dma_free(sc, &me->me_M); 2597 } 2598 if (me->me_E.dma_map != NULL) { 2599 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2600 ubsec_dma_free(sc, &me->me_E); 2601 } 2602 if (me->me_C.dma_map != NULL) { 2603 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2604 ubsec_dma_free(sc, &me->me_C); 2605 } 2606 if (me->me_epb.dma_map != NULL) 2607 ubsec_dma_free(sc, &me->me_epb); 2608 free(me, M_DEVBUF); 2609 } 2610 krp->krp_status = err; 2611 crypto_kdone(krp); 2612 return (0); 2613 } 2614 2615 /* 2616 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2617 */ 2618 static int 2619 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, 2620 int hint) 2621 { 2622 struct ubsec_q2_modexp *me; 2623 struct ubsec_mcr *mcr; 2624 struct ubsec_ctx_modexp *ctx; 2625 struct ubsec_pktbuf *epb; 2626 int err = 0; 2627 u_int nbits, normbits, mbits, shiftbits, ebits; 2628 2629 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2630 if (me == NULL) { 2631 err = ENOMEM; 2632 goto errout; 2633 } 2634 memset(me, 0, sizeof *me); 2635 me->me_krp = krp; 2636 me->me_q.q_type = UBS_CTXOP_MODEXP; 2637 2638 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2639 if (nbits <= 512) 2640 normbits = 512; 2641 else if (nbits <= 768) 2642 normbits = 768; 2643 else if (nbits <= 1024) 2644 normbits = 1024; 2645 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2646 normbits = 1536; 2647 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2648 normbits = 2048; 2649 else { 2650 err = E2BIG; 2651 goto errout; 2652 } 2653 2654 shiftbits = normbits - nbits; 2655 2656 /* XXX ??? */ 2657 me->me_modbits = nbits; 2658 me->me_shiftbits = shiftbits; 2659 me->me_normbits = normbits; 2660 2661 /* Sanity check: result bits must be >= true modulus bits. */ 2662 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2663 err = ERANGE; 2664 goto errout; 2665 } 2666 2667 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2668 &me->me_q.q_mcr, 0)) { 2669 err = ENOMEM; 2670 goto errout; 2671 } 2672 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2673 2674 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2675 &me->me_q.q_ctx, 0)) { 2676 err = ENOMEM; 2677 goto errout; 2678 } 2679 2680 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2681 if (mbits > nbits) { 2682 err = E2BIG; 2683 goto errout; 2684 } 2685 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2686 err = ENOMEM; 2687 goto errout; 2688 } 2689 memset(me->me_M.dma_vaddr, 0, normbits / 8); 2690 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2691 me->me_M.dma_vaddr, (mbits + 7) / 8); 2692 2693 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2694 err = ENOMEM; 2695 goto errout; 2696 } 2697 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2698 2699 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2700 if (ebits > nbits) { 2701 err = E2BIG; 2702 goto errout; 2703 } 2704 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2705 err = ENOMEM; 2706 goto errout; 2707 } 2708 memset(me->me_E.dma_vaddr, 0, normbits / 8); 2709 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2710 me->me_E.dma_vaddr, (ebits + 7) / 8); 2711 2712 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2713 &me->me_epb, 0)) { 2714 err = ENOMEM; 2715 goto errout; 2716 } 2717 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2718 epb->pb_addr = htole32(me->me_E.dma_paddr); 2719 epb->pb_next = 0; 2720 epb->pb_len = htole32((ebits + 7) / 8); 2721 2722 #ifdef UBSEC_DEBUG 2723 if (ubsec_debug) { 2724 printf("Epb "); 2725 ubsec_dump_pb(epb); 2726 } 2727 #endif 2728 2729 mcr->mcr_pkts = htole16(1); 2730 mcr->mcr_flags = 0; 2731 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2732 mcr->mcr_reserved = 0; 2733 mcr->mcr_pktlen = 0; 2734 2735 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2736 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2737 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2738 2739 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2740 mcr->mcr_opktbuf.pb_next = 0; 2741 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2742 2743 #ifdef DIAGNOSTIC 2744 /* Misaligned output buffer will hang the chip. */ 2745 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2746 panic("%s: modexp invalid addr 0x%x", 2747 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr)); 2748 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2749 panic("%s: modexp invalid len 0x%x", 2750 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len)); 2751 #endif 2752 2753 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2754 memset(ctx, 0, sizeof(*ctx)); 2755 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p, 2756 (nbits + 7) / 8); 2757 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2758 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2759 ctx->me_E_len = htole16(ebits); 2760 ctx->me_N_len = htole16(nbits); 2761 2762 #ifdef UBSEC_DEBUG 2763 if (ubsec_debug) { 2764 ubsec_dump_mcr(mcr); 2765 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2766 } 2767 #endif 2768 2769 /* 2770 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2771 * everything else. 2772 */ 2773 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2774 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2775 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2776 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2777 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2778 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2779 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2780 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2781 2782 /* Enqueue and we're done... */ 2783 mutex_spin_enter(&sc->sc_mtx); 2784 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2785 ubsec_feed2(sc); 2786 mutex_spin_exit(&sc->sc_mtx); 2787 2788 return (0); 2789 2790 errout: 2791 if (me != NULL) { 2792 if (me->me_q.q_mcr.dma_map != NULL) 2793 ubsec_dma_free(sc, &me->me_q.q_mcr); 2794 if (me->me_q.q_ctx.dma_map != NULL) { 2795 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2796 ubsec_dma_free(sc, &me->me_q.q_ctx); 2797 } 2798 if (me->me_M.dma_map != NULL) { 2799 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2800 ubsec_dma_free(sc, &me->me_M); 2801 } 2802 if (me->me_E.dma_map != NULL) { 2803 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2804 ubsec_dma_free(sc, &me->me_E); 2805 } 2806 if (me->me_C.dma_map != NULL) { 2807 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2808 ubsec_dma_free(sc, &me->me_C); 2809 } 2810 if (me->me_epb.dma_map != NULL) 2811 ubsec_dma_free(sc, &me->me_epb); 2812 free(me, M_DEVBUF); 2813 } 2814 krp->krp_status = err; 2815 crypto_kdone(krp); 2816 return (0); 2817 } 2818 2819 static int 2820 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, 2821 int hint) 2822 { 2823 struct ubsec_q2_rsapriv *rp = NULL; 2824 struct ubsec_mcr *mcr; 2825 struct ubsec_ctx_rsapriv *ctx; 2826 int err = 0; 2827 u_int padlen, msglen; 2828 2829 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2830 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2831 if (msglen > padlen) 2832 padlen = msglen; 2833 2834 if (padlen <= 256) 2835 padlen = 256; 2836 else if (padlen <= 384) 2837 padlen = 384; 2838 else if (padlen <= 512) 2839 padlen = 512; 2840 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2841 padlen = 768; 2842 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2843 padlen = 1024; 2844 else { 2845 err = E2BIG; 2846 goto errout; 2847 } 2848 2849 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2850 err = E2BIG; 2851 goto errout; 2852 } 2853 2854 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2855 err = E2BIG; 2856 goto errout; 2857 } 2858 2859 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2860 err = E2BIG; 2861 goto errout; 2862 } 2863 2864 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO); 2865 if (rp == NULL) 2866 return (ENOMEM); 2867 rp->rpr_krp = krp; 2868 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2869 2870 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2871 &rp->rpr_q.q_mcr, 0)) { 2872 err = ENOMEM; 2873 goto errout; 2874 } 2875 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2876 2877 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2878 &rp->rpr_q.q_ctx, 0)) { 2879 err = ENOMEM; 2880 goto errout; 2881 } 2882 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2883 memset(ctx, 0, sizeof *ctx); 2884 2885 /* Copy in p */ 2886 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2887 &ctx->rpr_buf[0 * (padlen / 8)], 2888 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2889 2890 /* Copy in q */ 2891 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2892 &ctx->rpr_buf[1 * (padlen / 8)], 2893 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2894 2895 /* Copy in dp */ 2896 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2897 &ctx->rpr_buf[2 * (padlen / 8)], 2898 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2899 2900 /* Copy in dq */ 2901 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2902 &ctx->rpr_buf[3 * (padlen / 8)], 2903 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2904 2905 /* Copy in pinv */ 2906 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2907 &ctx->rpr_buf[4 * (padlen / 8)], 2908 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2909 2910 msglen = padlen * 2; 2911 2912 /* Copy in input message (aligned buffer/length). */ 2913 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2914 /* Is this likely? */ 2915 err = E2BIG; 2916 goto errout; 2917 } 2918 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2919 err = ENOMEM; 2920 goto errout; 2921 } 2922 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8); 2923 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2924 rp->rpr_msgin.dma_vaddr, 2925 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2926 2927 /* Prepare space for output message (aligned buffer/length). */ 2928 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2929 /* Is this likely? */ 2930 err = E2BIG; 2931 goto errout; 2932 } 2933 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2934 err = ENOMEM; 2935 goto errout; 2936 } 2937 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8); 2938 2939 mcr->mcr_pkts = htole16(1); 2940 mcr->mcr_flags = 0; 2941 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2942 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2943 mcr->mcr_ipktbuf.pb_next = 0; 2944 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2945 mcr->mcr_reserved = 0; 2946 mcr->mcr_pktlen = htole16(msglen); 2947 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2948 mcr->mcr_opktbuf.pb_next = 0; 2949 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2950 2951 #ifdef DIAGNOSTIC 2952 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2953 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)", 2954 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr, 2955 (u_long) rp->rpr_msgin.dma_size); 2956 } 2957 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2958 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)", 2959 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr, 2960 (u_long) rp->rpr_msgout.dma_size); 2961 } 2962 #endif 2963 2964 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2965 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2966 ctx->rpr_q_len = htole16(padlen); 2967 ctx->rpr_p_len = htole16(padlen); 2968 2969 /* 2970 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2971 * everything else. 2972 */ 2973 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 2974 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2975 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 2976 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2977 2978 /* Enqueue and we're done... */ 2979 mutex_spin_enter(&sc->sc_mtx); 2980 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2981 ubsec_feed2(sc); 2982 ubsecstats.hst_modexpcrt++; 2983 mutex_spin_exit(&sc->sc_mtx); 2984 return (0); 2985 2986 errout: 2987 if (rp != NULL) { 2988 if (rp->rpr_q.q_mcr.dma_map != NULL) 2989 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2990 if (rp->rpr_msgin.dma_map != NULL) { 2991 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 2992 ubsec_dma_free(sc, &rp->rpr_msgin); 2993 } 2994 if (rp->rpr_msgout.dma_map != NULL) { 2995 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 2996 ubsec_dma_free(sc, &rp->rpr_msgout); 2997 } 2998 free(rp, M_DEVBUF); 2999 } 3000 krp->krp_status = err; 3001 crypto_kdone(krp); 3002 return (0); 3003 } 3004 3005 #ifdef UBSEC_DEBUG 3006 static void 3007 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 3008 { 3009 printf("addr 0x%x (0x%x) next 0x%x\n", 3010 pb->pb_addr, pb->pb_len, pb->pb_next); 3011 } 3012 3013 static void 3014 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c) 3015 { 3016 printf("CTX (0x%x):\n", c->ctx_len); 3017 switch (letoh16(c->ctx_op)) { 3018 case UBS_CTXOP_RNGBYPASS: 3019 case UBS_CTXOP_RNGSHA1: 3020 break; 3021 case UBS_CTXOP_MODEXP: 3022 { 3023 struct ubsec_ctx_modexp *cx = (void *)c; 3024 int i, len; 3025 3026 printf(" Elen %u, Nlen %u\n", 3027 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 3028 len = (cx->me_N_len + 7)/8; 3029 for (i = 0; i < len; i++) 3030 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 3031 printf("\n"); 3032 break; 3033 } 3034 default: 3035 printf("unknown context: %x\n", c->ctx_op); 3036 } 3037 printf("END CTX\n"); 3038 } 3039 3040 static void 3041 ubsec_dump_mcr(struct ubsec_mcr *mcr) 3042 { 3043 volatile struct ubsec_mcr_add *ma; 3044 int i; 3045 3046 printf("MCR:\n"); 3047 printf(" pkts: %u, flags 0x%x\n", 3048 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 3049 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 3050 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 3051 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 3052 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 3053 letoh16(ma->mcr_reserved)); 3054 printf(" %d: ipkt ", i); 3055 ubsec_dump_pb(&ma->mcr_ipktbuf); 3056 printf(" %d: opkt ", i); 3057 ubsec_dump_pb(&ma->mcr_opktbuf); 3058 ma++; 3059 } 3060 printf("END MCR\n"); 3061 } 3062 #endif /* UBSEC_DEBUG */ 3063 3064 /* 3065 * Return the number of significant bits of a big number. 3066 */ 3067 static int 3068 ubsec_ksigbits(struct crparam *cr) 3069 { 3070 u_int plen = (cr->crp_nbits + 7) / 8; 3071 int i, sig = plen * 8; 3072 u_int8_t c, *p = cr->crp_p; 3073 3074 for (i = plen - 1; i >= 0; i--) { 3075 c = p[i]; 3076 if (c != 0) { 3077 while ((c & 0x80) == 0) { 3078 sig--; 3079 c <<= 1; 3080 } 3081 break; 3082 } 3083 sig -= 8; 3084 } 3085 return (sig); 3086 } 3087 3088 static void 3089 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits, 3090 u_int8_t *dst, u_int dstbits) 3091 { 3092 u_int slen, dlen; 3093 int i, si, di, n; 3094 3095 slen = (srcbits + 7) / 8; 3096 dlen = (dstbits + 7) / 8; 3097 3098 for (i = 0; i < slen; i++) 3099 dst[i] = src[i]; 3100 for (i = 0; i < dlen - slen; i++) 3101 dst[slen + i] = 0; 3102 3103 n = shiftbits / 8; 3104 if (n != 0) { 3105 si = dlen - n - 1; 3106 di = dlen - 1; 3107 while (si >= 0) 3108 dst[di--] = dst[si--]; 3109 while (di >= 0) 3110 dst[di--] = 0; 3111 } 3112 3113 n = shiftbits % 8; 3114 if (n != 0) { 3115 for (i = dlen - 1; i > 0; i--) 3116 dst[i] = (dst[i] << n) | 3117 (dst[i - 1] >> (8 - n)); 3118 dst[0] = dst[0] << n; 3119 } 3120 } 3121 3122 static void 3123 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits, 3124 u_int8_t *dst, u_int dstbits) 3125 { 3126 int slen, dlen, i, n; 3127 3128 slen = (srcbits + 7) / 8; 3129 dlen = (dstbits + 7) / 8; 3130 3131 n = shiftbits / 8; 3132 for (i = 0; i < slen; i++) 3133 dst[i] = src[i + n]; 3134 for (i = 0; i < dlen - slen; i++) 3135 dst[slen + i] = 0; 3136 3137 n = shiftbits % 8; 3138 if (n != 0) { 3139 for (i = 0; i < (dlen - 1); i++) 3140 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 3141 dst[dlen - 1] = dst[dlen - 1] >> n; 3142 } 3143 } 3144
Indexes created Mon Sep 22 05:09:51 GMT 2025