ubsec.c revision 1.39
1 /* $NetBSD: ubsec.c,v 1.39 2014/04/18 22:25:58 bad Exp $ */ 2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.127 2003/06/04 14:04:58 jason Exp $ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Effort sponsored in part by the Defense Advanced Research Projects 32 * Agency (DARPA) and Air Force Research Laboratory, Air Force 33 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 34 * 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.39 2014/04/18 22:25:58 bad Exp $"); 39 40 #undef UBSEC_DEBUG 41 42 /* 43 * uBsec 5[56]01, bcm580xx, bcm582x hardware crypto accelerator 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/endian.h> 50 #ifdef __NetBSD__ 51 #define UBSEC_NO_RNG /* hangs on attach */ 52 #define letoh16 htole16 53 #define letoh32 htole32 54 #endif 55 #include <sys/errno.h> 56 #include <sys/malloc.h> 57 #include <sys/kernel.h> 58 #include <sys/mbuf.h> 59 #include <sys/device.h> 60 #include <sys/module.h> 61 #include <sys/queue.h> 62 #include <sys/sysctl.h> 63 64 #include <opencrypto/cryptodev.h> 65 #include <opencrypto/xform.h> 66 #ifdef __OpenBSD__ 67 #include <dev/rndvar.h> 68 #include <sys/md5k.h> 69 #else 70 #include <sys/cprng.h> 71 #include <sys/md5.h> 72 #endif 73 #include <sys/sha1.h> 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pcidevs.h> 78 79 #include <dev/pci/ubsecreg.h> 80 #include <dev/pci/ubsecvar.h> 81 82 /* 83 * Prototypes and count for the pci_device structure 84 */ 85 static int ubsec_probe(device_t, cfdata_t, void *); 86 static void ubsec_attach(device_t, device_t, void *); 87 static int ubsec_detach(device_t, int); 88 static int ubsec_sysctl_init(void); 89 static void ubsec_reset_board(struct ubsec_softc *); 90 static void ubsec_init_board(struct ubsec_softc *); 91 static void ubsec_init_pciregs(struct pci_attach_args *pa); 92 static void ubsec_cleanchip(struct ubsec_softc *); 93 static void ubsec_totalreset(struct ubsec_softc *); 94 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *); 95 96 #ifdef __OpenBSD__ 97 struct cfattach ubsec_ca = { 98 sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 99 }; 100 101 struct cfdriver ubsec_cd = { 102 0, "ubsec", DV_DULL 103 }; 104 #else 105 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 106 ubsec_detach, NULL); 107 extern struct cfdriver ubsec_cd; 108 #endif 109 110 /* patchable */ 111 #ifdef UBSEC_DEBUG 112 extern int ubsec_debug; 113 int ubsec_debug=1; 114 #endif 115 116 static int ubsec_intr(void *); 117 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *); 118 static int ubsec_freesession(void*, u_int64_t); 119 static int ubsec_process(void*, struct cryptop *, int hint); 120 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 121 static void ubsec_feed(struct ubsec_softc *); 122 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 123 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 124 static void ubsec_feed2(struct ubsec_softc *); 125 static void ubsec_feed4(struct ubsec_softc *); 126 #ifndef UBSEC_NO_RNG 127 static void ubsec_rng(void *); 128 static void ubsec_rng_locked(void *); 129 static void ubsec_rng_get(size_t, void *); 130 #endif /* UBSEC_NO_RNG */ 131 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 132 struct ubsec_dma_alloc *, int); 133 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 134 static int ubsec_dmamap_aligned(bus_dmamap_t); 135 136 static int ubsec_kprocess(void*, struct cryptkop *, int); 137 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, 138 struct cryptkop *, int); 139 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, 140 struct cryptkop *, int); 141 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, 142 struct cryptkop *, int); 143 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 144 static int ubsec_ksigbits(struct crparam *); 145 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 146 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 147 148 #ifdef UBSEC_DEBUG 149 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 150 static void ubsec_dump_mcr(struct ubsec_mcr *); 151 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *); 152 #endif 153 154 #define READ_REG(sc,r) \ 155 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 156 157 #define WRITE_REG(sc,reg,val) \ 158 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 159 160 #define SWAP32(x) (x) = htole32(ntohl((x))) 161 #ifndef HTOLE32 162 #define HTOLE32(x) (x) = htole32(x) 163 #endif 164 165 struct ubsec_stats ubsecstats; 166 167 static struct sysctllog *ubsec_sysctllog; 168 169 /* 170 * ubsec_maxbatch controls the number of crypto ops to voluntarily 171 * collect into one submission to the hardware. This batching happens 172 * when ops are dispatched from the crypto subsystem with a hint that 173 * more are to follow immediately. These ops must also not be marked 174 * with a ``no delay'' flag. 175 */ 176 static int ubsec_maxbatch = 1; 177 178 /* 179 * ubsec_maxaggr controls the number of crypto ops to submit to the 180 * hardware as a unit. This aggregation reduces the number of interrupts 181 * to the host at the expense of increased latency (for all but the last 182 * operation). For network traffic setting this to one yields the highest 183 * performance but at the expense of more interrupt processing. 184 */ 185 static int ubsec_maxaggr = 1; 186 187 static const struct ubsec_product { 188 pci_vendor_id_t ubsec_vendor; 189 pci_product_id_t ubsec_product; 190 int ubsec_flags; 191 int ubsec_statmask; 192 int ubsec_maxaggr; 193 const char *ubsec_name; 194 } ubsec_products[] = { 195 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501, 196 0, 197 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 198 UBS_MIN_AGGR, 199 "Bluesteel 5501" 200 }, 201 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601, 202 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 203 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 204 UBS_MIN_AGGR, 205 "Bluesteel 5601" 206 }, 207 208 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801, 209 0, 210 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 211 UBS_MIN_AGGR, 212 "Broadcom BCM5801" 213 }, 214 215 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802, 216 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 217 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 218 UBS_MIN_AGGR, 219 "Broadcom BCM5802" 220 }, 221 222 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805, 223 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 224 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 225 UBS_MIN_AGGR, 226 "Broadcom BCM5805" 227 }, 228 229 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820, 230 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 231 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 232 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 233 UBS_MIN_AGGR, 234 "Broadcom BCM5820" 235 }, 236 237 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821, 238 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 239 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 240 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 241 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 242 UBS_MIN_AGGR, 243 "Broadcom BCM5821" 244 }, 245 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K, 246 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 247 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 248 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 249 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 250 UBS_MIN_AGGR, 251 "Sun Crypto Accelerator 1000" 252 }, 253 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821, 254 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 255 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 256 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 257 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 258 UBS_MIN_AGGR, 259 "Broadcom BCM5821 (Sun)" 260 }, 261 262 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822, 263 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 264 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 265 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 266 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 267 UBS_MIN_AGGR, 268 "Broadcom BCM5822" 269 }, 270 271 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823, 272 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 273 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 274 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 275 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 276 UBS_MIN_AGGR, 277 "Broadcom BCM5823" 278 }, 279 280 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825, 281 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 282 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 283 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 284 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 285 UBS_MIN_AGGR, 286 "Broadcom BCM5825" 287 }, 288 289 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860, 290 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 291 UBS_FLAGS_LONGCTX | 292 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 293 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY, 294 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 295 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 296 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 297 UBS_MAX_AGGR, 298 "Broadcom BCM5860" 299 }, 300 301 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861, 302 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 303 UBS_FLAGS_LONGCTX | 304 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 305 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY, 306 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 307 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 308 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 309 UBS_MAX_AGGR, 310 "Broadcom BCM5861" 311 }, 312 313 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862, 314 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 315 UBS_FLAGS_LONGCTX | 316 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 317 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY, 318 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 319 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 320 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 321 UBS_MAX_AGGR, 322 "Broadcom BCM5862" 323 }, 324 325 { 0, 0, 326 0, 327 0, 328 0, 329 NULL 330 } 331 }; 332 333 static const struct ubsec_product * 334 ubsec_lookup(const struct pci_attach_args *pa) 335 { 336 const struct ubsec_product *up; 337 338 for (up = ubsec_products; up->ubsec_name != NULL; up++) { 339 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor && 340 PCI_PRODUCT(pa->pa_id) == up->ubsec_product) 341 return (up); 342 } 343 return (NULL); 344 } 345 346 static int 347 ubsec_probe(device_t parent, cfdata_t match, void *aux) 348 { 349 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 350 351 if (ubsec_lookup(pa) != NULL) 352 return (1); 353 354 return (0); 355 } 356 357 static void 358 ubsec_attach(device_t parent, device_t self, void *aux) 359 { 360 struct ubsec_softc *sc = device_private(self); 361 struct pci_attach_args *pa = aux; 362 const struct ubsec_product *up; 363 pci_chipset_tag_t pc = pa->pa_pc; 364 pci_intr_handle_t ih; 365 const char *intrstr = NULL; 366 pcireg_t memtype; 367 struct ubsec_dma *dmap; 368 u_int32_t cmd, i; 369 char intrbuf[PCI_INTRSTR_LEN]; 370 371 sc->sc_dev = self; 372 sc->sc_pct = pc; 373 374 up = ubsec_lookup(pa); 375 if (up == NULL) { 376 printf("\n"); 377 panic("ubsec_attach: impossible"); 378 } 379 380 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1); 381 382 SIMPLEQ_INIT(&sc->sc_queue); 383 SIMPLEQ_INIT(&sc->sc_qchip); 384 SIMPLEQ_INIT(&sc->sc_queue2); 385 SIMPLEQ_INIT(&sc->sc_qchip2); 386 SIMPLEQ_INIT(&sc->sc_queue4); 387 SIMPLEQ_INIT(&sc->sc_qchip4); 388 SIMPLEQ_INIT(&sc->sc_q2free); 389 390 sc->sc_flags = up->ubsec_flags; 391 sc->sc_statmask = up->ubsec_statmask; 392 sc->sc_maxaggr = up->ubsec_maxaggr; 393 394 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 395 cmd |= PCI_COMMAND_MASTER_ENABLE; 396 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 397 398 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR); 399 if (pci_mapreg_map(pa, BS_BAR, memtype, 0, 400 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) { 401 aprint_error_dev(self, "can't find mem space"); 402 return; 403 } 404 405 sc->sc_dmat = pa->pa_dmat; 406 407 if (pci_intr_map(pa, &ih)) { 408 aprint_error_dev(self, "couldn't map interrupt\n"); 409 return; 410 } 411 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 412 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc); 413 if (sc->sc_ih == NULL) { 414 aprint_error_dev(self, "couldn't establish interrupt"); 415 if (intrstr != NULL) 416 aprint_error(" at %s", intrstr); 417 aprint_error("\n"); 418 return; 419 } 420 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 421 422 sc->sc_cid = crypto_get_driverid(0); 423 if (sc->sc_cid < 0) { 424 aprint_error_dev(self, "couldn't get crypto driver id\n"); 425 pci_intr_disestablish(pc, sc->sc_ih); 426 return; 427 } 428 429 sc->sc_rng_need = RND_POOLBITS / NBBY; 430 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 431 432 SIMPLEQ_INIT(&sc->sc_freequeue); 433 dmap = sc->sc_dmaa; 434 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 435 struct ubsec_q *q; 436 437 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 438 M_DEVBUF, M_ZERO|M_NOWAIT); 439 if (q == NULL) { 440 aprint_error_dev(self, "can't allocate queue buffers\n"); 441 break; 442 } 443 444 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 445 &dmap->d_alloc, 0)) { 446 aprint_error_dev(self, "can't allocate dma buffers\n"); 447 free(q, M_DEVBUF); 448 break; 449 } 450 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 451 452 q->q_dma = dmap; 453 sc->sc_queuea[i] = q; 454 455 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 456 } 457 458 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 459 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 460 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 461 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 462 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 463 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 464 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 465 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 466 467 /* 468 * Reset Broadcom chip 469 */ 470 ubsec_reset_board(sc); 471 472 /* 473 * Init Broadcom specific PCI settings 474 */ 475 ubsec_init_pciregs(pa); 476 477 /* 478 * Init Broadcom chip 479 */ 480 ubsec_init_board(sc); 481 482 #ifndef UBSEC_NO_RNG 483 if (sc->sc_flags & UBS_FLAGS_RNG) { 484 if (sc->sc_flags & UBS_FLAGS_RNG4) 485 sc->sc_statmask |= BS_STAT_MCR4_DONE; 486 else 487 sc->sc_statmask |= BS_STAT_MCR2_DONE; 488 489 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 490 &sc->sc_rng.rng_q.q_mcr, 0)) 491 goto skip_rng; 492 493 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 494 &sc->sc_rng.rng_q.q_ctx, 0)) { 495 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 496 goto skip_rng; 497 } 498 499 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 500 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 501 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 502 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 503 goto skip_rng; 504 } 505 506 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc); 507 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev), 508 RND_TYPE_RNG, 509 RND_FLAG_NO_ESTIMATE|RND_FLAG_HASCB); 510 if (hz >= 100) 511 sc->sc_rnghz = hz / 100; 512 else 513 sc->sc_rnghz = 1; 514 #ifdef __OpenBSD__ 515 timeout_set(&sc->sc_rngto, ubsec_rng, sc); 516 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 517 #else 518 callout_init(&sc->sc_rngto, 0); 519 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc); 520 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 521 #endif 522 skip_rng: 523 if (sc->sc_rnghz) 524 aprint_normal_dev(self, "random number generator enabled\n"); 525 else 526 aprint_error_dev(self, "WARNING: random number generator " 527 "disabled\n"); 528 } 529 #endif /* UBSEC_NO_RNG */ 530 531 if (sc->sc_flags & UBS_FLAGS_KEY) { 532 sc->sc_statmask |= BS_STAT_MCR2_DONE; 533 534 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 535 ubsec_kprocess, sc); 536 #if 0 537 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 538 ubsec_kprocess, sc); 539 #endif 540 } 541 } 542 543 static int 544 ubsec_detach(device_t self, int flags) 545 { 546 struct ubsec_softc *sc = device_private(self); 547 struct ubsec_q *q, *qtmp; 548 volatile u_int32_t ctrl; 549 550 /* disable interrupts */ 551 /* XXX wait/abort current ops? where is DMAERR enabled? */ 552 ctrl = READ_REG(sc, BS_CTRL); 553 554 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR); 555 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 556 ctrl &= ~BS_CTRL_MCR4INT; 557 558 WRITE_REG(sc, BS_CTRL, ctrl); 559 560 #ifndef UBSEC_NO_RNG 561 if (sc->sc_flags & UBS_FLAGS_RNG) { 562 callout_halt(&sc->sc_rngto, NULL); 563 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 564 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 565 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 566 rnd_detach_source(&sc->sc_rnd_source); 567 } 568 #endif /* UBSEC_NO_RNG */ 569 570 crypto_unregister_all(sc->sc_cid); 571 572 mutex_spin_enter(&sc->sc_mtx); 573 574 ubsec_totalreset(sc); /* XXX leaves the chip running */ 575 576 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) { 577 ubsec_dma_free(sc, &q->q_dma->d_alloc); 578 if (q->q_src_map != NULL) 579 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 580 if (q->q_cached_dst_map != NULL) 581 bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map); 582 free(q, M_DEVBUF); 583 } 584 585 mutex_spin_exit(&sc->sc_mtx); 586 587 if (sc->sc_ih != NULL) { 588 pci_intr_disestablish(sc->sc_pct, sc->sc_ih); 589 sc->sc_ih = NULL; 590 } 591 592 if (sc->sc_memsize != 0) { 593 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize); 594 sc->sc_memsize = 0; 595 } 596 597 return 0; 598 } 599 600 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto"); 601 602 #ifdef _MODULE 603 #include "ioconf.c" 604 #endif 605 606 static int 607 ubsec_modcmd(modcmd_t cmd, void *data) 608 { 609 int error = 0; 610 611 switch (cmd) { 612 case MODULE_CMD_INIT: 613 #ifdef _MODULE 614 error = config_init_component(cfdriver_ioconf_ubsec, 615 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 616 #endif 617 if (error == 0) 618 error = ubsec_sysctl_init(); 619 return error; 620 case MODULE_CMD_FINI: 621 if (ubsec_sysctllog != NULL) 622 sysctl_teardown(&ubsec_sysctllog); 623 #ifdef _MODULE 624 error = config_fini_component(cfdriver_ioconf_ubsec, 625 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 626 #endif 627 return error; 628 default: 629 return ENOTTY; 630 } 631 } 632 633 static int 634 ubsec_sysctl_init(void) 635 { 636 const struct sysctlnode *node = NULL; 637 638 ubsec_sysctllog = NULL; 639 640 sysctl_createv(&ubsec_sysctllog, 0, NULL, &node, 641 CTLFLAG_PERMANENT, 642 CTLTYPE_NODE, "ubsec", 643 SYSCTL_DESCR("ubsec opetions"), 644 NULL, 0, NULL, 0, 645 CTL_HW, CTL_CREATE, CTL_EOL); 646 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 647 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 648 CTLTYPE_INT, "maxbatch", 649 SYSCTL_DESCR("max ops to batch w/o interrupt"), 650 NULL, 0, &ubsec_maxbatch, 0, 651 CTL_CREATE, CTL_EOL); 652 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 653 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 654 CTLTYPE_INT, "maxaggr", 655 SYSCTL_DESCR("max ops to aggregate under one interrupt"), 656 NULL, 0, &ubsec_maxaggr, 0, 657 CTL_CREATE, CTL_EOL); 658 659 return 0; 660 } 661 662 /* 663 * UBSEC Interrupt routine 664 */ 665 static int 666 ubsec_intr(void *arg) 667 { 668 struct ubsec_softc *sc = arg; 669 volatile u_int32_t stat; 670 struct ubsec_q *q; 671 struct ubsec_dma *dmap; 672 int flags; 673 int npkts = 0, i; 674 675 mutex_spin_enter(&sc->sc_mtx); 676 stat = READ_REG(sc, BS_STAT); 677 stat &= sc->sc_statmask; 678 if (stat == 0) { 679 mutex_spin_exit(&sc->sc_mtx); 680 return (0); 681 } 682 683 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 684 685 /* 686 * Check to see if we have any packets waiting for us 687 */ 688 if ((stat & BS_STAT_MCR1_DONE)) { 689 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 690 q = SIMPLEQ_FIRST(&sc->sc_qchip); 691 dmap = q->q_dma; 692 693 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 694 break; 695 696 q = SIMPLEQ_FIRST(&sc->sc_qchip); 697 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 698 699 npkts = q->q_nstacked_mcrs; 700 sc->sc_nqchip -= 1+npkts; 701 /* 702 * search for further sc_qchip ubsec_q's that share 703 * the same MCR, and complete them too, they must be 704 * at the top. 705 */ 706 for (i = 0; i < npkts; i++) { 707 if(q->q_stacked_mcr[i]) 708 ubsec_callback(sc, q->q_stacked_mcr[i]); 709 else 710 break; 711 } 712 ubsec_callback(sc, q); 713 } 714 715 /* 716 * Don't send any more packet to chip if there has been 717 * a DMAERR. 718 */ 719 if (!(stat & BS_STAT_DMAERR)) 720 ubsec_feed(sc); 721 } 722 723 /* 724 * Check to see if we have any key setups/rng's waiting for us 725 */ 726 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 727 (stat & BS_STAT_MCR2_DONE)) { 728 struct ubsec_q2 *q2; 729 struct ubsec_mcr *mcr; 730 731 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 732 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 733 734 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 735 0, q2->q_mcr.dma_map->dm_mapsize, 736 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 737 738 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 739 740 /* A bug in new devices requires to swap this field */ 741 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 742 flags = htole16(mcr->mcr_flags); 743 else 744 flags = mcr->mcr_flags; 745 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 746 bus_dmamap_sync(sc->sc_dmat, 747 q2->q_mcr.dma_map, 0, 748 q2->q_mcr.dma_map->dm_mapsize, 749 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 750 break; 751 } 752 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 753 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next); 754 ubsec_callback2(sc, q2); 755 /* 756 * Don't send any more packet to chip if there has been 757 * a DMAERR. 758 */ 759 if (!(stat & BS_STAT_DMAERR)) 760 ubsec_feed2(sc); 761 } 762 } 763 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) { 764 struct ubsec_q2 *q2; 765 struct ubsec_mcr *mcr; 766 767 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) { 768 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4); 769 770 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 771 0, q2->q_mcr.dma_map->dm_mapsize, 772 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 773 774 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 775 776 /* A bug in new devices requires to swap this field */ 777 flags = htole16(mcr->mcr_flags); 778 779 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 780 bus_dmamap_sync(sc->sc_dmat, 781 q2->q_mcr.dma_map, 0, 782 q2->q_mcr.dma_map->dm_mapsize, 783 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 784 break; 785 } 786 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next); 787 ubsec_callback2(sc, q2); 788 /* 789 * Don't send any more packet to chip if there has been 790 * a DMAERR. 791 */ 792 if (!(stat & BS_STAT_DMAERR)) 793 ubsec_feed4(sc); 794 } 795 } 796 797 /* 798 * Check to see if we got any DMA Error 799 */ 800 if (stat & BS_STAT_DMAERR) { 801 #ifdef UBSEC_DEBUG 802 if (ubsec_debug) { 803 volatile u_int32_t a = READ_REG(sc, BS_ERR); 804 805 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev), 806 (a & BS_ERR_READ) ? "read" : "write", 807 a & BS_ERR_ADDR); 808 } 809 #endif /* UBSEC_DEBUG */ 810 ubsecstats.hst_dmaerr++; 811 ubsec_totalreset(sc); 812 ubsec_feed(sc); 813 } 814 815 if (sc->sc_needwakeup) { /* XXX check high watermark */ 816 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 817 #ifdef UBSEC_DEBUG 818 if (ubsec_debug) 819 printf("%s: wakeup crypto (%x)\n", device_xname(sc->sc_dev), 820 sc->sc_needwakeup); 821 #endif /* UBSEC_DEBUG */ 822 sc->sc_needwakeup &= ~wkeup; 823 crypto_unblock(sc->sc_cid, wkeup); 824 } 825 mutex_spin_exit(&sc->sc_mtx); 826 return (1); 827 } 828 829 /* 830 * ubsec_feed() - aggregate and post requests to chip 831 * OpenBSD comments: 832 * It is assumed that the caller set splnet() 833 */ 834 static void 835 ubsec_feed(struct ubsec_softc *sc) 836 { 837 struct ubsec_q *q, *q2; 838 int npkts, i; 839 void *v; 840 u_int32_t stat; 841 #ifdef UBSEC_DEBUG 842 static int max; 843 #endif /* UBSEC_DEBUG */ 844 845 npkts = sc->sc_nqueue; 846 if (npkts > ubsecstats.hst_maxqueue) 847 ubsecstats.hst_maxqueue = npkts; 848 if (npkts < 2) 849 goto feed1; 850 851 /* 852 * Decide how many ops to combine in a single MCR. We cannot 853 * aggregate more than UBS_MAX_AGGR because this is the number 854 * of slots defined in the data structure. Otherwise we clamp 855 * based on the tunable parameter ubsec_maxaggr. Note that 856 * aggregation can happen in two ways: either by batching ops 857 * from above or because the h/w backs up and throttles us. 858 * Aggregating ops reduces the number of interrupts to the host 859 * but also (potentially) increases the latency for processing 860 * completed ops as we only get an interrupt when all aggregated 861 * ops have completed. 862 */ 863 if (npkts > sc->sc_maxaggr) 864 npkts = sc->sc_maxaggr; 865 if (npkts > ubsec_maxaggr) 866 npkts = ubsec_maxaggr; 867 if (npkts > ubsecstats.hst_maxbatch) 868 ubsecstats.hst_maxbatch = npkts; 869 if (npkts < 2) 870 goto feed1; 871 ubsecstats.hst_totbatch += npkts-1; 872 873 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 874 if (stat & BS_STAT_DMAERR) { 875 ubsec_totalreset(sc); 876 ubsecstats.hst_dmaerr++; 877 } else { 878 ubsecstats.hst_mcr1full++; 879 } 880 return; 881 } 882 883 #ifdef UBSEC_DEBUG 884 if (ubsec_debug) 885 printf("merging %d records\n", npkts); 886 /* XXX temporary aggregation statistics reporting code */ 887 if (max < npkts) { 888 max = npkts; 889 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), max); 890 } 891 #endif /* UBSEC_DEBUG */ 892 893 q = SIMPLEQ_FIRST(&sc->sc_queue); 894 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 895 --sc->sc_nqueue; 896 897 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 898 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 899 if (q->q_dst_map != NULL) 900 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 901 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 902 903 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 904 905 for (i = 0; i < q->q_nstacked_mcrs; i++) { 906 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 907 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 908 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 909 if (q2->q_dst_map != NULL) 910 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 911 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 912 q2= SIMPLEQ_FIRST(&sc->sc_queue); 913 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next); 914 --sc->sc_nqueue; 915 916 v = ((void *)&q2->q_dma->d_dma->d_mcr); 917 v = (char*)v + (sizeof(struct ubsec_mcr) - 918 sizeof(struct ubsec_mcr_add)); 919 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, sizeof(struct ubsec_mcr_add)); 920 q->q_stacked_mcr[i] = q2; 921 } 922 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 923 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 924 sc->sc_nqchip += npkts; 925 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 926 ubsecstats.hst_maxqchip = sc->sc_nqchip; 927 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 928 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 929 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 930 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 931 offsetof(struct ubsec_dmachunk, d_mcr)); 932 return; 933 934 feed1: 935 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 936 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 937 if (stat & BS_STAT_DMAERR) { 938 ubsec_totalreset(sc); 939 ubsecstats.hst_dmaerr++; 940 } else { 941 ubsecstats.hst_mcr1full++; 942 } 943 break; 944 } 945 946 q = SIMPLEQ_FIRST(&sc->sc_queue); 947 948 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 949 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 950 if (q->q_dst_map != NULL) 951 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 952 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 953 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 954 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 955 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 956 957 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 958 offsetof(struct ubsec_dmachunk, d_mcr)); 959 #ifdef UBSEC_DEBUG 960 if (ubsec_debug) 961 printf("feed: q->chip %p %08x stat %08x\n", 962 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr, 963 stat); 964 #endif /* UBSEC_DEBUG */ 965 q = SIMPLEQ_FIRST(&sc->sc_queue); 966 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 967 --sc->sc_nqueue; 968 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 969 sc->sc_nqchip++; 970 } 971 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 972 ubsecstats.hst_maxqchip = sc->sc_nqchip; 973 } 974 975 /* 976 * Allocate a new 'session' and return an encoded session id. 'sidp' 977 * contains our registration id, and should contain an encoded session 978 * id on successful allocation. 979 */ 980 static int 981 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 982 { 983 struct cryptoini *c, *encini = NULL, *macini = NULL; 984 struct ubsec_softc *sc; 985 struct ubsec_session *ses = NULL; 986 MD5_CTX md5ctx; 987 SHA1_CTX sha1ctx; 988 int i, sesn; 989 990 sc = arg; 991 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/); 992 993 if (sidp == NULL || cri == NULL || sc == NULL) 994 return (EINVAL); 995 996 for (c = cri; c != NULL; c = c->cri_next) { 997 if (c->cri_alg == CRYPTO_MD5_HMAC_96 || 998 c->cri_alg == CRYPTO_SHA1_HMAC_96) { 999 if (macini) 1000 return (EINVAL); 1001 macini = c; 1002 } else if (c->cri_alg == CRYPTO_DES_CBC || 1003 c->cri_alg == CRYPTO_3DES_CBC) { 1004 if (encini) 1005 return (EINVAL); 1006 encini = c; 1007 } else 1008 return (EINVAL); 1009 } 1010 if (encini == NULL && macini == NULL) 1011 return (EINVAL); 1012 1013 if (sc->sc_sessions == NULL) { 1014 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 1015 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1016 if (ses == NULL) 1017 return (ENOMEM); 1018 sesn = 0; 1019 sc->sc_nsessions = 1; 1020 } else { 1021 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 1022 if (sc->sc_sessions[sesn].ses_used == 0) { 1023 ses = &sc->sc_sessions[sesn]; 1024 break; 1025 } 1026 } 1027 1028 if (ses == NULL) { 1029 sesn = sc->sc_nsessions; 1030 ses = (struct ubsec_session *)malloc((sesn + 1) * 1031 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1032 if (ses == NULL) 1033 return (ENOMEM); 1034 memcpy(ses, sc->sc_sessions, sesn * 1035 sizeof(struct ubsec_session)); 1036 memset(sc->sc_sessions, 0, sesn * 1037 sizeof(struct ubsec_session)); 1038 free(sc->sc_sessions, M_DEVBUF); 1039 sc->sc_sessions = ses; 1040 ses = &sc->sc_sessions[sesn]; 1041 sc->sc_nsessions++; 1042 } 1043 } 1044 1045 memset(ses, 0, sizeof(struct ubsec_session)); 1046 ses->ses_used = 1; 1047 if (encini) { 1048 /* get an IV, network byte order */ 1049 #ifdef __NetBSD__ 1050 cprng_fast(ses->ses_iv, sizeof(ses->ses_iv)); 1051 #else 1052 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv)); 1053 #endif 1054 1055 /* Go ahead and compute key in ubsec's byte order */ 1056 if (encini->cri_alg == CRYPTO_DES_CBC) { 1057 memcpy(&ses->ses_deskey[0], encini->cri_key, 8); 1058 memcpy(&ses->ses_deskey[2], encini->cri_key, 8); 1059 memcpy(&ses->ses_deskey[4], encini->cri_key, 8); 1060 } else 1061 memcpy(ses->ses_deskey, encini->cri_key, 24); 1062 1063 SWAP32(ses->ses_deskey[0]); 1064 SWAP32(ses->ses_deskey[1]); 1065 SWAP32(ses->ses_deskey[2]); 1066 SWAP32(ses->ses_deskey[3]); 1067 SWAP32(ses->ses_deskey[4]); 1068 SWAP32(ses->ses_deskey[5]); 1069 } 1070 1071 if (macini) { 1072 for (i = 0; i < macini->cri_klen / 8; i++) 1073 macini->cri_key[i] ^= HMAC_IPAD_VAL; 1074 1075 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1076 MD5Init(&md5ctx); 1077 MD5Update(&md5ctx, macini->cri_key, 1078 macini->cri_klen / 8); 1079 MD5Update(&md5ctx, hmac_ipad_buffer, 1080 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1081 memcpy(ses->ses_hminner, md5ctx.state, 1082 sizeof(md5ctx.state)); 1083 } else { 1084 SHA1Init(&sha1ctx); 1085 SHA1Update(&sha1ctx, macini->cri_key, 1086 macini->cri_klen / 8); 1087 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1088 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1089 memcpy(ses->ses_hminner, sha1ctx.state, 1090 sizeof(sha1ctx.state)); 1091 } 1092 1093 for (i = 0; i < macini->cri_klen / 8; i++) 1094 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1095 1096 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1097 MD5Init(&md5ctx); 1098 MD5Update(&md5ctx, macini->cri_key, 1099 macini->cri_klen / 8); 1100 MD5Update(&md5ctx, hmac_opad_buffer, 1101 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1102 memcpy(ses->ses_hmouter, md5ctx.state, 1103 sizeof(md5ctx.state)); 1104 } else { 1105 SHA1Init(&sha1ctx); 1106 SHA1Update(&sha1ctx, macini->cri_key, 1107 macini->cri_klen / 8); 1108 SHA1Update(&sha1ctx, hmac_opad_buffer, 1109 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1110 memcpy(ses->ses_hmouter, sha1ctx.state, 1111 sizeof(sha1ctx.state)); 1112 } 1113 1114 for (i = 0; i < macini->cri_klen / 8; i++) 1115 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1116 } 1117 1118 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn); 1119 return (0); 1120 } 1121 1122 /* 1123 * Deallocate a session. 1124 */ 1125 static int 1126 ubsec_freesession(void *arg, u_int64_t tid) 1127 { 1128 struct ubsec_softc *sc; 1129 int session; 1130 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1131 1132 sc = arg; 1133 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/); 1134 1135 session = UBSEC_SESSION(sid); 1136 if (session >= sc->sc_nsessions) 1137 return (EINVAL); 1138 1139 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 1140 return (0); 1141 } 1142 1143 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */ 1144 static void 1145 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1146 { 1147 struct ubsec_operand *op = arg; 1148 1149 KASSERT(nsegs <= UBS_MAX_SCATTER 1150 /*, ("Too many DMA segments returned when mapping operand")*/); 1151 #ifdef UBSEC_DEBUG 1152 if (ubsec_debug) 1153 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 1154 (u_int) mapsize, nsegs); 1155 #endif 1156 op->mapsize = mapsize; 1157 op->nsegs = nsegs; 1158 memcpy(op->segs, seg, nsegs * sizeof (seg[0])); 1159 } 1160 #endif 1161 1162 static int 1163 ubsec_process(void *arg, struct cryptop *crp, int hint) 1164 { 1165 struct ubsec_q *q = NULL; 1166 #ifdef __OpenBSD__ 1167 int card; 1168 #endif 1169 int err = 0, i, j, nicealign; 1170 struct ubsec_softc *sc; 1171 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1172 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1173 int sskip, dskip, stheend, dtheend; 1174 int16_t coffset; 1175 struct ubsec_session *ses; 1176 struct ubsec_pktctx ctx; 1177 struct ubsec_dma *dmap = NULL; 1178 1179 sc = arg; 1180 KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/); 1181 1182 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1183 ubsecstats.hst_invalid++; 1184 return (EINVAL); 1185 } 1186 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1187 ubsecstats.hst_badsession++; 1188 return (EINVAL); 1189 } 1190 1191 mutex_spin_enter(&sc->sc_mtx); 1192 1193 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1194 ubsecstats.hst_queuefull++; 1195 sc->sc_needwakeup |= CRYPTO_SYMQ; 1196 mutex_spin_exit(&sc->sc_mtx); 1197 return(ERESTART); 1198 } 1199 1200 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1201 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next); 1202 mutex_spin_exit(&sc->sc_mtx); 1203 1204 dmap = q->q_dma; /* Save dma pointer */ 1205 /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */ 1206 memset(q, 0, offsetof(struct ubsec_q, q_src_map)); 1207 memset(&ctx, 0, sizeof(ctx)); 1208 1209 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1210 q->q_dma = dmap; 1211 ses = &sc->sc_sessions[q->q_sesn]; 1212 1213 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1214 q->q_src_m = (struct mbuf *)crp->crp_buf; 1215 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1216 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1217 q->q_src_io = (struct uio *)crp->crp_buf; 1218 q->q_dst_io = (struct uio *)crp->crp_buf; 1219 } else { 1220 ubsecstats.hst_badflags++; 1221 err = EINVAL; 1222 goto errout; /* XXX we don't handle contiguous blocks! */ 1223 } 1224 1225 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr)); 1226 1227 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1228 dmap->d_dma->d_mcr.mcr_flags = 0; 1229 q->q_crp = crp; 1230 1231 crd1 = crp->crp_desc; 1232 if (crd1 == NULL) { 1233 ubsecstats.hst_nodesc++; 1234 err = EINVAL; 1235 goto errout; 1236 } 1237 crd2 = crd1->crd_next; 1238 1239 if (crd2 == NULL) { 1240 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1241 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) { 1242 maccrd = crd1; 1243 enccrd = NULL; 1244 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1245 crd1->crd_alg == CRYPTO_3DES_CBC) { 1246 maccrd = NULL; 1247 enccrd = crd1; 1248 } else { 1249 ubsecstats.hst_badalg++; 1250 err = EINVAL; 1251 goto errout; 1252 } 1253 } else { 1254 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1255 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) && 1256 (crd2->crd_alg == CRYPTO_DES_CBC || 1257 crd2->crd_alg == CRYPTO_3DES_CBC) && 1258 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1259 maccrd = crd1; 1260 enccrd = crd2; 1261 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1262 crd1->crd_alg == CRYPTO_3DES_CBC) && 1263 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 1264 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) && 1265 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1266 enccrd = crd1; 1267 maccrd = crd2; 1268 } else { 1269 /* 1270 * We cannot order the ubsec as requested 1271 */ 1272 ubsecstats.hst_badalg++; 1273 err = EINVAL; 1274 goto errout; 1275 } 1276 } 1277 1278 if (enccrd) { 1279 encoffset = enccrd->crd_skip; 1280 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1281 1282 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1283 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1284 1285 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1286 memcpy(ctx.pc_iv, enccrd->crd_iv, 8); 1287 else { 1288 ctx.pc_iv[0] = ses->ses_iv[0]; 1289 ctx.pc_iv[1] = ses->ses_iv[1]; 1290 } 1291 1292 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1293 if (crp->crp_flags & CRYPTO_F_IMBUF) 1294 m_copyback(q->q_src_m, 1295 enccrd->crd_inject, 1296 8, (void *)ctx.pc_iv); 1297 else if (crp->crp_flags & CRYPTO_F_IOV) 1298 cuio_copyback(q->q_src_io, 1299 enccrd->crd_inject, 1300 8, (void *)ctx.pc_iv); 1301 } 1302 } else { 1303 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1304 1305 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1306 memcpy(ctx.pc_iv, enccrd->crd_iv, 8); 1307 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1308 m_copydata(q->q_src_m, enccrd->crd_inject, 1309 8, (void *)ctx.pc_iv); 1310 else if (crp->crp_flags & CRYPTO_F_IOV) 1311 cuio_copydata(q->q_src_io, 1312 enccrd->crd_inject, 8, 1313 (void *)ctx.pc_iv); 1314 } 1315 1316 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1317 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1318 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1319 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1320 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1321 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1322 SWAP32(ctx.pc_iv[0]); 1323 SWAP32(ctx.pc_iv[1]); 1324 } 1325 1326 if (maccrd) { 1327 macoffset = maccrd->crd_skip; 1328 1329 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96) 1330 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1331 else 1332 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1333 1334 for (i = 0; i < 5; i++) { 1335 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1336 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1337 1338 HTOLE32(ctx.pc_hminner[i]); 1339 HTOLE32(ctx.pc_hmouter[i]); 1340 } 1341 } 1342 1343 if (enccrd && maccrd) { 1344 /* 1345 * ubsec cannot handle packets where the end of encryption 1346 * and authentication are not the same, or where the 1347 * encrypted part begins before the authenticated part. 1348 */ 1349 if ((encoffset + enccrd->crd_len) != 1350 (macoffset + maccrd->crd_len)) { 1351 ubsecstats.hst_lenmismatch++; 1352 err = EINVAL; 1353 goto errout; 1354 } 1355 if (enccrd->crd_skip < maccrd->crd_skip) { 1356 ubsecstats.hst_skipmismatch++; 1357 err = EINVAL; 1358 goto errout; 1359 } 1360 sskip = maccrd->crd_skip; 1361 cpskip = dskip = enccrd->crd_skip; 1362 stheend = maccrd->crd_len; 1363 dtheend = enccrd->crd_len; 1364 coffset = enccrd->crd_skip - maccrd->crd_skip; 1365 cpoffset = cpskip + dtheend; 1366 #ifdef UBSEC_DEBUG 1367 if (ubsec_debug) { 1368 printf("mac: skip %d, len %d, inject %d\n", 1369 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1370 printf("enc: skip %d, len %d, inject %d\n", 1371 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1372 printf("src: skip %d, len %d\n", sskip, stheend); 1373 printf("dst: skip %d, len %d\n", dskip, dtheend); 1374 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1375 coffset, stheend, cpskip, cpoffset); 1376 } 1377 #endif 1378 } else { 1379 cpskip = dskip = sskip = macoffset + encoffset; 1380 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1381 cpoffset = cpskip + dtheend; 1382 coffset = 0; 1383 } 1384 ctx.pc_offset = htole16(coffset >> 2); 1385 1386 if (q->q_src_map == NULL) { 1387 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */ 1388 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER, 1389 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) { 1390 err = ENOMEM; 1391 goto errout; 1392 } 1393 } 1394 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1395 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1396 q->q_src_m, BUS_DMA_NOWAIT) != 0) { 1397 ubsecstats.hst_noload++; 1398 err = ENOMEM; 1399 goto errout; 1400 } 1401 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1402 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1403 q->q_src_io, BUS_DMA_NOWAIT) != 0) { 1404 ubsecstats.hst_noload++; 1405 err = ENOMEM; 1406 goto errout; 1407 } 1408 } 1409 nicealign = ubsec_dmamap_aligned(q->q_src_map); 1410 1411 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1412 1413 #ifdef UBSEC_DEBUG 1414 if (ubsec_debug) 1415 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1416 #endif 1417 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) { 1418 struct ubsec_pktbuf *pb; 1419 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len; 1420 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr; 1421 1422 if (sskip >= packl) { 1423 sskip -= packl; 1424 continue; 1425 } 1426 1427 packl -= sskip; 1428 packp += sskip; 1429 sskip = 0; 1430 1431 if (packl > 0xfffc) { 1432 err = EIO; 1433 goto errout; 1434 } 1435 1436 if (j == 0) 1437 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1438 else 1439 pb = &dmap->d_dma->d_sbuf[j - 1]; 1440 1441 pb->pb_addr = htole32(packp); 1442 1443 if (stheend) { 1444 if (packl > stheend) { 1445 pb->pb_len = htole32(stheend); 1446 stheend = 0; 1447 } else { 1448 pb->pb_len = htole32(packl); 1449 stheend -= packl; 1450 } 1451 } else 1452 pb->pb_len = htole32(packl); 1453 1454 if ((i + 1) == q->q_src_map->dm_nsegs) 1455 pb->pb_next = 0; 1456 else 1457 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1458 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1459 j++; 1460 } 1461 1462 if (enccrd == NULL && maccrd != NULL) { 1463 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1464 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1465 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1466 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1467 #ifdef UBSEC_DEBUG 1468 if (ubsec_debug) 1469 printf("opkt: %x %x %x\n", 1470 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1471 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1472 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1473 1474 #endif 1475 } else { 1476 if (crp->crp_flags & CRYPTO_F_IOV) { 1477 if (!nicealign) { 1478 ubsecstats.hst_iovmisaligned++; 1479 err = EINVAL; 1480 goto errout; 1481 } 1482 if (q->q_dst_map == NULL) { 1483 if (q->q_cached_dst_map == NULL) { 1484 /* XXX: ``what the heck's that'' 0xfff0? */ 1485 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1486 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1487 &q->q_cached_dst_map) != 0) { 1488 ubsecstats.hst_nomap++; 1489 err = ENOMEM; 1490 goto errout; 1491 } 1492 } 1493 q->q_dst_map = q->q_cached_dst_map; 1494 } 1495 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1496 q->q_dst_io, BUS_DMA_NOWAIT) != 0) { 1497 ubsecstats.hst_noload++; 1498 err = ENOMEM; 1499 goto errout; 1500 } 1501 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1502 if (nicealign) { 1503 q->q_dst_m = q->q_src_m; 1504 q->q_dst_map = q->q_src_map; 1505 } else { 1506 int totlen, len; 1507 struct mbuf *m, *top, **mp; 1508 1509 ubsecstats.hst_unaligned++; 1510 totlen = q->q_src_map->dm_mapsize; 1511 if (q->q_src_m->m_flags & M_PKTHDR) { 1512 len = MHLEN; 1513 MGETHDR(m, M_DONTWAIT, MT_DATA); 1514 /*XXX FIXME: m_dup_pkthdr */ 1515 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) { 1516 m_free(m); 1517 m = NULL; 1518 } 1519 } else { 1520 len = MLEN; 1521 MGET(m, M_DONTWAIT, MT_DATA); 1522 } 1523 if (m == NULL) { 1524 ubsecstats.hst_nombuf++; 1525 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1526 goto errout; 1527 } 1528 if (len == MHLEN) 1529 /*XXX was M_DUP_PKTHDR*/ 1530 M_COPY_PKTHDR(m, q->q_src_m); 1531 if (totlen >= MINCLSIZE) { 1532 MCLGET(m, M_DONTWAIT); 1533 if ((m->m_flags & M_EXT) == 0) { 1534 m_free(m); 1535 ubsecstats.hst_nomcl++; 1536 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1537 goto errout; 1538 } 1539 len = MCLBYTES; 1540 } 1541 m->m_len = len; 1542 top = NULL; 1543 mp = ⊤ 1544 1545 while (totlen > 0) { 1546 if (top) { 1547 MGET(m, M_DONTWAIT, MT_DATA); 1548 if (m == NULL) { 1549 m_freem(top); 1550 ubsecstats.hst_nombuf++; 1551 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1552 goto errout; 1553 } 1554 len = MLEN; 1555 } 1556 if (top && totlen >= MINCLSIZE) { 1557 MCLGET(m, M_DONTWAIT); 1558 if ((m->m_flags & M_EXT) == 0) { 1559 *mp = m; 1560 m_freem(top); 1561 ubsecstats.hst_nomcl++; 1562 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1563 goto errout; 1564 } 1565 len = MCLBYTES; 1566 } 1567 m->m_len = len = min(totlen, len); 1568 totlen -= len; 1569 *mp = m; 1570 mp = &m->m_next; 1571 } 1572 q->q_dst_m = top; 1573 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1574 cpskip, cpoffset); 1575 if (q->q_dst_map == NULL) { 1576 if (q->q_cached_dst_map == NULL) { 1577 /* XXX again, what the heck is that 0xfff0? */ 1578 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1579 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1580 &q->q_cached_dst_map) != 0) { 1581 ubsecstats.hst_nomap++; 1582 err = ENOMEM; 1583 goto errout; 1584 } 1585 } 1586 q->q_dst_map = q->q_cached_dst_map; 1587 } 1588 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1589 q->q_dst_map, q->q_dst_m, 1590 BUS_DMA_NOWAIT) != 0) { 1591 ubsecstats.hst_noload++; 1592 err = ENOMEM; 1593 goto errout; 1594 } 1595 } 1596 } else { 1597 ubsecstats.hst_badflags++; 1598 err = EINVAL; 1599 goto errout; 1600 } 1601 1602 #ifdef UBSEC_DEBUG 1603 if (ubsec_debug) 1604 printf("dst skip: %d\n", dskip); 1605 #endif 1606 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) { 1607 struct ubsec_pktbuf *pb; 1608 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len; 1609 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr; 1610 1611 if (dskip >= packl) { 1612 dskip -= packl; 1613 continue; 1614 } 1615 1616 packl -= dskip; 1617 packp += dskip; 1618 dskip = 0; 1619 1620 if (packl > 0xfffc) { 1621 err = EIO; 1622 goto errout; 1623 } 1624 1625 if (j == 0) 1626 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1627 else 1628 pb = &dmap->d_dma->d_dbuf[j - 1]; 1629 1630 pb->pb_addr = htole32(packp); 1631 1632 if (dtheend) { 1633 if (packl > dtheend) { 1634 pb->pb_len = htole32(dtheend); 1635 dtheend = 0; 1636 } else { 1637 pb->pb_len = htole32(packl); 1638 dtheend -= packl; 1639 } 1640 } else 1641 pb->pb_len = htole32(packl); 1642 1643 if ((i + 1) == q->q_dst_map->dm_nsegs) { 1644 if (maccrd) 1645 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1646 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1647 else 1648 pb->pb_next = 0; 1649 } else 1650 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1651 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1652 j++; 1653 } 1654 } 1655 1656 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1657 offsetof(struct ubsec_dmachunk, d_ctx)); 1658 1659 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1660 struct ubsec_pktctx_long *ctxl; 1661 1662 ctxl = (struct ubsec_pktctx_long *)((char *)dmap->d_alloc.dma_vaddr + 1663 offsetof(struct ubsec_dmachunk, d_ctx)); 1664 1665 /* transform small context into long context */ 1666 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1667 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1668 ctxl->pc_flags = ctx.pc_flags; 1669 ctxl->pc_offset = ctx.pc_offset; 1670 for (i = 0; i < 6; i++) 1671 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1672 for (i = 0; i < 5; i++) 1673 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1674 for (i = 0; i < 5; i++) 1675 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1676 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1677 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1678 } else 1679 memcpy((char *)dmap->d_alloc.dma_vaddr + 1680 offsetof(struct ubsec_dmachunk, d_ctx), &ctx, 1681 sizeof(struct ubsec_pktctx)); 1682 1683 mutex_spin_enter(&sc->sc_mtx); 1684 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1685 sc->sc_nqueue++; 1686 ubsecstats.hst_ipackets++; 1687 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize; 1688 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1689 ubsec_feed(sc); 1690 mutex_spin_exit(&sc->sc_mtx); 1691 return (0); 1692 1693 errout: 1694 if (q != NULL) { 1695 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1696 m_freem(q->q_dst_m); 1697 1698 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1699 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1700 } 1701 if (q->q_src_map != NULL) { 1702 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1703 } 1704 1705 mutex_spin_enter(&sc->sc_mtx); 1706 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1707 mutex_spin_exit(&sc->sc_mtx); 1708 } 1709 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */ 1710 if (err == EINVAL) 1711 ubsecstats.hst_invalid++; 1712 else 1713 ubsecstats.hst_nomem++; 1714 #endif 1715 if (err != ERESTART) { 1716 crp->crp_etype = err; 1717 crypto_done(crp); 1718 } else { 1719 sc->sc_needwakeup |= CRYPTO_SYMQ; 1720 } 1721 return (err); 1722 } 1723 1724 static void 1725 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1726 { 1727 struct cryptop *crp = (struct cryptop *)q->q_crp; 1728 struct cryptodesc *crd; 1729 struct ubsec_dma *dmap = q->q_dma; 1730 1731 ubsecstats.hst_opackets++; 1732 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1733 1734 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0, 1735 dmap->d_alloc.dma_map->dm_mapsize, 1736 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1737 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1738 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1739 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1740 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1741 } 1742 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 1743 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1744 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1745 1746 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1747 m_freem(q->q_src_m); 1748 crp->crp_buf = (void *)q->q_dst_m; 1749 } 1750 1751 /* copy out IV for future use */ 1752 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1753 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1754 if (crd->crd_alg != CRYPTO_DES_CBC && 1755 crd->crd_alg != CRYPTO_3DES_CBC) 1756 continue; 1757 if (crp->crp_flags & CRYPTO_F_IMBUF) 1758 m_copydata((struct mbuf *)crp->crp_buf, 1759 crd->crd_skip + crd->crd_len - 8, 8, 1760 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1761 else if (crp->crp_flags & CRYPTO_F_IOV) { 1762 cuio_copydata((struct uio *)crp->crp_buf, 1763 crd->crd_skip + crd->crd_len - 8, 8, 1764 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1765 } 1766 break; 1767 } 1768 } 1769 1770 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1771 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 && 1772 crd->crd_alg != CRYPTO_SHA1_HMAC_96) 1773 continue; 1774 if (crp->crp_flags & CRYPTO_F_IMBUF) 1775 m_copyback((struct mbuf *)crp->crp_buf, 1776 crd->crd_inject, 12, 1777 (void *)dmap->d_dma->d_macbuf); 1778 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1779 bcopy((void *)dmap->d_dma->d_macbuf, 1780 crp->crp_mac, 12); 1781 break; 1782 } 1783 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1784 crypto_done(crp); 1785 } 1786 1787 static void 1788 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1789 { 1790 int i, j, dlen, slen; 1791 char *dptr, *sptr; 1792 1793 j = 0; 1794 sptr = srcm->m_data; 1795 slen = srcm->m_len; 1796 dptr = dstm->m_data; 1797 dlen = dstm->m_len; 1798 1799 while (1) { 1800 for (i = 0; i < min(slen, dlen); i++) { 1801 if (j < hoffset || j >= toffset) 1802 *dptr++ = *sptr++; 1803 slen--; 1804 dlen--; 1805 j++; 1806 } 1807 if (slen == 0) { 1808 srcm = srcm->m_next; 1809 if (srcm == NULL) 1810 return; 1811 sptr = srcm->m_data; 1812 slen = srcm->m_len; 1813 } 1814 if (dlen == 0) { 1815 dstm = dstm->m_next; 1816 if (dstm == NULL) 1817 return; 1818 dptr = dstm->m_data; 1819 dlen = dstm->m_len; 1820 } 1821 } 1822 } 1823 1824 /* 1825 * feed the key generator, must be called at splnet() or higher. 1826 */ 1827 static void 1828 ubsec_feed2(struct ubsec_softc *sc) 1829 { 1830 struct ubsec_q2 *q; 1831 1832 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1833 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1834 break; 1835 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1836 1837 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1838 q->q_mcr.dma_map->dm_mapsize, 1839 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1840 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1841 q->q_ctx.dma_map->dm_mapsize, 1842 BUS_DMASYNC_PREWRITE); 1843 1844 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1845 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1846 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next); 1847 --sc->sc_nqueue2; 1848 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1849 } 1850 } 1851 1852 /* 1853 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices) 1854 */ 1855 void 1856 ubsec_feed4(struct ubsec_softc *sc) 1857 { 1858 struct ubsec_q2 *q; 1859 1860 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) { 1861 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL) 1862 break; 1863 q = SIMPLEQ_FIRST(&sc->sc_queue4); 1864 1865 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1866 q->q_mcr.dma_map->dm_mapsize, 1867 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1868 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1869 q->q_ctx.dma_map->dm_mapsize, 1870 BUS_DMASYNC_PREWRITE); 1871 1872 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr); 1873 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next); 1874 --sc->sc_nqueue4; 1875 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next); 1876 } 1877 } 1878 1879 /* 1880 * Callback for handling random numbers 1881 */ 1882 static void 1883 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1884 { 1885 struct cryptkop *krp; 1886 struct ubsec_ctx_keyop *ctx; 1887 1888 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1889 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1890 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1891 1892 switch (q->q_type) { 1893 #ifndef UBSEC_NO_RNG 1894 case UBS_CTXOP_RNGSHA1: 1895 case UBS_CTXOP_RNGBYPASS: { 1896 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1897 u_int32_t *p; 1898 int i; 1899 1900 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 1901 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1902 p = (u_int32_t *)rng->rng_buf.dma_vaddr; 1903 #ifndef __NetBSD__ 1904 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++) 1905 add_true_randomness(letoh32(*p)); 1906 #else 1907 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t); 1908 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY); 1909 sc->sc_rng_need -= i; 1910 #endif 1911 rng->rng_used = 0; 1912 #ifdef __OpenBSD__ 1913 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 1914 #else 1915 if (sc->sc_rng_need > 0) { 1916 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 1917 } 1918 #endif 1919 break; 1920 } 1921 #endif 1922 case UBS_CTXOP_MODEXP: { 1923 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1924 u_int rlen, clen; 1925 1926 krp = me->me_krp; 1927 rlen = (me->me_modbits + 7) / 8; 1928 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1929 1930 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 1931 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1932 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 1933 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1934 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 1935 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1936 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 1937 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1938 1939 if (clen < rlen) 1940 krp->krp_status = E2BIG; 1941 else { 1942 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1943 memset(krp->krp_param[krp->krp_iparams].crp_p, 0, 1944 (krp->krp_param[krp->krp_iparams].crp_nbits 1945 + 7) / 8); 1946 bcopy(me->me_C.dma_vaddr, 1947 krp->krp_param[krp->krp_iparams].crp_p, 1948 (me->me_modbits + 7) / 8); 1949 } else 1950 ubsec_kshift_l(me->me_shiftbits, 1951 me->me_C.dma_vaddr, me->me_normbits, 1952 krp->krp_param[krp->krp_iparams].crp_p, 1953 krp->krp_param[krp->krp_iparams].crp_nbits); 1954 } 1955 1956 crypto_kdone(krp); 1957 1958 /* bzero all potentially sensitive data */ 1959 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 1960 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 1961 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 1962 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 1963 1964 /* Can't free here, so put us on the free list. */ 1965 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1966 break; 1967 } 1968 case UBS_CTXOP_RSAPRIV: { 1969 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1970 u_int len; 1971 1972 krp = rp->rpr_krp; 1973 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0, 1974 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1975 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0, 1976 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1977 1978 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1979 bcopy(rp->rpr_msgout.dma_vaddr, 1980 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1981 1982 crypto_kdone(krp); 1983 1984 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 1985 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 1986 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size); 1987 1988 /* Can't free here, so put us on the free list. */ 1989 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1990 break; 1991 } 1992 default: 1993 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev), 1994 letoh16(ctx->ctx_op)); 1995 break; 1996 } 1997 } 1998 1999 #ifndef UBSEC_NO_RNG 2000 2001 static void 2002 ubsec_rng_get(size_t bytes, void *vsc) 2003 { 2004 struct ubsec_softc *sc = vsc; 2005 2006 mutex_spin_enter(&sc->sc_mtx); 2007 sc->sc_rng_need = bytes; 2008 ubsec_rng_locked(sc); 2009 mutex_spin_exit(&sc->sc_mtx); 2010 2011 } 2012 2013 static void 2014 ubsec_rng(void *vsc) 2015 { 2016 struct ubsec_softc *sc = vsc; 2017 mutex_spin_enter(&sc->sc_mtx); 2018 ubsec_rng_locked(sc); 2019 mutex_spin_exit(&sc->sc_mtx); 2020 } 2021 2022 static void 2023 ubsec_rng_locked(void *vsc) 2024 { 2025 struct ubsec_softc *sc = vsc; 2026 struct ubsec_q2_rng *rng = &sc->sc_rng; 2027 struct ubsec_mcr *mcr; 2028 struct ubsec_ctx_rngbypass *ctx; 2029 int *nqueue; 2030 2031 /* Caller is responsible to lock and release sc_mtx. */ 2032 KASSERT(mutex_owned(&sc->sc_mtx)); 2033 2034 if (rng->rng_used) { 2035 return; 2036 } 2037 2038 if (sc->sc_rng_need < 1) { 2039 callout_stop(&sc->sc_rngto); 2040 return; 2041 } 2042 2043 if (sc->sc_flags & UBS_FLAGS_RNG4) 2044 nqueue = &sc->sc_nqueue4; 2045 else 2046 nqueue = &sc->sc_nqueue2; 2047 2048 (*nqueue)++; 2049 if (*nqueue >= UBS_MAX_NQUEUE) 2050 goto out; 2051 2052 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 2053 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 2054 2055 mcr->mcr_pkts = htole16(1); 2056 mcr->mcr_flags = 0; 2057 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 2058 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 2059 mcr->mcr_ipktbuf.pb_len = 0; 2060 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 2061 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 2062 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 2063 UBS_PKTBUF_LEN); 2064 mcr->mcr_opktbuf.pb_next = 0; 2065 2066 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 2067 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1); 2068 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1; 2069 2070 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2071 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2072 2073 if (sc->sc_flags & UBS_FLAGS_RNG4) { 2074 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next); 2075 ubsec_feed4(sc); 2076 } else { 2077 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 2078 ubsec_feed2(sc); 2079 } 2080 rng->rng_used = 1; 2081 ubsecstats.hst_rng++; 2082 2083 return; 2084 2085 out: 2086 /* 2087 * Something weird happened, generate our own call back. 2088 */ 2089 (*nqueue)--; 2090 #ifdef __OpenBSD__ 2091 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2092 #else 2093 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2094 #endif 2095 } 2096 #endif /* UBSEC_NO_RNG */ 2097 2098 static int 2099 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size, 2100 struct ubsec_dma_alloc *dma,int mapflags) 2101 { 2102 int r; 2103 2104 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, 2105 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0) 2106 goto fail_0; 2107 2108 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, 2109 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0) 2110 goto fail_1; 2111 2112 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2113 BUS_DMA_NOWAIT, &dma->dma_map)) != 0) 2114 goto fail_2; 2115 2116 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 2117 size, NULL, BUS_DMA_NOWAIT)) != 0) 2118 goto fail_3; 2119 2120 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2121 dma->dma_size = size; 2122 return (0); 2123 2124 fail_3: 2125 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2126 fail_2: 2127 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size); 2128 fail_1: 2129 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2130 fail_0: 2131 dma->dma_map = NULL; 2132 return (r); 2133 } 2134 2135 static void 2136 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 2137 { 2138 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 2139 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size); 2140 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2141 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2142 } 2143 2144 /* 2145 * Resets the board. Values in the regesters are left as is 2146 * from the reset (i.e. initial values are assigned elsewhere). 2147 */ 2148 static void 2149 ubsec_reset_board(struct ubsec_softc *sc) 2150 { 2151 volatile u_int32_t ctrl; 2152 2153 ctrl = READ_REG(sc, BS_CTRL); 2154 ctrl |= BS_CTRL_RESET; 2155 WRITE_REG(sc, BS_CTRL, ctrl); 2156 2157 /* 2158 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2159 */ 2160 DELAY(10); 2161 2162 /* Enable RNG and interrupts on newer devices */ 2163 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2164 #ifndef UBSEC_NO_RNG 2165 WRITE_REG(sc, BS_CFG, BS_CFG_RNG); 2166 #endif 2167 WRITE_REG(sc, BS_INT, BS_INT_DMAINT); 2168 } 2169 } 2170 2171 /* 2172 * Init Broadcom registers 2173 */ 2174 static void 2175 ubsec_init_board(struct ubsec_softc *sc) 2176 { 2177 u_int32_t ctrl; 2178 2179 ctrl = READ_REG(sc, BS_CTRL); 2180 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2181 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2182 2183 /* 2184 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)). 2185 * anyone got hw docs? 2186 */ 2187 if (sc->sc_flags & UBS_FLAGS_KEY) 2188 ctrl |= BS_CTRL_MCR2INT; 2189 else 2190 ctrl &= ~BS_CTRL_MCR2INT; 2191 2192 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2193 ctrl &= ~BS_CTRL_SWNORM; 2194 2195 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2196 ctrl |= BS_CTRL_BSIZE240; 2197 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */ 2198 2199 if (sc->sc_flags & UBS_FLAGS_RNG4) 2200 ctrl |= BS_CTRL_MCR4INT; 2201 else 2202 ctrl &= ~BS_CTRL_MCR4INT; 2203 } 2204 2205 WRITE_REG(sc, BS_CTRL, ctrl); 2206 } 2207 2208 /* 2209 * Init Broadcom PCI registers 2210 */ 2211 static void 2212 ubsec_init_pciregs(struct pci_attach_args *pa) 2213 { 2214 pci_chipset_tag_t pc = pa->pa_pc; 2215 u_int32_t misc; 2216 2217 /* 2218 * This will set the cache line size to 1, this will 2219 * force the BCM58xx chip just to do burst read/writes. 2220 * Cache line read/writes are to slow 2221 */ 2222 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG); 2223 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT)) 2224 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT); 2225 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc); 2226 } 2227 2228 /* 2229 * Clean up after a chip crash. 2230 * It is assumed that the caller in splnet() 2231 */ 2232 static void 2233 ubsec_cleanchip(struct ubsec_softc *sc) 2234 { 2235 struct ubsec_q *q; 2236 2237 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2238 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2239 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 2240 ubsec_free_q(sc, q); 2241 } 2242 sc->sc_nqchip = 0; 2243 } 2244 2245 /* 2246 * free a ubsec_q 2247 * It is assumed that the caller is within splnet() 2248 */ 2249 static int 2250 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2251 { 2252 struct ubsec_q *q2; 2253 struct cryptop *crp; 2254 int npkts; 2255 int i; 2256 2257 npkts = q->q_nstacked_mcrs; 2258 2259 for (i = 0; i < npkts; i++) { 2260 if(q->q_stacked_mcr[i]) { 2261 q2 = q->q_stacked_mcr[i]; 2262 2263 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2264 m_freem(q2->q_dst_m); 2265 2266 crp = (struct cryptop *)q2->q_crp; 2267 2268 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2269 2270 crp->crp_etype = EFAULT; 2271 crypto_done(crp); 2272 } else { 2273 break; 2274 } 2275 } 2276 2277 /* 2278 * Free header MCR 2279 */ 2280 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2281 m_freem(q->q_dst_m); 2282 2283 crp = (struct cryptop *)q->q_crp; 2284 2285 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2286 2287 crp->crp_etype = EFAULT; 2288 crypto_done(crp); 2289 return(0); 2290 } 2291 2292 /* 2293 * Routine to reset the chip and clean up. 2294 * It is assumed that the caller is in splnet() 2295 */ 2296 static void 2297 ubsec_totalreset(struct ubsec_softc *sc) 2298 { 2299 ubsec_reset_board(sc); 2300 ubsec_init_board(sc); 2301 ubsec_cleanchip(sc); 2302 } 2303 2304 static int 2305 ubsec_dmamap_aligned(bus_dmamap_t map) 2306 { 2307 int i; 2308 2309 for (i = 0; i < map->dm_nsegs; i++) { 2310 if (map->dm_segs[i].ds_addr & 3) 2311 return (0); 2312 if ((i != (map->dm_nsegs - 1)) && 2313 (map->dm_segs[i].ds_len & 3)) 2314 return (0); 2315 } 2316 return (1); 2317 } 2318 2319 #ifdef __OpenBSD__ 2320 struct ubsec_softc * 2321 ubsec_kfind(struct cryptkop *krp) 2322 { 2323 struct ubsec_softc *sc; 2324 int i; 2325 2326 for (i = 0; i < ubsec_cd.cd_ndevs; i++) { 2327 sc = ubsec_cd.cd_devs[i]; 2328 if (sc == NULL) 2329 continue; 2330 if (sc->sc_cid == krp->krp_hid) 2331 return (sc); 2332 } 2333 return (NULL); 2334 } 2335 #endif 2336 2337 static void 2338 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2339 { 2340 switch (q->q_type) { 2341 case UBS_CTXOP_MODEXP: { 2342 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2343 2344 ubsec_dma_free(sc, &me->me_q.q_mcr); 2345 ubsec_dma_free(sc, &me->me_q.q_ctx); 2346 ubsec_dma_free(sc, &me->me_M); 2347 ubsec_dma_free(sc, &me->me_E); 2348 ubsec_dma_free(sc, &me->me_C); 2349 ubsec_dma_free(sc, &me->me_epb); 2350 free(me, M_DEVBUF); 2351 break; 2352 } 2353 case UBS_CTXOP_RSAPRIV: { 2354 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2355 2356 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2357 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2358 ubsec_dma_free(sc, &rp->rpr_msgin); 2359 ubsec_dma_free(sc, &rp->rpr_msgout); 2360 free(rp, M_DEVBUF); 2361 break; 2362 } 2363 default: 2364 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev), 2365 q->q_type); 2366 break; 2367 } 2368 } 2369 2370 static int 2371 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2372 { 2373 struct ubsec_softc *sc; 2374 int r; 2375 2376 if (krp == NULL || krp->krp_callback == NULL) 2377 return (EINVAL); 2378 #ifdef __OpenBSD__ 2379 if ((sc = ubsec_kfind(krp)) == NULL) 2380 return (EINVAL); 2381 #else 2382 sc = arg; 2383 KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/); 2384 #endif 2385 2386 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2387 struct ubsec_q2 *q; 2388 2389 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2390 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next); 2391 ubsec_kfree(sc, q); 2392 } 2393 2394 switch (krp->krp_op) { 2395 case CRK_MOD_EXP: 2396 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2397 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2398 else 2399 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2400 break; 2401 case CRK_MOD_EXP_CRT: 2402 r = ubsec_kprocess_rsapriv(sc, krp, hint); 2403 break; 2404 default: 2405 printf("%s: kprocess: invalid op 0x%x\n", 2406 device_xname(sc->sc_dev), krp->krp_op); 2407 krp->krp_status = EOPNOTSUPP; 2408 crypto_kdone(krp); 2409 r = 0; 2410 } 2411 return (r); 2412 } 2413 2414 /* 2415 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2416 */ 2417 static int 2418 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, 2419 int hint) 2420 { 2421 struct ubsec_q2_modexp *me; 2422 struct ubsec_mcr *mcr; 2423 struct ubsec_ctx_modexp *ctx; 2424 struct ubsec_pktbuf *epb; 2425 int err = 0; 2426 u_int nbits, normbits, mbits, shiftbits, ebits; 2427 2428 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2429 if (me == NULL) { 2430 err = ENOMEM; 2431 goto errout; 2432 } 2433 memset(me, 0, sizeof *me); 2434 me->me_krp = krp; 2435 me->me_q.q_type = UBS_CTXOP_MODEXP; 2436 2437 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2438 if (nbits <= 512) 2439 normbits = 512; 2440 else if (nbits <= 768) 2441 normbits = 768; 2442 else if (nbits <= 1024) 2443 normbits = 1024; 2444 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2445 normbits = 1536; 2446 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2447 normbits = 2048; 2448 else { 2449 err = E2BIG; 2450 goto errout; 2451 } 2452 2453 shiftbits = normbits - nbits; 2454 2455 me->me_modbits = nbits; 2456 me->me_shiftbits = shiftbits; 2457 me->me_normbits = normbits; 2458 2459 /* Sanity check: result bits must be >= true modulus bits. */ 2460 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2461 err = ERANGE; 2462 goto errout; 2463 } 2464 2465 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2466 &me->me_q.q_mcr, 0)) { 2467 err = ENOMEM; 2468 goto errout; 2469 } 2470 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2471 2472 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2473 &me->me_q.q_ctx, 0)) { 2474 err = ENOMEM; 2475 goto errout; 2476 } 2477 2478 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2479 if (mbits > nbits) { 2480 err = E2BIG; 2481 goto errout; 2482 } 2483 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2484 err = ENOMEM; 2485 goto errout; 2486 } 2487 ubsec_kshift_r(shiftbits, 2488 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2489 me->me_M.dma_vaddr, normbits); 2490 2491 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2492 err = ENOMEM; 2493 goto errout; 2494 } 2495 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2496 2497 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2498 if (ebits > nbits) { 2499 err = E2BIG; 2500 goto errout; 2501 } 2502 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2503 err = ENOMEM; 2504 goto errout; 2505 } 2506 ubsec_kshift_r(shiftbits, 2507 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2508 me->me_E.dma_vaddr, normbits); 2509 2510 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2511 &me->me_epb, 0)) { 2512 err = ENOMEM; 2513 goto errout; 2514 } 2515 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2516 epb->pb_addr = htole32(me->me_E.dma_paddr); 2517 epb->pb_next = 0; 2518 epb->pb_len = htole32(normbits / 8); 2519 2520 #ifdef UBSEC_DEBUG 2521 if (ubsec_debug) { 2522 printf("Epb "); 2523 ubsec_dump_pb(epb); 2524 } 2525 #endif 2526 2527 mcr->mcr_pkts = htole16(1); 2528 mcr->mcr_flags = 0; 2529 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2530 mcr->mcr_reserved = 0; 2531 mcr->mcr_pktlen = 0; 2532 2533 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2534 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2535 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2536 2537 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2538 mcr->mcr_opktbuf.pb_next = 0; 2539 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2540 2541 #ifdef DIAGNOSTIC 2542 /* Misaligned output buffer will hang the chip. */ 2543 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2544 panic("%s: modexp invalid addr 0x%x", 2545 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr)); 2546 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2547 panic("%s: modexp invalid len 0x%x", 2548 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len)); 2549 #endif 2550 2551 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2552 memset(ctx, 0, sizeof(*ctx)); 2553 ubsec_kshift_r(shiftbits, 2554 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2555 ctx->me_N, normbits); 2556 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2557 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2558 ctx->me_E_len = htole16(nbits); 2559 ctx->me_N_len = htole16(nbits); 2560 2561 #ifdef UBSEC_DEBUG 2562 if (ubsec_debug) { 2563 ubsec_dump_mcr(mcr); 2564 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2565 } 2566 #endif 2567 2568 /* 2569 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2570 * everything else. 2571 */ 2572 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2573 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2574 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2575 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2576 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2577 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2578 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2579 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2580 2581 /* Enqueue and we're done... */ 2582 mutex_spin_enter(&sc->sc_mtx); 2583 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2584 ubsec_feed2(sc); 2585 ubsecstats.hst_modexp++; 2586 mutex_spin_exit(&sc->sc_mtx); 2587 2588 return (0); 2589 2590 errout: 2591 if (me != NULL) { 2592 if (me->me_q.q_mcr.dma_map != NULL) 2593 ubsec_dma_free(sc, &me->me_q.q_mcr); 2594 if (me->me_q.q_ctx.dma_map != NULL) { 2595 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2596 ubsec_dma_free(sc, &me->me_q.q_ctx); 2597 } 2598 if (me->me_M.dma_map != NULL) { 2599 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2600 ubsec_dma_free(sc, &me->me_M); 2601 } 2602 if (me->me_E.dma_map != NULL) { 2603 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2604 ubsec_dma_free(sc, &me->me_E); 2605 } 2606 if (me->me_C.dma_map != NULL) { 2607 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2608 ubsec_dma_free(sc, &me->me_C); 2609 } 2610 if (me->me_epb.dma_map != NULL) 2611 ubsec_dma_free(sc, &me->me_epb); 2612 free(me, M_DEVBUF); 2613 } 2614 krp->krp_status = err; 2615 crypto_kdone(krp); 2616 return (0); 2617 } 2618 2619 /* 2620 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2621 */ 2622 static int 2623 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, 2624 int hint) 2625 { 2626 struct ubsec_q2_modexp *me; 2627 struct ubsec_mcr *mcr; 2628 struct ubsec_ctx_modexp *ctx; 2629 struct ubsec_pktbuf *epb; 2630 int err = 0; 2631 u_int nbits, normbits, mbits, shiftbits, ebits; 2632 2633 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2634 if (me == NULL) { 2635 err = ENOMEM; 2636 goto errout; 2637 } 2638 memset(me, 0, sizeof *me); 2639 me->me_krp = krp; 2640 me->me_q.q_type = UBS_CTXOP_MODEXP; 2641 2642 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2643 if (nbits <= 512) 2644 normbits = 512; 2645 else if (nbits <= 768) 2646 normbits = 768; 2647 else if (nbits <= 1024) 2648 normbits = 1024; 2649 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2650 normbits = 1536; 2651 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2652 normbits = 2048; 2653 else { 2654 err = E2BIG; 2655 goto errout; 2656 } 2657 2658 shiftbits = normbits - nbits; 2659 2660 /* XXX ??? */ 2661 me->me_modbits = nbits; 2662 me->me_shiftbits = shiftbits; 2663 me->me_normbits = normbits; 2664 2665 /* Sanity check: result bits must be >= true modulus bits. */ 2666 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2667 err = ERANGE; 2668 goto errout; 2669 } 2670 2671 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2672 &me->me_q.q_mcr, 0)) { 2673 err = ENOMEM; 2674 goto errout; 2675 } 2676 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2677 2678 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2679 &me->me_q.q_ctx, 0)) { 2680 err = ENOMEM; 2681 goto errout; 2682 } 2683 2684 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2685 if (mbits > nbits) { 2686 err = E2BIG; 2687 goto errout; 2688 } 2689 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2690 err = ENOMEM; 2691 goto errout; 2692 } 2693 memset(me->me_M.dma_vaddr, 0, normbits / 8); 2694 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2695 me->me_M.dma_vaddr, (mbits + 7) / 8); 2696 2697 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2698 err = ENOMEM; 2699 goto errout; 2700 } 2701 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2702 2703 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2704 if (ebits > nbits) { 2705 err = E2BIG; 2706 goto errout; 2707 } 2708 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2709 err = ENOMEM; 2710 goto errout; 2711 } 2712 memset(me->me_E.dma_vaddr, 0, normbits / 8); 2713 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2714 me->me_E.dma_vaddr, (ebits + 7) / 8); 2715 2716 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2717 &me->me_epb, 0)) { 2718 err = ENOMEM; 2719 goto errout; 2720 } 2721 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2722 epb->pb_addr = htole32(me->me_E.dma_paddr); 2723 epb->pb_next = 0; 2724 epb->pb_len = htole32((ebits + 7) / 8); 2725 2726 #ifdef UBSEC_DEBUG 2727 if (ubsec_debug) { 2728 printf("Epb "); 2729 ubsec_dump_pb(epb); 2730 } 2731 #endif 2732 2733 mcr->mcr_pkts = htole16(1); 2734 mcr->mcr_flags = 0; 2735 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2736 mcr->mcr_reserved = 0; 2737 mcr->mcr_pktlen = 0; 2738 2739 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2740 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2741 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2742 2743 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2744 mcr->mcr_opktbuf.pb_next = 0; 2745 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2746 2747 #ifdef DIAGNOSTIC 2748 /* Misaligned output buffer will hang the chip. */ 2749 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2750 panic("%s: modexp invalid addr 0x%x", 2751 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr)); 2752 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2753 panic("%s: modexp invalid len 0x%x", 2754 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len)); 2755 #endif 2756 2757 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2758 memset(ctx, 0, sizeof(*ctx)); 2759 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p, 2760 (nbits + 7) / 8); 2761 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2762 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2763 ctx->me_E_len = htole16(ebits); 2764 ctx->me_N_len = htole16(nbits); 2765 2766 #ifdef UBSEC_DEBUG 2767 if (ubsec_debug) { 2768 ubsec_dump_mcr(mcr); 2769 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2770 } 2771 #endif 2772 2773 /* 2774 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2775 * everything else. 2776 */ 2777 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2778 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2779 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2780 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2781 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2782 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2783 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2784 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2785 2786 /* Enqueue and we're done... */ 2787 mutex_spin_enter(&sc->sc_mtx); 2788 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2789 ubsec_feed2(sc); 2790 mutex_spin_exit(&sc->sc_mtx); 2791 2792 return (0); 2793 2794 errout: 2795 if (me != NULL) { 2796 if (me->me_q.q_mcr.dma_map != NULL) 2797 ubsec_dma_free(sc, &me->me_q.q_mcr); 2798 if (me->me_q.q_ctx.dma_map != NULL) { 2799 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2800 ubsec_dma_free(sc, &me->me_q.q_ctx); 2801 } 2802 if (me->me_M.dma_map != NULL) { 2803 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2804 ubsec_dma_free(sc, &me->me_M); 2805 } 2806 if (me->me_E.dma_map != NULL) { 2807 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2808 ubsec_dma_free(sc, &me->me_E); 2809 } 2810 if (me->me_C.dma_map != NULL) { 2811 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2812 ubsec_dma_free(sc, &me->me_C); 2813 } 2814 if (me->me_epb.dma_map != NULL) 2815 ubsec_dma_free(sc, &me->me_epb); 2816 free(me, M_DEVBUF); 2817 } 2818 krp->krp_status = err; 2819 crypto_kdone(krp); 2820 return (0); 2821 } 2822 2823 static int 2824 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, 2825 int hint) 2826 { 2827 struct ubsec_q2_rsapriv *rp = NULL; 2828 struct ubsec_mcr *mcr; 2829 struct ubsec_ctx_rsapriv *ctx; 2830 int err = 0; 2831 u_int padlen, msglen; 2832 2833 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2834 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2835 if (msglen > padlen) 2836 padlen = msglen; 2837 2838 if (padlen <= 256) 2839 padlen = 256; 2840 else if (padlen <= 384) 2841 padlen = 384; 2842 else if (padlen <= 512) 2843 padlen = 512; 2844 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2845 padlen = 768; 2846 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2847 padlen = 1024; 2848 else { 2849 err = E2BIG; 2850 goto errout; 2851 } 2852 2853 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2854 err = E2BIG; 2855 goto errout; 2856 } 2857 2858 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2859 err = E2BIG; 2860 goto errout; 2861 } 2862 2863 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2864 err = E2BIG; 2865 goto errout; 2866 } 2867 2868 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO); 2869 if (rp == NULL) 2870 return (ENOMEM); 2871 rp->rpr_krp = krp; 2872 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2873 2874 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2875 &rp->rpr_q.q_mcr, 0)) { 2876 err = ENOMEM; 2877 goto errout; 2878 } 2879 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2880 2881 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2882 &rp->rpr_q.q_ctx, 0)) { 2883 err = ENOMEM; 2884 goto errout; 2885 } 2886 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2887 memset(ctx, 0, sizeof *ctx); 2888 2889 /* Copy in p */ 2890 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2891 &ctx->rpr_buf[0 * (padlen / 8)], 2892 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2893 2894 /* Copy in q */ 2895 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2896 &ctx->rpr_buf[1 * (padlen / 8)], 2897 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2898 2899 /* Copy in dp */ 2900 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2901 &ctx->rpr_buf[2 * (padlen / 8)], 2902 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2903 2904 /* Copy in dq */ 2905 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2906 &ctx->rpr_buf[3 * (padlen / 8)], 2907 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2908 2909 /* Copy in pinv */ 2910 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2911 &ctx->rpr_buf[4 * (padlen / 8)], 2912 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2913 2914 msglen = padlen * 2; 2915 2916 /* Copy in input message (aligned buffer/length). */ 2917 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2918 /* Is this likely? */ 2919 err = E2BIG; 2920 goto errout; 2921 } 2922 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2923 err = ENOMEM; 2924 goto errout; 2925 } 2926 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8); 2927 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2928 rp->rpr_msgin.dma_vaddr, 2929 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2930 2931 /* Prepare space for output message (aligned buffer/length). */ 2932 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2933 /* Is this likely? */ 2934 err = E2BIG; 2935 goto errout; 2936 } 2937 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2938 err = ENOMEM; 2939 goto errout; 2940 } 2941 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8); 2942 2943 mcr->mcr_pkts = htole16(1); 2944 mcr->mcr_flags = 0; 2945 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2946 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2947 mcr->mcr_ipktbuf.pb_next = 0; 2948 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2949 mcr->mcr_reserved = 0; 2950 mcr->mcr_pktlen = htole16(msglen); 2951 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2952 mcr->mcr_opktbuf.pb_next = 0; 2953 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2954 2955 #ifdef DIAGNOSTIC 2956 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2957 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)", 2958 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr, 2959 (u_long) rp->rpr_msgin.dma_size); 2960 } 2961 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2962 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)", 2963 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr, 2964 (u_long) rp->rpr_msgout.dma_size); 2965 } 2966 #endif 2967 2968 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2969 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2970 ctx->rpr_q_len = htole16(padlen); 2971 ctx->rpr_p_len = htole16(padlen); 2972 2973 /* 2974 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2975 * everything else. 2976 */ 2977 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 2978 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2979 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 2980 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2981 2982 /* Enqueue and we're done... */ 2983 mutex_spin_enter(&sc->sc_mtx); 2984 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2985 ubsec_feed2(sc); 2986 ubsecstats.hst_modexpcrt++; 2987 mutex_spin_exit(&sc->sc_mtx); 2988 return (0); 2989 2990 errout: 2991 if (rp != NULL) { 2992 if (rp->rpr_q.q_mcr.dma_map != NULL) 2993 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2994 if (rp->rpr_msgin.dma_map != NULL) { 2995 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 2996 ubsec_dma_free(sc, &rp->rpr_msgin); 2997 } 2998 if (rp->rpr_msgout.dma_map != NULL) { 2999 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 3000 ubsec_dma_free(sc, &rp->rpr_msgout); 3001 } 3002 free(rp, M_DEVBUF); 3003 } 3004 krp->krp_status = err; 3005 crypto_kdone(krp); 3006 return (0); 3007 } 3008 3009 #ifdef UBSEC_DEBUG 3010 static void 3011 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 3012 { 3013 printf("addr 0x%x (0x%x) next 0x%x\n", 3014 pb->pb_addr, pb->pb_len, pb->pb_next); 3015 } 3016 3017 static void 3018 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c) 3019 { 3020 printf("CTX (0x%x):\n", c->ctx_len); 3021 switch (letoh16(c->ctx_op)) { 3022 case UBS_CTXOP_RNGBYPASS: 3023 case UBS_CTXOP_RNGSHA1: 3024 break; 3025 case UBS_CTXOP_MODEXP: 3026 { 3027 struct ubsec_ctx_modexp *cx = (void *)c; 3028 int i, len; 3029 3030 printf(" Elen %u, Nlen %u\n", 3031 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 3032 len = (cx->me_N_len + 7)/8; 3033 for (i = 0; i < len; i++) 3034 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 3035 printf("\n"); 3036 break; 3037 } 3038 default: 3039 printf("unknown context: %x\n", c->ctx_op); 3040 } 3041 printf("END CTX\n"); 3042 } 3043 3044 static void 3045 ubsec_dump_mcr(struct ubsec_mcr *mcr) 3046 { 3047 volatile struct ubsec_mcr_add *ma; 3048 int i; 3049 3050 printf("MCR:\n"); 3051 printf(" pkts: %u, flags 0x%x\n", 3052 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 3053 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 3054 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 3055 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 3056 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 3057 letoh16(ma->mcr_reserved)); 3058 printf(" %d: ipkt ", i); 3059 ubsec_dump_pb(&ma->mcr_ipktbuf); 3060 printf(" %d: opkt ", i); 3061 ubsec_dump_pb(&ma->mcr_opktbuf); 3062 ma++; 3063 } 3064 printf("END MCR\n"); 3065 } 3066 #endif /* UBSEC_DEBUG */ 3067 3068 /* 3069 * Return the number of significant bits of a big number. 3070 */ 3071 static int 3072 ubsec_ksigbits(struct crparam *cr) 3073 { 3074 u_int plen = (cr->crp_nbits + 7) / 8; 3075 int i, sig = plen * 8; 3076 u_int8_t c, *p = cr->crp_p; 3077 3078 for (i = plen - 1; i >= 0; i--) { 3079 c = p[i]; 3080 if (c != 0) { 3081 while ((c & 0x80) == 0) { 3082 sig--; 3083 c <<= 1; 3084 } 3085 break; 3086 } 3087 sig -= 8; 3088 } 3089 return (sig); 3090 } 3091 3092 static void 3093 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits, 3094 u_int8_t *dst, u_int dstbits) 3095 { 3096 u_int slen, dlen; 3097 int i, si, di, n; 3098 3099 slen = (srcbits + 7) / 8; 3100 dlen = (dstbits + 7) / 8; 3101 3102 for (i = 0; i < slen; i++) 3103 dst[i] = src[i]; 3104 for (i = 0; i < dlen - slen; i++) 3105 dst[slen + i] = 0; 3106 3107 n = shiftbits / 8; 3108 if (n != 0) { 3109 si = dlen - n - 1; 3110 di = dlen - 1; 3111 while (si >= 0) 3112 dst[di--] = dst[si--]; 3113 while (di >= 0) 3114 dst[di--] = 0; 3115 } 3116 3117 n = shiftbits % 8; 3118 if (n != 0) { 3119 for (i = dlen - 1; i > 0; i--) 3120 dst[i] = (dst[i] << n) | 3121 (dst[i - 1] >> (8 - n)); 3122 dst[0] = dst[0] << n; 3123 } 3124 } 3125 3126 static void 3127 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits, 3128 u_int8_t *dst, u_int dstbits) 3129 { 3130 int slen, dlen, i, n; 3131 3132 slen = (srcbits + 7) / 8; 3133 dlen = (dstbits + 7) / 8; 3134 3135 n = shiftbits / 8; 3136 for (i = 0; i < slen; i++) 3137 dst[i] = src[i + n]; 3138 for (i = 0; i < dlen - slen; i++) 3139 dst[slen + i] = 0; 3140 3141 n = shiftbits % 8; 3142 if (n != 0) { 3143 for (i = 0; i < (dlen - 1); i++) 3144 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 3145 dst[dlen - 1] = dst[dlen - 1] >> n; 3146 } 3147 } 3148
Indexes created Wed Oct 01 19:09:53 GMT 2025