ubsec.c revision 1.41.4.1
1 /* $NetBSD: ubsec.c,v 1.41.4.1 2015/06/06 14:40:12 skrll Exp $ */ 2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Effort sponsored in part by the Defense Advanced Research Projects 32 * Agency (DARPA) and Air Force Research Laboratory, Air Force 33 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 34 * 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.41.4.1 2015/06/06 14:40:12 skrll Exp $"); 39 40 #undef UBSEC_DEBUG 41 42 /* 43 * uBsec 5[56]01, 58xx hardware crypto accelerator 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/endian.h> 50 #ifdef __NetBSD__ 51 #define UBSEC_NO_RNG /* hangs on attach */ 52 #define letoh16 htole16 53 #define letoh32 htole32 54 #endif 55 #include <sys/errno.h> 56 #include <sys/malloc.h> 57 #include <sys/kernel.h> 58 #include <sys/mbuf.h> 59 #include <sys/device.h> 60 #include <sys/module.h> 61 #include <sys/queue.h> 62 #include <sys/sysctl.h> 63 64 #include <opencrypto/cryptodev.h> 65 #include <opencrypto/xform.h> 66 #ifdef __OpenBSD__ 67 #include <dev/rndvar.h> 68 #include <sys/md5k.h> 69 #else 70 #include <sys/cprng.h> 71 #include <sys/md5.h> 72 #include <sys/rndpool.h> 73 #include <sys/rndsource.h> 74 #endif 75 #include <sys/sha1.h> 76 77 #include <dev/pci/pcireg.h> 78 #include <dev/pci/pcivar.h> 79 #include <dev/pci/pcidevs.h> 80 81 #include <dev/pci/ubsecreg.h> 82 #include <dev/pci/ubsecvar.h> 83 84 /* 85 * Prototypes and count for the pci_device structure 86 */ 87 static int ubsec_probe(device_t, cfdata_t, void *); 88 static void ubsec_attach(device_t, device_t, void *); 89 static int ubsec_detach(device_t, int); 90 static int ubsec_sysctl_init(void); 91 static void ubsec_reset_board(struct ubsec_softc *); 92 static void ubsec_init_board(struct ubsec_softc *); 93 static void ubsec_init_pciregs(struct pci_attach_args *pa); 94 static void ubsec_cleanchip(struct ubsec_softc *); 95 static void ubsec_totalreset(struct ubsec_softc *); 96 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *); 97 98 #ifdef __OpenBSD__ 99 struct cfattach ubsec_ca = { 100 sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 101 }; 102 103 struct cfdriver ubsec_cd = { 104 0, "ubsec", DV_DULL 105 }; 106 #else 107 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 108 ubsec_detach, NULL); 109 extern struct cfdriver ubsec_cd; 110 #endif 111 112 /* patchable */ 113 #ifdef UBSEC_DEBUG 114 extern int ubsec_debug; 115 int ubsec_debug=1; 116 #endif 117 118 static int ubsec_intr(void *); 119 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *); 120 static int ubsec_freesession(void*, u_int64_t); 121 static int ubsec_process(void*, struct cryptop *, int hint); 122 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 123 static void ubsec_feed(struct ubsec_softc *); 124 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 125 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 126 static void ubsec_feed2(struct ubsec_softc *); 127 static void ubsec_feed4(struct ubsec_softc *); 128 #ifndef UBSEC_NO_RNG 129 static void ubsec_rng(void *); 130 static void ubsec_rng_locked(void *); 131 static void ubsec_rng_get(size_t, void *); 132 #endif /* UBSEC_NO_RNG */ 133 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 134 struct ubsec_dma_alloc *, int); 135 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 136 static int ubsec_dmamap_aligned(bus_dmamap_t); 137 138 static int ubsec_kprocess(void*, struct cryptkop *, int); 139 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, 140 struct cryptkop *, int); 141 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, 142 struct cryptkop *, int); 143 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, 144 struct cryptkop *, int); 145 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 146 static int ubsec_ksigbits(struct crparam *); 147 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 148 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 149 150 #ifdef UBSEC_DEBUG 151 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 152 static void ubsec_dump_mcr(struct ubsec_mcr *); 153 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *); 154 #endif 155 156 #define READ_REG(sc,r) \ 157 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 158 159 #define WRITE_REG(sc,reg,val) \ 160 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 161 162 #define SWAP32(x) (x) = htole32(ntohl((x))) 163 #ifndef HTOLE32 164 #define HTOLE32(x) (x) = htole32(x) 165 #endif 166 167 struct ubsec_stats ubsecstats; 168 169 static struct sysctllog *ubsec_sysctllog; 170 171 /* 172 * ubsec_maxbatch controls the number of crypto ops to voluntarily 173 * collect into one submission to the hardware. This batching happens 174 * when ops are dispatched from the crypto subsystem with a hint that 175 * more are to follow immediately. These ops must also not be marked 176 * with a ``no delay'' flag. 177 */ 178 static int ubsec_maxbatch = 1; 179 180 /* 181 * ubsec_maxaggr controls the number of crypto ops to submit to the 182 * hardware as a unit. This aggregation reduces the number of interrupts 183 * to the host at the expense of increased latency (for all but the last 184 * operation). For network traffic setting this to one yields the highest 185 * performance but at the expense of more interrupt processing. 186 */ 187 static int ubsec_maxaggr = 1; 188 189 static const struct ubsec_product { 190 pci_vendor_id_t ubsec_vendor; 191 pci_product_id_t ubsec_product; 192 int ubsec_flags; 193 int ubsec_statmask; 194 int ubsec_maxaggr; 195 const char *ubsec_name; 196 } ubsec_products[] = { 197 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501, 198 0, 199 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 200 UBS_MIN_AGGR, 201 "Bluesteel 5501" 202 }, 203 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601, 204 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 205 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 206 UBS_MIN_AGGR, 207 "Bluesteel 5601" 208 }, 209 210 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801, 211 0, 212 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 213 UBS_MIN_AGGR, 214 "Broadcom BCM5801" 215 }, 216 217 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802, 218 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 219 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 220 UBS_MIN_AGGR, 221 "Broadcom BCM5802" 222 }, 223 224 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805, 225 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 226 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 227 UBS_MIN_AGGR, 228 "Broadcom BCM5805" 229 }, 230 231 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820, 232 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 233 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 234 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 235 UBS_MIN_AGGR, 236 "Broadcom BCM5820" 237 }, 238 239 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821, 240 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 241 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 242 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 243 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 244 UBS_MIN_AGGR, 245 "Broadcom BCM5821" 246 }, 247 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K, 248 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 249 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 250 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 251 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 252 UBS_MIN_AGGR, 253 "Sun Crypto Accelerator 1000" 254 }, 255 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821, 256 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 257 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 258 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 259 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 260 UBS_MIN_AGGR, 261 "Broadcom BCM5821 (Sun)" 262 }, 263 264 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822, 265 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 266 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 267 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 268 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 269 UBS_MIN_AGGR, 270 "Broadcom BCM5822" 271 }, 272 273 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823, 274 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 275 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 276 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 277 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 278 UBS_MIN_AGGR, 279 "Broadcom BCM5823" 280 }, 281 282 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825, 283 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 284 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 285 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 286 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 287 UBS_MIN_AGGR, 288 "Broadcom BCM5825" 289 }, 290 291 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860, 292 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 293 UBS_FLAGS_LONGCTX | 294 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 295 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 296 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 297 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 298 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 299 UBS_MAX_AGGR, 300 "Broadcom BCM5860" 301 }, 302 303 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861, 304 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 305 UBS_FLAGS_LONGCTX | 306 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 307 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 308 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 309 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 310 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 311 UBS_MAX_AGGR, 312 "Broadcom BCM5861" 313 }, 314 315 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862, 316 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 317 UBS_FLAGS_LONGCTX | 318 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 319 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 320 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 321 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 322 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 323 UBS_MAX_AGGR, 324 "Broadcom BCM5862" 325 }, 326 327 { 0, 0, 328 0, 329 0, 330 0, 331 NULL 332 } 333 }; 334 335 static const struct ubsec_product * 336 ubsec_lookup(const struct pci_attach_args *pa) 337 { 338 const struct ubsec_product *up; 339 340 for (up = ubsec_products; up->ubsec_name != NULL; up++) { 341 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor && 342 PCI_PRODUCT(pa->pa_id) == up->ubsec_product) 343 return (up); 344 } 345 return (NULL); 346 } 347 348 static int 349 ubsec_probe(device_t parent, cfdata_t match, void *aux) 350 { 351 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 352 353 if (ubsec_lookup(pa) != NULL) 354 return (1); 355 356 return (0); 357 } 358 359 static void 360 ubsec_attach(device_t parent, device_t self, void *aux) 361 { 362 struct ubsec_softc *sc = device_private(self); 363 struct pci_attach_args *pa = aux; 364 const struct ubsec_product *up; 365 pci_chipset_tag_t pc = pa->pa_pc; 366 pci_intr_handle_t ih; 367 const char *intrstr = NULL; 368 pcireg_t memtype; 369 struct ubsec_dma *dmap; 370 u_int32_t cmd, i; 371 char intrbuf[PCI_INTRSTR_LEN]; 372 373 sc->sc_dev = self; 374 sc->sc_pct = pc; 375 376 up = ubsec_lookup(pa); 377 if (up == NULL) { 378 printf("\n"); 379 panic("ubsec_attach: impossible"); 380 } 381 382 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1); 383 384 SIMPLEQ_INIT(&sc->sc_queue); 385 SIMPLEQ_INIT(&sc->sc_qchip); 386 SIMPLEQ_INIT(&sc->sc_queue2); 387 SIMPLEQ_INIT(&sc->sc_qchip2); 388 SIMPLEQ_INIT(&sc->sc_queue4); 389 SIMPLEQ_INIT(&sc->sc_qchip4); 390 SIMPLEQ_INIT(&sc->sc_q2free); 391 392 sc->sc_flags = up->ubsec_flags; 393 sc->sc_statmask = up->ubsec_statmask; 394 sc->sc_maxaggr = up->ubsec_maxaggr; 395 396 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 397 cmd |= PCI_COMMAND_MASTER_ENABLE; 398 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 399 400 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR); 401 if (pci_mapreg_map(pa, BS_BAR, memtype, 0, 402 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) { 403 aprint_error_dev(self, "can't find mem space"); 404 return; 405 } 406 407 sc->sc_dmat = pa->pa_dmat; 408 409 if (pci_intr_map(pa, &ih)) { 410 aprint_error_dev(self, "couldn't map interrupt\n"); 411 return; 412 } 413 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 414 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, ubsec_intr, sc); 415 if (sc->sc_ih == NULL) { 416 aprint_error_dev(self, "couldn't establish interrupt"); 417 if (intrstr != NULL) 418 aprint_error(" at %s", intrstr); 419 aprint_error("\n"); 420 return; 421 } 422 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 423 424 sc->sc_cid = crypto_get_driverid(0); 425 if (sc->sc_cid < 0) { 426 aprint_error_dev(self, "couldn't get crypto driver id\n"); 427 pci_intr_disestablish(pc, sc->sc_ih); 428 return; 429 } 430 431 sc->sc_rng_need = RND_POOLBITS / NBBY; 432 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 433 434 SIMPLEQ_INIT(&sc->sc_freequeue); 435 dmap = sc->sc_dmaa; 436 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 437 struct ubsec_q *q; 438 439 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 440 M_DEVBUF, M_ZERO|M_NOWAIT); 441 if (q == NULL) { 442 aprint_error_dev(self, "can't allocate queue buffers\n"); 443 break; 444 } 445 446 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 447 &dmap->d_alloc, 0)) { 448 aprint_error_dev(self, "can't allocate dma buffers\n"); 449 free(q, M_DEVBUF); 450 break; 451 } 452 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 453 454 q->q_dma = dmap; 455 sc->sc_queuea[i] = q; 456 457 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 458 } 459 460 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 461 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 462 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 463 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 464 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 465 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 466 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 467 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 468 if (sc->sc_flags & UBS_FLAGS_AES) { 469 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 470 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 471 } 472 473 /* 474 * Reset Broadcom chip 475 */ 476 ubsec_reset_board(sc); 477 478 /* 479 * Init Broadcom specific PCI settings 480 */ 481 ubsec_init_pciregs(pa); 482 483 /* 484 * Init Broadcom chip 485 */ 486 ubsec_init_board(sc); 487 488 #ifndef UBSEC_NO_RNG 489 if (sc->sc_flags & UBS_FLAGS_RNG) { 490 if (sc->sc_flags & UBS_FLAGS_RNG4) 491 sc->sc_statmask |= BS_STAT_MCR4_DONE; 492 else 493 sc->sc_statmask |= BS_STAT_MCR2_DONE; 494 495 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 496 &sc->sc_rng.rng_q.q_mcr, 0)) 497 goto skip_rng; 498 499 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 500 &sc->sc_rng.rng_q.q_ctx, 0)) { 501 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 502 goto skip_rng; 503 } 504 505 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 506 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 507 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 508 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 509 goto skip_rng; 510 } 511 512 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc); 513 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev), 514 RND_TYPE_RNG, 515 RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB); 516 if (hz >= 100) 517 sc->sc_rnghz = hz / 100; 518 else 519 sc->sc_rnghz = 1; 520 #ifdef __OpenBSD__ 521 timeout_set(&sc->sc_rngto, ubsec_rng, sc); 522 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 523 #else 524 callout_init(&sc->sc_rngto, 0); 525 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc); 526 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 527 #endif 528 skip_rng: 529 if (sc->sc_rnghz) 530 aprint_normal_dev(self, "random number generator enabled\n"); 531 else 532 aprint_error_dev(self, "WARNING: random number generator " 533 "disabled\n"); 534 } 535 #endif /* UBSEC_NO_RNG */ 536 537 if (sc->sc_flags & UBS_FLAGS_KEY) { 538 sc->sc_statmask |= BS_STAT_MCR2_DONE; 539 540 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 541 ubsec_kprocess, sc); 542 #if 0 543 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 544 ubsec_kprocess, sc); 545 #endif 546 } 547 } 548 549 static int 550 ubsec_detach(device_t self, int flags) 551 { 552 struct ubsec_softc *sc = device_private(self); 553 struct ubsec_q *q, *qtmp; 554 volatile u_int32_t ctrl; 555 556 /* disable interrupts */ 557 /* XXX wait/abort current ops? where is DMAERR enabled? */ 558 ctrl = READ_REG(sc, BS_CTRL); 559 560 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR); 561 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 562 ctrl &= ~BS_CTRL_MCR4INT; 563 564 WRITE_REG(sc, BS_CTRL, ctrl); 565 566 #ifndef UBSEC_NO_RNG 567 if (sc->sc_flags & UBS_FLAGS_RNG) { 568 callout_halt(&sc->sc_rngto, NULL); 569 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 570 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 571 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 572 rnd_detach_source(&sc->sc_rnd_source); 573 } 574 #endif /* UBSEC_NO_RNG */ 575 576 crypto_unregister_all(sc->sc_cid); 577 578 mutex_spin_enter(&sc->sc_mtx); 579 580 ubsec_totalreset(sc); /* XXX leaves the chip running */ 581 582 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) { 583 ubsec_dma_free(sc, &q->q_dma->d_alloc); 584 if (q->q_src_map != NULL) 585 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 586 if (q->q_cached_dst_map != NULL) 587 bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map); 588 free(q, M_DEVBUF); 589 } 590 591 mutex_spin_exit(&sc->sc_mtx); 592 593 if (sc->sc_ih != NULL) { 594 pci_intr_disestablish(sc->sc_pct, sc->sc_ih); 595 sc->sc_ih = NULL; 596 } 597 598 if (sc->sc_memsize != 0) { 599 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize); 600 sc->sc_memsize = 0; 601 } 602 603 return 0; 604 } 605 606 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto"); 607 608 #ifdef _MODULE 609 #include "ioconf.c" 610 #endif 611 612 static int 613 ubsec_modcmd(modcmd_t cmd, void *data) 614 { 615 int error = 0; 616 617 switch (cmd) { 618 case MODULE_CMD_INIT: 619 #ifdef _MODULE 620 error = config_init_component(cfdriver_ioconf_ubsec, 621 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 622 #endif 623 if (error == 0) 624 error = ubsec_sysctl_init(); 625 return error; 626 case MODULE_CMD_FINI: 627 if (ubsec_sysctllog != NULL) 628 sysctl_teardown(&ubsec_sysctllog); 629 #ifdef _MODULE 630 error = config_fini_component(cfdriver_ioconf_ubsec, 631 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 632 #endif 633 return error; 634 default: 635 return ENOTTY; 636 } 637 } 638 639 static int 640 ubsec_sysctl_init(void) 641 { 642 const struct sysctlnode *node = NULL; 643 644 ubsec_sysctllog = NULL; 645 646 sysctl_createv(&ubsec_sysctllog, 0, NULL, &node, 647 CTLFLAG_PERMANENT, 648 CTLTYPE_NODE, "ubsec", 649 SYSCTL_DESCR("ubsec opetions"), 650 NULL, 0, NULL, 0, 651 CTL_HW, CTL_CREATE, CTL_EOL); 652 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 653 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 654 CTLTYPE_INT, "maxbatch", 655 SYSCTL_DESCR("max ops to batch w/o interrupt"), 656 NULL, 0, &ubsec_maxbatch, 0, 657 CTL_CREATE, CTL_EOL); 658 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 659 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 660 CTLTYPE_INT, "maxaggr", 661 SYSCTL_DESCR("max ops to aggregate under one interrupt"), 662 NULL, 0, &ubsec_maxaggr, 0, 663 CTL_CREATE, CTL_EOL); 664 665 return 0; 666 } 667 668 /* 669 * UBSEC Interrupt routine 670 */ 671 static int 672 ubsec_intr(void *arg) 673 { 674 struct ubsec_softc *sc = arg; 675 volatile u_int32_t stat; 676 struct ubsec_q *q; 677 struct ubsec_dma *dmap; 678 int flags; 679 int npkts = 0, i; 680 681 mutex_spin_enter(&sc->sc_mtx); 682 stat = READ_REG(sc, BS_STAT); 683 stat &= sc->sc_statmask; 684 if (stat == 0) { 685 mutex_spin_exit(&sc->sc_mtx); 686 return (0); 687 } 688 689 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 690 691 /* 692 * Check to see if we have any packets waiting for us 693 */ 694 if ((stat & BS_STAT_MCR1_DONE)) { 695 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 696 q = SIMPLEQ_FIRST(&sc->sc_qchip); 697 dmap = q->q_dma; 698 699 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 700 break; 701 702 q = SIMPLEQ_FIRST(&sc->sc_qchip); 703 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 704 705 npkts = q->q_nstacked_mcrs; 706 sc->sc_nqchip -= 1+npkts; 707 /* 708 * search for further sc_qchip ubsec_q's that share 709 * the same MCR, and complete them too, they must be 710 * at the top. 711 */ 712 for (i = 0; i < npkts; i++) { 713 if(q->q_stacked_mcr[i]) 714 ubsec_callback(sc, q->q_stacked_mcr[i]); 715 else 716 break; 717 } 718 ubsec_callback(sc, q); 719 } 720 721 /* 722 * Don't send any more packet to chip if there has been 723 * a DMAERR. 724 */ 725 if (!(stat & BS_STAT_DMAERR)) 726 ubsec_feed(sc); 727 } 728 729 /* 730 * Check to see if we have any key setups/rng's waiting for us 731 */ 732 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 733 (stat & BS_STAT_MCR2_DONE)) { 734 struct ubsec_q2 *q2; 735 struct ubsec_mcr *mcr; 736 737 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 738 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 739 740 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 741 0, q2->q_mcr.dma_map->dm_mapsize, 742 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 743 744 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 745 746 /* A bug in new devices requires to swap this field */ 747 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 748 flags = htole16(mcr->mcr_flags); 749 else 750 flags = mcr->mcr_flags; 751 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 752 bus_dmamap_sync(sc->sc_dmat, 753 q2->q_mcr.dma_map, 0, 754 q2->q_mcr.dma_map->dm_mapsize, 755 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 756 break; 757 } 758 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 759 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next); 760 ubsec_callback2(sc, q2); 761 /* 762 * Don't send any more packet to chip if there has been 763 * a DMAERR. 764 */ 765 if (!(stat & BS_STAT_DMAERR)) 766 ubsec_feed2(sc); 767 } 768 } 769 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) { 770 struct ubsec_q2 *q2; 771 struct ubsec_mcr *mcr; 772 773 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) { 774 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4); 775 776 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 777 0, q2->q_mcr.dma_map->dm_mapsize, 778 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 779 780 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 781 782 /* A bug in new devices requires to swap this field */ 783 flags = htole16(mcr->mcr_flags); 784 785 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 786 bus_dmamap_sync(sc->sc_dmat, 787 q2->q_mcr.dma_map, 0, 788 q2->q_mcr.dma_map->dm_mapsize, 789 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 790 break; 791 } 792 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next); 793 ubsec_callback2(sc, q2); 794 /* 795 * Don't send any more packet to chip if there has been 796 * a DMAERR. 797 */ 798 if (!(stat & BS_STAT_DMAERR)) 799 ubsec_feed4(sc); 800 } 801 } 802 803 /* 804 * Check to see if we got any DMA Error 805 */ 806 if (stat & BS_STAT_DMAERR) { 807 #ifdef UBSEC_DEBUG 808 if (ubsec_debug) { 809 volatile u_int32_t a = READ_REG(sc, BS_ERR); 810 811 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev), 812 (a & BS_ERR_READ) ? "read" : "write", 813 a & BS_ERR_ADDR); 814 } 815 #endif /* UBSEC_DEBUG */ 816 ubsecstats.hst_dmaerr++; 817 ubsec_totalreset(sc); 818 ubsec_feed(sc); 819 } 820 821 if (sc->sc_needwakeup) { /* XXX check high watermark */ 822 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 823 #ifdef UBSEC_DEBUG 824 if (ubsec_debug) 825 printf("%s: wakeup crypto (%x)\n", device_xname(sc->sc_dev), 826 sc->sc_needwakeup); 827 #endif /* UBSEC_DEBUG */ 828 sc->sc_needwakeup &= ~wkeup; 829 crypto_unblock(sc->sc_cid, wkeup); 830 } 831 mutex_spin_exit(&sc->sc_mtx); 832 return (1); 833 } 834 835 /* 836 * ubsec_feed() - aggregate and post requests to chip 837 * OpenBSD comments: 838 * It is assumed that the caller set splnet() 839 */ 840 static void 841 ubsec_feed(struct ubsec_softc *sc) 842 { 843 struct ubsec_q *q, *q2; 844 int npkts, i; 845 void *v; 846 u_int32_t stat; 847 #ifdef UBSEC_DEBUG 848 static int max; 849 #endif /* UBSEC_DEBUG */ 850 851 npkts = sc->sc_nqueue; 852 if (npkts > ubsecstats.hst_maxqueue) 853 ubsecstats.hst_maxqueue = npkts; 854 if (npkts < 2) 855 goto feed1; 856 857 /* 858 * Decide how many ops to combine in a single MCR. We cannot 859 * aggregate more than UBS_MAX_AGGR because this is the number 860 * of slots defined in the data structure. Otherwise we clamp 861 * based on the tunable parameter ubsec_maxaggr. Note that 862 * aggregation can happen in two ways: either by batching ops 863 * from above or because the h/w backs up and throttles us. 864 * Aggregating ops reduces the number of interrupts to the host 865 * but also (potentially) increases the latency for processing 866 * completed ops as we only get an interrupt when all aggregated 867 * ops have completed. 868 */ 869 if (npkts > sc->sc_maxaggr) 870 npkts = sc->sc_maxaggr; 871 if (npkts > ubsec_maxaggr) 872 npkts = ubsec_maxaggr; 873 if (npkts > ubsecstats.hst_maxbatch) 874 ubsecstats.hst_maxbatch = npkts; 875 if (npkts < 2) 876 goto feed1; 877 ubsecstats.hst_totbatch += npkts-1; 878 879 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 880 if (stat & BS_STAT_DMAERR) { 881 ubsec_totalreset(sc); 882 ubsecstats.hst_dmaerr++; 883 } else { 884 ubsecstats.hst_mcr1full++; 885 } 886 return; 887 } 888 889 #ifdef UBSEC_DEBUG 890 if (ubsec_debug) 891 printf("merging %d records\n", npkts); 892 /* XXX temporary aggregation statistics reporting code */ 893 if (max < npkts) { 894 max = npkts; 895 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), max); 896 } 897 #endif /* UBSEC_DEBUG */ 898 899 q = SIMPLEQ_FIRST(&sc->sc_queue); 900 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 901 --sc->sc_nqueue; 902 903 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 904 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 905 if (q->q_dst_map != NULL) 906 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 907 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 908 909 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 910 911 for (i = 0; i < q->q_nstacked_mcrs; i++) { 912 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 913 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 914 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 915 if (q2->q_dst_map != NULL) 916 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 917 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 918 q2= SIMPLEQ_FIRST(&sc->sc_queue); 919 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next); 920 --sc->sc_nqueue; 921 922 v = ((void *)&q2->q_dma->d_dma->d_mcr); 923 v = (char*)v + (sizeof(struct ubsec_mcr) - 924 sizeof(struct ubsec_mcr_add)); 925 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, sizeof(struct ubsec_mcr_add)); 926 q->q_stacked_mcr[i] = q2; 927 } 928 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 929 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 930 sc->sc_nqchip += npkts; 931 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 932 ubsecstats.hst_maxqchip = sc->sc_nqchip; 933 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 934 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 935 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 936 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 937 offsetof(struct ubsec_dmachunk, d_mcr)); 938 return; 939 940 feed1: 941 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 942 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 943 if (stat & BS_STAT_DMAERR) { 944 ubsec_totalreset(sc); 945 ubsecstats.hst_dmaerr++; 946 } else { 947 ubsecstats.hst_mcr1full++; 948 } 949 break; 950 } 951 952 q = SIMPLEQ_FIRST(&sc->sc_queue); 953 954 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 955 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 956 if (q->q_dst_map != NULL) 957 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 958 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 959 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 960 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 961 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 962 963 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 964 offsetof(struct ubsec_dmachunk, d_mcr)); 965 #ifdef UBSEC_DEBUG 966 if (ubsec_debug) 967 printf("feed: q->chip %p %08x stat %08x\n", 968 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr, 969 stat); 970 #endif /* UBSEC_DEBUG */ 971 q = SIMPLEQ_FIRST(&sc->sc_queue); 972 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 973 --sc->sc_nqueue; 974 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 975 sc->sc_nqchip++; 976 } 977 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 978 ubsecstats.hst_maxqchip = sc->sc_nqchip; 979 } 980 981 /* 982 * Allocate a new 'session' and return an encoded session id. 'sidp' 983 * contains our registration id, and should contain an encoded session 984 * id on successful allocation. 985 */ 986 static int 987 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 988 { 989 struct cryptoini *c, *encini = NULL, *macini = NULL; 990 struct ubsec_softc *sc; 991 struct ubsec_session *ses = NULL; 992 MD5_CTX md5ctx; 993 SHA1_CTX sha1ctx; 994 int i, sesn; 995 996 sc = arg; 997 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/); 998 999 if (sidp == NULL || cri == NULL || sc == NULL) 1000 return (EINVAL); 1001 1002 for (c = cri; c != NULL; c = c->cri_next) { 1003 if (c->cri_alg == CRYPTO_MD5_HMAC_96 || 1004 c->cri_alg == CRYPTO_SHA1_HMAC_96) { 1005 if (macini) 1006 return (EINVAL); 1007 macini = c; 1008 } else if (c->cri_alg == CRYPTO_DES_CBC || 1009 c->cri_alg == CRYPTO_3DES_CBC || 1010 c->cri_alg == CRYPTO_AES_CBC) { 1011 if (encini) 1012 return (EINVAL); 1013 encini = c; 1014 } else 1015 return (EINVAL); 1016 } 1017 if (encini == NULL && macini == NULL) 1018 return (EINVAL); 1019 1020 if (encini && encini->cri_alg == CRYPTO_AES_CBC) { 1021 switch (encini->cri_klen) { 1022 case 128: 1023 case 192: 1024 case 256: 1025 break; 1026 default: 1027 return (EINVAL); 1028 } 1029 } 1030 1031 if (sc->sc_sessions == NULL) { 1032 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 1033 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1034 if (ses == NULL) 1035 return (ENOMEM); 1036 sesn = 0; 1037 sc->sc_nsessions = 1; 1038 } else { 1039 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 1040 if (sc->sc_sessions[sesn].ses_used == 0) { 1041 ses = &sc->sc_sessions[sesn]; 1042 break; 1043 } 1044 } 1045 1046 if (ses == NULL) { 1047 sesn = sc->sc_nsessions; 1048 ses = (struct ubsec_session *)malloc((sesn + 1) * 1049 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1050 if (ses == NULL) 1051 return (ENOMEM); 1052 memcpy(ses, sc->sc_sessions, sesn * 1053 sizeof(struct ubsec_session)); 1054 memset(sc->sc_sessions, 0, sesn * 1055 sizeof(struct ubsec_session)); 1056 free(sc->sc_sessions, M_DEVBUF); 1057 sc->sc_sessions = ses; 1058 ses = &sc->sc_sessions[sesn]; 1059 sc->sc_nsessions++; 1060 } 1061 } 1062 1063 memset(ses, 0, sizeof(struct ubsec_session)); 1064 ses->ses_used = 1; 1065 if (encini) { 1066 /* get an IV, network byte order */ 1067 #ifdef __NetBSD__ 1068 cprng_fast(ses->ses_iv, sizeof(ses->ses_iv)); 1069 #else 1070 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv)); 1071 #endif 1072 1073 /* Go ahead and compute key in ubsec's byte order */ 1074 if (encini->cri_alg == CRYPTO_AES_CBC) { 1075 memcpy(ses->ses_key, encini->cri_key, 1076 encini->cri_klen / 8); 1077 } 1078 if (encini->cri_alg == CRYPTO_DES_CBC) { 1079 memcpy(&ses->ses_key[0], encini->cri_key, 8); 1080 memcpy(&ses->ses_key[2], encini->cri_key, 8); 1081 memcpy(&ses->ses_key[4], encini->cri_key, 8); 1082 } else 1083 memcpy(ses->ses_key, encini->cri_key, 24); 1084 1085 SWAP32(ses->ses_key[0]); 1086 SWAP32(ses->ses_key[1]); 1087 SWAP32(ses->ses_key[2]); 1088 SWAP32(ses->ses_key[3]); 1089 SWAP32(ses->ses_key[4]); 1090 SWAP32(ses->ses_key[5]); 1091 } 1092 1093 if (macini) { 1094 for (i = 0; i < macini->cri_klen / 8; i++) 1095 macini->cri_key[i] ^= HMAC_IPAD_VAL; 1096 1097 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1098 MD5Init(&md5ctx); 1099 MD5Update(&md5ctx, macini->cri_key, 1100 macini->cri_klen / 8); 1101 MD5Update(&md5ctx, hmac_ipad_buffer, 1102 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1103 memcpy(ses->ses_hminner, md5ctx.state, 1104 sizeof(md5ctx.state)); 1105 } else { 1106 SHA1Init(&sha1ctx); 1107 SHA1Update(&sha1ctx, macini->cri_key, 1108 macini->cri_klen / 8); 1109 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1110 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1111 memcpy(ses->ses_hminner, sha1ctx.state, 1112 sizeof(sha1ctx.state)); 1113 } 1114 1115 for (i = 0; i < macini->cri_klen / 8; i++) 1116 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1117 1118 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1119 MD5Init(&md5ctx); 1120 MD5Update(&md5ctx, macini->cri_key, 1121 macini->cri_klen / 8); 1122 MD5Update(&md5ctx, hmac_opad_buffer, 1123 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1124 memcpy(ses->ses_hmouter, md5ctx.state, 1125 sizeof(md5ctx.state)); 1126 } else { 1127 SHA1Init(&sha1ctx); 1128 SHA1Update(&sha1ctx, macini->cri_key, 1129 macini->cri_klen / 8); 1130 SHA1Update(&sha1ctx, hmac_opad_buffer, 1131 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1132 memcpy(ses->ses_hmouter, sha1ctx.state, 1133 sizeof(sha1ctx.state)); 1134 } 1135 1136 for (i = 0; i < macini->cri_klen / 8; i++) 1137 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1138 } 1139 1140 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn); 1141 return (0); 1142 } 1143 1144 /* 1145 * Deallocate a session. 1146 */ 1147 static int 1148 ubsec_freesession(void *arg, u_int64_t tid) 1149 { 1150 struct ubsec_softc *sc; 1151 int session; 1152 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1153 1154 sc = arg; 1155 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/); 1156 1157 session = UBSEC_SESSION(sid); 1158 if (session >= sc->sc_nsessions) 1159 return (EINVAL); 1160 1161 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 1162 return (0); 1163 } 1164 1165 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */ 1166 static void 1167 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 1168 { 1169 struct ubsec_operand *op = arg; 1170 1171 KASSERT(nsegs <= UBS_MAX_SCATTER 1172 /*, ("Too many DMA segments returned when mapping operand")*/); 1173 #ifdef UBSEC_DEBUG 1174 if (ubsec_debug) 1175 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 1176 (u_int) mapsize, nsegs); 1177 #endif 1178 op->mapsize = mapsize; 1179 op->nsegs = nsegs; 1180 memcpy(op->segs, seg, nsegs * sizeof (seg[0])); 1181 } 1182 #endif 1183 1184 static int 1185 ubsec_process(void *arg, struct cryptop *crp, int hint) 1186 { 1187 struct ubsec_q *q = NULL; 1188 #ifdef __OpenBSD__ 1189 int card; 1190 #endif 1191 int err = 0, i, j, nicealign; 1192 struct ubsec_softc *sc; 1193 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1194 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1195 int sskip, dskip, stheend, dtheend; 1196 int16_t coffset; 1197 struct ubsec_session *ses, key; 1198 struct ubsec_dma *dmap = NULL; 1199 u_int16_t flags = 0; 1200 int ivlen = 0, keylen = 0; 1201 1202 sc = arg; 1203 KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/); 1204 1205 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1206 ubsecstats.hst_invalid++; 1207 return (EINVAL); 1208 } 1209 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1210 ubsecstats.hst_badsession++; 1211 return (EINVAL); 1212 } 1213 1214 mutex_spin_enter(&sc->sc_mtx); 1215 1216 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1217 ubsecstats.hst_queuefull++; 1218 sc->sc_needwakeup |= CRYPTO_SYMQ; 1219 mutex_spin_exit(&sc->sc_mtx); 1220 return(ERESTART); 1221 } 1222 1223 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1224 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next); 1225 mutex_spin_exit(&sc->sc_mtx); 1226 1227 dmap = q->q_dma; /* Save dma pointer */ 1228 /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */ 1229 memset(q, 0, offsetof(struct ubsec_q, q_src_map)); 1230 memset(&key, 0, sizeof(key)); 1231 1232 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1233 q->q_dma = dmap; 1234 ses = &sc->sc_sessions[q->q_sesn]; 1235 1236 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1237 q->q_src_m = (struct mbuf *)crp->crp_buf; 1238 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1239 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1240 q->q_src_io = (struct uio *)crp->crp_buf; 1241 q->q_dst_io = (struct uio *)crp->crp_buf; 1242 } else { 1243 ubsecstats.hst_badflags++; 1244 err = EINVAL; 1245 goto errout; /* XXX we don't handle contiguous blocks! */ 1246 } 1247 1248 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr)); 1249 1250 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1251 dmap->d_dma->d_mcr.mcr_flags = 0; 1252 q->q_crp = crp; 1253 1254 crd1 = crp->crp_desc; 1255 if (crd1 == NULL) { 1256 ubsecstats.hst_nodesc++; 1257 err = EINVAL; 1258 goto errout; 1259 } 1260 crd2 = crd1->crd_next; 1261 1262 if (crd2 == NULL) { 1263 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1264 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) { 1265 maccrd = crd1; 1266 enccrd = NULL; 1267 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1268 crd1->crd_alg == CRYPTO_3DES_CBC || 1269 crd1->crd_alg == CRYPTO_AES_CBC) { 1270 maccrd = NULL; 1271 enccrd = crd1; 1272 } else { 1273 ubsecstats.hst_badalg++; 1274 err = EINVAL; 1275 goto errout; 1276 } 1277 } else { 1278 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1279 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) && 1280 (crd2->crd_alg == CRYPTO_DES_CBC || 1281 crd2->crd_alg == CRYPTO_3DES_CBC || 1282 crd2->crd_alg == CRYPTO_AES_CBC) && 1283 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1284 maccrd = crd1; 1285 enccrd = crd2; 1286 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1287 crd1->crd_alg == CRYPTO_3DES_CBC || 1288 crd1->crd_alg == CRYPTO_AES_CBC) && 1289 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 1290 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) && 1291 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1292 enccrd = crd1; 1293 maccrd = crd2; 1294 } else { 1295 /* 1296 * We cannot order the ubsec as requested 1297 */ 1298 ubsecstats.hst_badalg++; 1299 err = EINVAL; 1300 goto errout; 1301 } 1302 } 1303 1304 if (enccrd) { 1305 if (enccrd->crd_alg == CRYPTO_AES_CBC) { 1306 if ((sc->sc_flags & UBS_FLAGS_AES) == 0) { 1307 /* 1308 * We cannot order the ubsec as requested 1309 */ 1310 ubsecstats.hst_badalg++; 1311 err = EINVAL; 1312 goto errout; 1313 } 1314 flags |= htole16(UBS_PKTCTX_ENC_AES); 1315 switch (enccrd->crd_klen) { 1316 case 128: 1317 case 192: 1318 case 256: 1319 keylen = enccrd->crd_klen / 8; 1320 break; 1321 default: 1322 err = EINVAL; 1323 goto errout; 1324 } 1325 ivlen = 16; 1326 } else { 1327 flags |= htole16(UBS_PKTCTX_ENC_3DES); 1328 ivlen = 8; 1329 keylen = 24; 1330 } 1331 1332 encoffset = enccrd->crd_skip; 1333 1334 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1335 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1336 1337 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1338 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1339 else { 1340 for (i = 0; i < (ivlen / 4); i++) 1341 key.ses_iv[i] = ses->ses_iv[i]; 1342 } 1343 1344 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1345 if (crp->crp_flags & CRYPTO_F_IMBUF) 1346 m_copyback(q->q_src_m, 1347 enccrd->crd_inject, 1348 ivlen, (void *)key.ses_iv); 1349 else if (crp->crp_flags & CRYPTO_F_IOV) 1350 cuio_copyback(q->q_src_io, 1351 enccrd->crd_inject, 1352 ivlen, (void *)key.ses_iv); 1353 } 1354 } else { 1355 flags |= htole16(UBS_PKTCTX_INBOUND); 1356 1357 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1358 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1359 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1360 m_copydata(q->q_src_m, enccrd->crd_inject, 1361 ivlen, (void *)key.ses_iv); 1362 else if (crp->crp_flags & CRYPTO_F_IOV) 1363 cuio_copydata(q->q_src_io, 1364 enccrd->crd_inject, 8, 1365 (void *)key.ses_iv); 1366 } 1367 1368 for (i = 0; i < (keylen / 4); i++) 1369 key.ses_key[i] = ses->ses_key[i]; 1370 for (i = 0; i < (ivlen / 4); i++) 1371 SWAP32(key.ses_iv[i]); 1372 } 1373 1374 if (maccrd) { 1375 macoffset = maccrd->crd_skip; 1376 1377 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96) 1378 flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1379 else 1380 flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1381 1382 for (i = 0; i < 5; i++) { 1383 key.ses_hminner[i] = ses->ses_hminner[i]; 1384 key.ses_hmouter[i] = ses->ses_hmouter[i]; 1385 1386 HTOLE32(key.ses_hminner[i]); 1387 HTOLE32(key.ses_hmouter[i]); 1388 } 1389 } 1390 1391 if (enccrd && maccrd) { 1392 /* 1393 * ubsec cannot handle packets where the end of encryption 1394 * and authentication are not the same, or where the 1395 * encrypted part begins before the authenticated part. 1396 */ 1397 if ((encoffset + enccrd->crd_len) != 1398 (macoffset + maccrd->crd_len)) { 1399 ubsecstats.hst_lenmismatch++; 1400 err = EINVAL; 1401 goto errout; 1402 } 1403 if (enccrd->crd_skip < maccrd->crd_skip) { 1404 ubsecstats.hst_skipmismatch++; 1405 err = EINVAL; 1406 goto errout; 1407 } 1408 sskip = maccrd->crd_skip; 1409 cpskip = dskip = enccrd->crd_skip; 1410 stheend = maccrd->crd_len; 1411 dtheend = enccrd->crd_len; 1412 coffset = enccrd->crd_skip - maccrd->crd_skip; 1413 cpoffset = cpskip + dtheend; 1414 #ifdef UBSEC_DEBUG 1415 if (ubsec_debug) { 1416 printf("mac: skip %d, len %d, inject %d\n", 1417 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1418 printf("enc: skip %d, len %d, inject %d\n", 1419 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1420 printf("src: skip %d, len %d\n", sskip, stheend); 1421 printf("dst: skip %d, len %d\n", dskip, dtheend); 1422 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1423 coffset, stheend, cpskip, cpoffset); 1424 } 1425 #endif 1426 } else { 1427 cpskip = dskip = sskip = macoffset + encoffset; 1428 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1429 cpoffset = cpskip + dtheend; 1430 coffset = 0; 1431 } 1432 1433 if (q->q_src_map == NULL) { 1434 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */ 1435 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER, 1436 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) { 1437 err = ENOMEM; 1438 goto errout; 1439 } 1440 } 1441 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1442 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1443 q->q_src_m, BUS_DMA_NOWAIT) != 0) { 1444 ubsecstats.hst_noload++; 1445 err = ENOMEM; 1446 goto errout; 1447 } 1448 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1449 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1450 q->q_src_io, BUS_DMA_NOWAIT) != 0) { 1451 ubsecstats.hst_noload++; 1452 err = ENOMEM; 1453 goto errout; 1454 } 1455 } 1456 nicealign = ubsec_dmamap_aligned(q->q_src_map); 1457 1458 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1459 1460 #ifdef UBSEC_DEBUG 1461 if (ubsec_debug) 1462 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1463 #endif 1464 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) { 1465 struct ubsec_pktbuf *pb; 1466 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len; 1467 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr; 1468 1469 if (sskip >= packl) { 1470 sskip -= packl; 1471 continue; 1472 } 1473 1474 packl -= sskip; 1475 packp += sskip; 1476 sskip = 0; 1477 1478 if (packl > 0xfffc) { 1479 err = EIO; 1480 goto errout; 1481 } 1482 1483 if (j == 0) 1484 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1485 else 1486 pb = &dmap->d_dma->d_sbuf[j - 1]; 1487 1488 pb->pb_addr = htole32(packp); 1489 1490 if (stheend) { 1491 if (packl > stheend) { 1492 pb->pb_len = htole32(stheend); 1493 stheend = 0; 1494 } else { 1495 pb->pb_len = htole32(packl); 1496 stheend -= packl; 1497 } 1498 } else 1499 pb->pb_len = htole32(packl); 1500 1501 if ((i + 1) == q->q_src_map->dm_nsegs) 1502 pb->pb_next = 0; 1503 else 1504 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1505 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1506 j++; 1507 } 1508 1509 if (enccrd == NULL && maccrd != NULL) { 1510 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1511 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1512 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1513 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1514 #ifdef UBSEC_DEBUG 1515 if (ubsec_debug) 1516 printf("opkt: %x %x %x\n", 1517 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1518 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1519 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1520 1521 #endif 1522 } else { 1523 if (crp->crp_flags & CRYPTO_F_IOV) { 1524 if (!nicealign) { 1525 ubsecstats.hst_iovmisaligned++; 1526 err = EINVAL; 1527 goto errout; 1528 } 1529 if (q->q_dst_map == NULL) { 1530 if (q->q_cached_dst_map == NULL) { 1531 /* XXX: ``what the heck's that'' 0xfff0? */ 1532 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1533 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1534 &q->q_cached_dst_map) != 0) { 1535 ubsecstats.hst_nomap++; 1536 err = ENOMEM; 1537 goto errout; 1538 } 1539 } 1540 q->q_dst_map = q->q_cached_dst_map; 1541 } 1542 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1543 q->q_dst_io, BUS_DMA_NOWAIT) != 0) { 1544 ubsecstats.hst_noload++; 1545 err = ENOMEM; 1546 goto errout; 1547 } 1548 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1549 if (nicealign) { 1550 q->q_dst_m = q->q_src_m; 1551 q->q_dst_map = q->q_src_map; 1552 } else { 1553 int totlen, len; 1554 struct mbuf *m, *top, **mp; 1555 1556 ubsecstats.hst_unaligned++; 1557 totlen = q->q_src_map->dm_mapsize; 1558 if (q->q_src_m->m_flags & M_PKTHDR) { 1559 len = MHLEN; 1560 MGETHDR(m, M_DONTWAIT, MT_DATA); 1561 /*XXX FIXME: m_dup_pkthdr */ 1562 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) { 1563 m_free(m); 1564 m = NULL; 1565 } 1566 } else { 1567 len = MLEN; 1568 MGET(m, M_DONTWAIT, MT_DATA); 1569 } 1570 if (m == NULL) { 1571 ubsecstats.hst_nombuf++; 1572 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1573 goto errout; 1574 } 1575 if (len == MHLEN) 1576 /*XXX was M_DUP_PKTHDR*/ 1577 M_COPY_PKTHDR(m, q->q_src_m); 1578 if (totlen >= MINCLSIZE) { 1579 MCLGET(m, M_DONTWAIT); 1580 if ((m->m_flags & M_EXT) == 0) { 1581 m_free(m); 1582 ubsecstats.hst_nomcl++; 1583 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1584 goto errout; 1585 } 1586 len = MCLBYTES; 1587 } 1588 m->m_len = len; 1589 top = NULL; 1590 mp = ⊤ 1591 1592 while (totlen > 0) { 1593 if (top) { 1594 MGET(m, M_DONTWAIT, MT_DATA); 1595 if (m == NULL) { 1596 m_freem(top); 1597 ubsecstats.hst_nombuf++; 1598 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1599 goto errout; 1600 } 1601 len = MLEN; 1602 } 1603 if (top && totlen >= MINCLSIZE) { 1604 MCLGET(m, M_DONTWAIT); 1605 if ((m->m_flags & M_EXT) == 0) { 1606 *mp = m; 1607 m_freem(top); 1608 ubsecstats.hst_nomcl++; 1609 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1610 goto errout; 1611 } 1612 len = MCLBYTES; 1613 } 1614 m->m_len = len = min(totlen, len); 1615 totlen -= len; 1616 *mp = m; 1617 mp = &m->m_next; 1618 } 1619 q->q_dst_m = top; 1620 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1621 cpskip, cpoffset); 1622 if (q->q_dst_map == NULL) { 1623 if (q->q_cached_dst_map == NULL) { 1624 /* XXX again, what the heck is that 0xfff0? */ 1625 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1626 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1627 &q->q_cached_dst_map) != 0) { 1628 ubsecstats.hst_nomap++; 1629 err = ENOMEM; 1630 goto errout; 1631 } 1632 } 1633 q->q_dst_map = q->q_cached_dst_map; 1634 } 1635 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1636 q->q_dst_map, q->q_dst_m, 1637 BUS_DMA_NOWAIT) != 0) { 1638 ubsecstats.hst_noload++; 1639 err = ENOMEM; 1640 goto errout; 1641 } 1642 } 1643 } else { 1644 ubsecstats.hst_badflags++; 1645 err = EINVAL; 1646 goto errout; 1647 } 1648 1649 #ifdef UBSEC_DEBUG 1650 if (ubsec_debug) 1651 printf("dst skip: %d\n", dskip); 1652 #endif 1653 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) { 1654 struct ubsec_pktbuf *pb; 1655 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len; 1656 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr; 1657 1658 if (dskip >= packl) { 1659 dskip -= packl; 1660 continue; 1661 } 1662 1663 packl -= dskip; 1664 packp += dskip; 1665 dskip = 0; 1666 1667 if (packl > 0xfffc) { 1668 err = EIO; 1669 goto errout; 1670 } 1671 1672 if (j == 0) 1673 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1674 else 1675 pb = &dmap->d_dma->d_dbuf[j - 1]; 1676 1677 pb->pb_addr = htole32(packp); 1678 1679 if (dtheend) { 1680 if (packl > dtheend) { 1681 pb->pb_len = htole32(dtheend); 1682 dtheend = 0; 1683 } else { 1684 pb->pb_len = htole32(packl); 1685 dtheend -= packl; 1686 } 1687 } else 1688 pb->pb_len = htole32(packl); 1689 1690 if ((i + 1) == q->q_dst_map->dm_nsegs) { 1691 if (maccrd) 1692 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1693 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1694 else 1695 pb->pb_next = 0; 1696 } else 1697 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1698 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1699 j++; 1700 } 1701 } 1702 1703 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1704 offsetof(struct ubsec_dmachunk, d_ctx)); 1705 1706 if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) { 1707 struct ubsec_pktctx_aes128 *aes128; 1708 struct ubsec_pktctx_aes192 *aes192; 1709 struct ubsec_pktctx_aes256 *aes256; 1710 struct ubsec_pktctx_hdr *ph; 1711 u_int8_t *ctx; 1712 1713 ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1714 offsetof(struct ubsec_dmachunk, d_ctx); 1715 1716 ph = (struct ubsec_pktctx_hdr *)ctx; 1717 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES); 1718 ph->ph_flags = flags; 1719 ph->ph_offset = htole16(coffset >> 2); 1720 1721 switch (enccrd->crd_klen) { 1722 case 128: 1723 aes128 = (struct ubsec_pktctx_aes128 *)ctx; 1724 ph->ph_len = htole16(sizeof(*aes128)); 1725 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128); 1726 for (i = 0; i < 4; i++) 1727 aes128->pc_aeskey[i] = key.ses_key[i]; 1728 for (i = 0; i < 5; i++) 1729 aes128->pc_hminner[i] = key.ses_hminner[i]; 1730 for (i = 0; i < 5; i++) 1731 aes128->pc_hmouter[i] = key.ses_hmouter[i]; 1732 for (i = 0; i < 4; i++) 1733 aes128->pc_iv[i] = key.ses_iv[i]; 1734 break; 1735 case 192: 1736 aes192 = (struct ubsec_pktctx_aes192 *)ctx; 1737 ph->ph_len = htole16(sizeof(*aes192)); 1738 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192); 1739 for (i = 0; i < 6; i++) 1740 aes192->pc_aeskey[i] = key.ses_key[i]; 1741 for (i = 0; i < 5; i++) 1742 aes192->pc_hminner[i] = key.ses_hminner[i]; 1743 for (i = 0; i < 5; i++) 1744 aes192->pc_hmouter[i] = key.ses_hmouter[i]; 1745 for (i = 0; i < 4; i++) 1746 aes192->pc_iv[i] = key.ses_iv[i]; 1747 break; 1748 case 256: 1749 aes256 = (struct ubsec_pktctx_aes256 *)ctx; 1750 ph->ph_len = htole16(sizeof(*aes256)); 1751 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256); 1752 for (i = 0; i < 8; i++) 1753 aes256->pc_aeskey[i] = key.ses_key[i]; 1754 for (i = 0; i < 5; i++) 1755 aes256->pc_hminner[i] = key.ses_hminner[i]; 1756 for (i = 0; i < 5; i++) 1757 aes256->pc_hmouter[i] = key.ses_hmouter[i]; 1758 for (i = 0; i < 4; i++) 1759 aes256->pc_iv[i] = key.ses_iv[i]; 1760 break; 1761 } 1762 } else if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1763 struct ubsec_pktctx_3des *ctx; 1764 struct ubsec_pktctx_hdr *ph; 1765 1766 ctx = (struct ubsec_pktctx_3des *) 1767 ((u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1768 offsetof(struct ubsec_dmachunk, d_ctx)); 1769 1770 ph = (struct ubsec_pktctx_hdr *)ctx; 1771 ph->ph_len = htole16(sizeof(*ctx)); 1772 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES); 1773 ph->ph_flags = flags; 1774 ph->ph_offset = htole16(coffset >> 2); 1775 1776 for (i = 0; i < 6; i++) 1777 ctx->pc_deskey[i] = key.ses_key[i]; 1778 for (i = 0; i < 5; i++) 1779 ctx->pc_hminner[i] = key.ses_hminner[i]; 1780 for (i = 0; i < 5; i++) 1781 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1782 for (i = 0; i < 2; i++) 1783 ctx->pc_iv[i] = key.ses_iv[i]; 1784 } else { 1785 struct ubsec_pktctx *ctx = (struct ubsec_pktctx *) 1786 ((u_int8_t *)dmap->d_alloc.dma_vaddr + 1787 offsetof(struct ubsec_dmachunk, d_ctx)); 1788 1789 ctx->pc_flags = flags; 1790 ctx->pc_offset = htole16(coffset >> 2); 1791 for (i = 0; i < 6; i++) 1792 ctx->pc_deskey[i] = key.ses_key[i]; 1793 for (i = 0; i < 5; i++) 1794 ctx->pc_hminner[i] = key.ses_hminner[i]; 1795 for (i = 0; i < 5; i++) 1796 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1797 for (i = 0; i < 2; i++) 1798 ctx->pc_iv[i] = key.ses_iv[i]; 1799 } 1800 1801 mutex_spin_enter(&sc->sc_mtx); 1802 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1803 sc->sc_nqueue++; 1804 ubsecstats.hst_ipackets++; 1805 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize; 1806 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1807 ubsec_feed(sc); 1808 mutex_spin_exit(&sc->sc_mtx); 1809 return (0); 1810 1811 errout: 1812 if (q != NULL) { 1813 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1814 m_freem(q->q_dst_m); 1815 1816 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1817 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1818 } 1819 if (q->q_src_map != NULL) { 1820 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1821 } 1822 1823 mutex_spin_enter(&sc->sc_mtx); 1824 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1825 mutex_spin_exit(&sc->sc_mtx); 1826 } 1827 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */ 1828 if (err == EINVAL) 1829 ubsecstats.hst_invalid++; 1830 else 1831 ubsecstats.hst_nomem++; 1832 #endif 1833 if (err != ERESTART) { 1834 crp->crp_etype = err; 1835 crypto_done(crp); 1836 } else { 1837 sc->sc_needwakeup |= CRYPTO_SYMQ; 1838 } 1839 return (err); 1840 } 1841 1842 static void 1843 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1844 { 1845 struct cryptop *crp = (struct cryptop *)q->q_crp; 1846 struct cryptodesc *crd; 1847 struct ubsec_dma *dmap = q->q_dma; 1848 1849 ubsecstats.hst_opackets++; 1850 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1851 1852 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0, 1853 dmap->d_alloc.dma_map->dm_mapsize, 1854 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1855 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1856 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1857 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1858 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1859 } 1860 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 1861 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1862 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1863 1864 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1865 m_freem(q->q_src_m); 1866 crp->crp_buf = (void *)q->q_dst_m; 1867 } 1868 1869 /* copy out IV for future use */ 1870 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1871 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1872 if (crd->crd_alg != CRYPTO_DES_CBC && 1873 crd->crd_alg != CRYPTO_3DES_CBC && 1874 crd->crd_alg != CRYPTO_AES_CBC) 1875 continue; 1876 if (crp->crp_flags & CRYPTO_F_IMBUF) 1877 m_copydata((struct mbuf *)crp->crp_buf, 1878 crd->crd_skip + crd->crd_len - 8, 8, 1879 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1880 else if (crp->crp_flags & CRYPTO_F_IOV) { 1881 cuio_copydata((struct uio *)crp->crp_buf, 1882 crd->crd_skip + crd->crd_len - 8, 8, 1883 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1884 } 1885 break; 1886 } 1887 } 1888 1889 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1890 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 && 1891 crd->crd_alg != CRYPTO_SHA1_HMAC_96) 1892 continue; 1893 if (crp->crp_flags & CRYPTO_F_IMBUF) 1894 m_copyback((struct mbuf *)crp->crp_buf, 1895 crd->crd_inject, 12, 1896 (void *)dmap->d_dma->d_macbuf); 1897 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1898 bcopy((void *)dmap->d_dma->d_macbuf, 1899 crp->crp_mac, 12); 1900 break; 1901 } 1902 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1903 crypto_done(crp); 1904 } 1905 1906 static void 1907 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1908 { 1909 int i, j, dlen, slen; 1910 char *dptr, *sptr; 1911 1912 j = 0; 1913 sptr = srcm->m_data; 1914 slen = srcm->m_len; 1915 dptr = dstm->m_data; 1916 dlen = dstm->m_len; 1917 1918 while (1) { 1919 for (i = 0; i < min(slen, dlen); i++) { 1920 if (j < hoffset || j >= toffset) 1921 *dptr++ = *sptr++; 1922 slen--; 1923 dlen--; 1924 j++; 1925 } 1926 if (slen == 0) { 1927 srcm = srcm->m_next; 1928 if (srcm == NULL) 1929 return; 1930 sptr = srcm->m_data; 1931 slen = srcm->m_len; 1932 } 1933 if (dlen == 0) { 1934 dstm = dstm->m_next; 1935 if (dstm == NULL) 1936 return; 1937 dptr = dstm->m_data; 1938 dlen = dstm->m_len; 1939 } 1940 } 1941 } 1942 1943 /* 1944 * feed the key generator, must be called at splnet() or higher. 1945 */ 1946 static void 1947 ubsec_feed2(struct ubsec_softc *sc) 1948 { 1949 struct ubsec_q2 *q; 1950 1951 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1952 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1953 break; 1954 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1955 1956 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1957 q->q_mcr.dma_map->dm_mapsize, 1958 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1959 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1960 q->q_ctx.dma_map->dm_mapsize, 1961 BUS_DMASYNC_PREWRITE); 1962 1963 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1964 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1965 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next); 1966 --sc->sc_nqueue2; 1967 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1968 } 1969 } 1970 1971 /* 1972 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices) 1973 */ 1974 void 1975 ubsec_feed4(struct ubsec_softc *sc) 1976 { 1977 struct ubsec_q2 *q; 1978 1979 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) { 1980 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL) 1981 break; 1982 q = SIMPLEQ_FIRST(&sc->sc_queue4); 1983 1984 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1985 q->q_mcr.dma_map->dm_mapsize, 1986 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1987 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1988 q->q_ctx.dma_map->dm_mapsize, 1989 BUS_DMASYNC_PREWRITE); 1990 1991 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr); 1992 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next); 1993 --sc->sc_nqueue4; 1994 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next); 1995 } 1996 } 1997 1998 /* 1999 * Callback for handling random numbers 2000 */ 2001 static void 2002 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 2003 { 2004 struct cryptkop *krp; 2005 struct ubsec_ctx_keyop *ctx; 2006 2007 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 2008 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 2009 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2010 2011 switch (q->q_type) { 2012 #ifndef UBSEC_NO_RNG 2013 case UBS_CTXOP_RNGSHA1: 2014 case UBS_CTXOP_RNGBYPASS: { 2015 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 2016 u_int32_t *p; 2017 int i; 2018 2019 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2020 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2021 p = (u_int32_t *)rng->rng_buf.dma_vaddr; 2022 #ifndef __NetBSD__ 2023 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++) 2024 add_true_randomness(letoh32(*p)); 2025 #else 2026 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t); 2027 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY); 2028 sc->sc_rng_need -= i; 2029 #endif 2030 rng->rng_used = 0; 2031 #ifdef __OpenBSD__ 2032 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2033 #else 2034 if (sc->sc_rng_need > 0) { 2035 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2036 } 2037 #endif 2038 break; 2039 } 2040 #endif 2041 case UBS_CTXOP_MODEXP: { 2042 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2043 u_int rlen, clen; 2044 2045 krp = me->me_krp; 2046 rlen = (me->me_modbits + 7) / 8; 2047 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 2048 2049 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2050 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2051 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2052 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2053 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2054 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2055 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2056 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2057 2058 if (clen < rlen) 2059 krp->krp_status = E2BIG; 2060 else { 2061 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 2062 memset(krp->krp_param[krp->krp_iparams].crp_p, 0, 2063 (krp->krp_param[krp->krp_iparams].crp_nbits 2064 + 7) / 8); 2065 bcopy(me->me_C.dma_vaddr, 2066 krp->krp_param[krp->krp_iparams].crp_p, 2067 (me->me_modbits + 7) / 8); 2068 } else 2069 ubsec_kshift_l(me->me_shiftbits, 2070 me->me_C.dma_vaddr, me->me_normbits, 2071 krp->krp_param[krp->krp_iparams].crp_p, 2072 krp->krp_param[krp->krp_iparams].crp_nbits); 2073 } 2074 2075 crypto_kdone(krp); 2076 2077 /* bzero all potentially sensitive data */ 2078 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2079 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2080 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2081 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2082 2083 /* Can't free here, so put us on the free list. */ 2084 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 2085 break; 2086 } 2087 case UBS_CTXOP_RSAPRIV: { 2088 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2089 u_int len; 2090 2091 krp = rp->rpr_krp; 2092 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0, 2093 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2094 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0, 2095 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2096 2097 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 2098 bcopy(rp->rpr_msgout.dma_vaddr, 2099 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 2100 2101 crypto_kdone(krp); 2102 2103 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 2104 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 2105 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size); 2106 2107 /* Can't free here, so put us on the free list. */ 2108 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 2109 break; 2110 } 2111 default: 2112 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev), 2113 letoh16(ctx->ctx_op)); 2114 break; 2115 } 2116 } 2117 2118 #ifndef UBSEC_NO_RNG 2119 2120 static void 2121 ubsec_rng_get(size_t bytes, void *vsc) 2122 { 2123 struct ubsec_softc *sc = vsc; 2124 2125 mutex_spin_enter(&sc->sc_mtx); 2126 sc->sc_rng_need = bytes; 2127 ubsec_rng_locked(sc); 2128 mutex_spin_exit(&sc->sc_mtx); 2129 2130 } 2131 2132 static void 2133 ubsec_rng(void *vsc) 2134 { 2135 struct ubsec_softc *sc = vsc; 2136 mutex_spin_enter(&sc->sc_mtx); 2137 ubsec_rng_locked(sc); 2138 mutex_spin_exit(&sc->sc_mtx); 2139 } 2140 2141 static void 2142 ubsec_rng_locked(void *vsc) 2143 { 2144 struct ubsec_softc *sc = vsc; 2145 struct ubsec_q2_rng *rng = &sc->sc_rng; 2146 struct ubsec_mcr *mcr; 2147 struct ubsec_ctx_rngbypass *ctx; 2148 int *nqueue; 2149 2150 /* Caller is responsible to lock and release sc_mtx. */ 2151 KASSERT(mutex_owned(&sc->sc_mtx)); 2152 2153 if (rng->rng_used) { 2154 return; 2155 } 2156 2157 if (sc->sc_rng_need < 1) { 2158 callout_stop(&sc->sc_rngto); 2159 return; 2160 } 2161 2162 if (sc->sc_flags & UBS_FLAGS_RNG4) 2163 nqueue = &sc->sc_nqueue4; 2164 else 2165 nqueue = &sc->sc_nqueue2; 2166 2167 (*nqueue)++; 2168 if (*nqueue >= UBS_MAX_NQUEUE) 2169 goto out; 2170 2171 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 2172 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 2173 2174 mcr->mcr_pkts = htole16(1); 2175 mcr->mcr_flags = 0; 2176 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 2177 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 2178 mcr->mcr_ipktbuf.pb_len = 0; 2179 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 2180 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 2181 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 2182 UBS_PKTBUF_LEN); 2183 mcr->mcr_opktbuf.pb_next = 0; 2184 2185 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 2186 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1); 2187 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1; 2188 2189 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2190 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2191 2192 if (sc->sc_flags & UBS_FLAGS_RNG4) { 2193 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next); 2194 ubsec_feed4(sc); 2195 } else { 2196 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 2197 ubsec_feed2(sc); 2198 } 2199 rng->rng_used = 1; 2200 ubsecstats.hst_rng++; 2201 2202 return; 2203 2204 out: 2205 /* 2206 * Something weird happened, generate our own call back. 2207 */ 2208 (*nqueue)--; 2209 #ifdef __OpenBSD__ 2210 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2211 #else 2212 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2213 #endif 2214 } 2215 #endif /* UBSEC_NO_RNG */ 2216 2217 static int 2218 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size, 2219 struct ubsec_dma_alloc *dma,int mapflags) 2220 { 2221 int r; 2222 2223 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, 2224 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0) 2225 goto fail_0; 2226 2227 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, 2228 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0) 2229 goto fail_1; 2230 2231 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2232 BUS_DMA_NOWAIT, &dma->dma_map)) != 0) 2233 goto fail_2; 2234 2235 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 2236 size, NULL, BUS_DMA_NOWAIT)) != 0) 2237 goto fail_3; 2238 2239 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2240 dma->dma_size = size; 2241 return (0); 2242 2243 fail_3: 2244 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2245 fail_2: 2246 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size); 2247 fail_1: 2248 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2249 fail_0: 2250 dma->dma_map = NULL; 2251 return (r); 2252 } 2253 2254 static void 2255 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 2256 { 2257 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 2258 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size); 2259 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2260 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2261 } 2262 2263 /* 2264 * Resets the board. Values in the regesters are left as is 2265 * from the reset (i.e. initial values are assigned elsewhere). 2266 */ 2267 static void 2268 ubsec_reset_board(struct ubsec_softc *sc) 2269 { 2270 volatile u_int32_t ctrl; 2271 2272 ctrl = READ_REG(sc, BS_CTRL); 2273 ctrl |= BS_CTRL_RESET; 2274 WRITE_REG(sc, BS_CTRL, ctrl); 2275 2276 /* 2277 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2278 */ 2279 DELAY(10); 2280 2281 /* Enable RNG and interrupts on newer devices */ 2282 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2283 #ifndef UBSEC_NO_RNG 2284 WRITE_REG(sc, BS_CFG, BS_CFG_RNG); 2285 #endif 2286 WRITE_REG(sc, BS_INT, BS_INT_DMAINT); 2287 } 2288 } 2289 2290 /* 2291 * Init Broadcom registers 2292 */ 2293 static void 2294 ubsec_init_board(struct ubsec_softc *sc) 2295 { 2296 u_int32_t ctrl; 2297 2298 ctrl = READ_REG(sc, BS_CTRL); 2299 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2300 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2301 2302 /* 2303 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)). 2304 * anyone got hw docs? 2305 */ 2306 if (sc->sc_flags & UBS_FLAGS_KEY) 2307 ctrl |= BS_CTRL_MCR2INT; 2308 else 2309 ctrl &= ~BS_CTRL_MCR2INT; 2310 2311 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2312 ctrl &= ~BS_CTRL_SWNORM; 2313 2314 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2315 ctrl |= BS_CTRL_BSIZE240; 2316 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */ 2317 2318 if (sc->sc_flags & UBS_FLAGS_RNG4) 2319 ctrl |= BS_CTRL_MCR4INT; 2320 else 2321 ctrl &= ~BS_CTRL_MCR4INT; 2322 } 2323 2324 WRITE_REG(sc, BS_CTRL, ctrl); 2325 } 2326 2327 /* 2328 * Init Broadcom PCI registers 2329 */ 2330 static void 2331 ubsec_init_pciregs(struct pci_attach_args *pa) 2332 { 2333 pci_chipset_tag_t pc = pa->pa_pc; 2334 u_int32_t misc; 2335 2336 /* 2337 * This will set the cache line size to 1, this will 2338 * force the BCM58xx chip just to do burst read/writes. 2339 * Cache line read/writes are to slow 2340 */ 2341 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG); 2342 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT)) 2343 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT); 2344 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc); 2345 } 2346 2347 /* 2348 * Clean up after a chip crash. 2349 * It is assumed that the caller in splnet() 2350 */ 2351 static void 2352 ubsec_cleanchip(struct ubsec_softc *sc) 2353 { 2354 struct ubsec_q *q; 2355 2356 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2357 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2358 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 2359 ubsec_free_q(sc, q); 2360 } 2361 sc->sc_nqchip = 0; 2362 } 2363 2364 /* 2365 * free a ubsec_q 2366 * It is assumed that the caller is within splnet() 2367 */ 2368 static int 2369 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2370 { 2371 struct ubsec_q *q2; 2372 struct cryptop *crp; 2373 int npkts; 2374 int i; 2375 2376 npkts = q->q_nstacked_mcrs; 2377 2378 for (i = 0; i < npkts; i++) { 2379 if(q->q_stacked_mcr[i]) { 2380 q2 = q->q_stacked_mcr[i]; 2381 2382 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2383 m_freem(q2->q_dst_m); 2384 2385 crp = (struct cryptop *)q2->q_crp; 2386 2387 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2388 2389 crp->crp_etype = EFAULT; 2390 crypto_done(crp); 2391 } else { 2392 break; 2393 } 2394 } 2395 2396 /* 2397 * Free header MCR 2398 */ 2399 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2400 m_freem(q->q_dst_m); 2401 2402 crp = (struct cryptop *)q->q_crp; 2403 2404 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2405 2406 crp->crp_etype = EFAULT; 2407 crypto_done(crp); 2408 return(0); 2409 } 2410 2411 /* 2412 * Routine to reset the chip and clean up. 2413 * It is assumed that the caller is in splnet() 2414 */ 2415 static void 2416 ubsec_totalreset(struct ubsec_softc *sc) 2417 { 2418 ubsec_reset_board(sc); 2419 ubsec_init_board(sc); 2420 ubsec_cleanchip(sc); 2421 } 2422 2423 static int 2424 ubsec_dmamap_aligned(bus_dmamap_t map) 2425 { 2426 int i; 2427 2428 for (i = 0; i < map->dm_nsegs; i++) { 2429 if (map->dm_segs[i].ds_addr & 3) 2430 return (0); 2431 if ((i != (map->dm_nsegs - 1)) && 2432 (map->dm_segs[i].ds_len & 3)) 2433 return (0); 2434 } 2435 return (1); 2436 } 2437 2438 #ifdef __OpenBSD__ 2439 struct ubsec_softc * 2440 ubsec_kfind(struct cryptkop *krp) 2441 { 2442 struct ubsec_softc *sc; 2443 int i; 2444 2445 for (i = 0; i < ubsec_cd.cd_ndevs; i++) { 2446 sc = ubsec_cd.cd_devs[i]; 2447 if (sc == NULL) 2448 continue; 2449 if (sc->sc_cid == krp->krp_hid) 2450 return (sc); 2451 } 2452 return (NULL); 2453 } 2454 #endif 2455 2456 static void 2457 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2458 { 2459 switch (q->q_type) { 2460 case UBS_CTXOP_MODEXP: { 2461 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2462 2463 ubsec_dma_free(sc, &me->me_q.q_mcr); 2464 ubsec_dma_free(sc, &me->me_q.q_ctx); 2465 ubsec_dma_free(sc, &me->me_M); 2466 ubsec_dma_free(sc, &me->me_E); 2467 ubsec_dma_free(sc, &me->me_C); 2468 ubsec_dma_free(sc, &me->me_epb); 2469 free(me, M_DEVBUF); 2470 break; 2471 } 2472 case UBS_CTXOP_RSAPRIV: { 2473 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2474 2475 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2476 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2477 ubsec_dma_free(sc, &rp->rpr_msgin); 2478 ubsec_dma_free(sc, &rp->rpr_msgout); 2479 free(rp, M_DEVBUF); 2480 break; 2481 } 2482 default: 2483 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev), 2484 q->q_type); 2485 break; 2486 } 2487 } 2488 2489 static int 2490 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2491 { 2492 struct ubsec_softc *sc; 2493 int r; 2494 2495 if (krp == NULL || krp->krp_callback == NULL) 2496 return (EINVAL); 2497 #ifdef __OpenBSD__ 2498 if ((sc = ubsec_kfind(krp)) == NULL) 2499 return (EINVAL); 2500 #else 2501 sc = arg; 2502 KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/); 2503 #endif 2504 2505 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2506 struct ubsec_q2 *q; 2507 2508 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2509 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next); 2510 ubsec_kfree(sc, q); 2511 } 2512 2513 switch (krp->krp_op) { 2514 case CRK_MOD_EXP: 2515 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2516 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2517 else 2518 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2519 break; 2520 case CRK_MOD_EXP_CRT: 2521 r = ubsec_kprocess_rsapriv(sc, krp, hint); 2522 break; 2523 default: 2524 printf("%s: kprocess: invalid op 0x%x\n", 2525 device_xname(sc->sc_dev), krp->krp_op); 2526 krp->krp_status = EOPNOTSUPP; 2527 crypto_kdone(krp); 2528 r = 0; 2529 } 2530 return (r); 2531 } 2532 2533 /* 2534 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2535 */ 2536 static int 2537 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, 2538 int hint) 2539 { 2540 struct ubsec_q2_modexp *me; 2541 struct ubsec_mcr *mcr; 2542 struct ubsec_ctx_modexp *ctx; 2543 struct ubsec_pktbuf *epb; 2544 int err = 0; 2545 u_int nbits, normbits, mbits, shiftbits, ebits; 2546 2547 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2548 if (me == NULL) { 2549 err = ENOMEM; 2550 goto errout; 2551 } 2552 memset(me, 0, sizeof *me); 2553 me->me_krp = krp; 2554 me->me_q.q_type = UBS_CTXOP_MODEXP; 2555 2556 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2557 if (nbits <= 512) 2558 normbits = 512; 2559 else if (nbits <= 768) 2560 normbits = 768; 2561 else if (nbits <= 1024) 2562 normbits = 1024; 2563 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2564 normbits = 1536; 2565 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2566 normbits = 2048; 2567 else { 2568 err = E2BIG; 2569 goto errout; 2570 } 2571 2572 shiftbits = normbits - nbits; 2573 2574 me->me_modbits = nbits; 2575 me->me_shiftbits = shiftbits; 2576 me->me_normbits = normbits; 2577 2578 /* Sanity check: result bits must be >= true modulus bits. */ 2579 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2580 err = ERANGE; 2581 goto errout; 2582 } 2583 2584 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2585 &me->me_q.q_mcr, 0)) { 2586 err = ENOMEM; 2587 goto errout; 2588 } 2589 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2590 2591 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2592 &me->me_q.q_ctx, 0)) { 2593 err = ENOMEM; 2594 goto errout; 2595 } 2596 2597 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2598 if (mbits > nbits) { 2599 err = E2BIG; 2600 goto errout; 2601 } 2602 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2603 err = ENOMEM; 2604 goto errout; 2605 } 2606 ubsec_kshift_r(shiftbits, 2607 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2608 me->me_M.dma_vaddr, normbits); 2609 2610 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2611 err = ENOMEM; 2612 goto errout; 2613 } 2614 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2615 2616 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2617 if (ebits > nbits) { 2618 err = E2BIG; 2619 goto errout; 2620 } 2621 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2622 err = ENOMEM; 2623 goto errout; 2624 } 2625 ubsec_kshift_r(shiftbits, 2626 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2627 me->me_E.dma_vaddr, normbits); 2628 2629 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2630 &me->me_epb, 0)) { 2631 err = ENOMEM; 2632 goto errout; 2633 } 2634 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2635 epb->pb_addr = htole32(me->me_E.dma_paddr); 2636 epb->pb_next = 0; 2637 epb->pb_len = htole32(normbits / 8); 2638 2639 #ifdef UBSEC_DEBUG 2640 if (ubsec_debug) { 2641 printf("Epb "); 2642 ubsec_dump_pb(epb); 2643 } 2644 #endif 2645 2646 mcr->mcr_pkts = htole16(1); 2647 mcr->mcr_flags = 0; 2648 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2649 mcr->mcr_reserved = 0; 2650 mcr->mcr_pktlen = 0; 2651 2652 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2653 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2654 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2655 2656 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2657 mcr->mcr_opktbuf.pb_next = 0; 2658 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2659 2660 #ifdef DIAGNOSTIC 2661 /* Misaligned output buffer will hang the chip. */ 2662 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2663 panic("%s: modexp invalid addr 0x%x", 2664 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr)); 2665 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2666 panic("%s: modexp invalid len 0x%x", 2667 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len)); 2668 #endif 2669 2670 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2671 memset(ctx, 0, sizeof(*ctx)); 2672 ubsec_kshift_r(shiftbits, 2673 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2674 ctx->me_N, normbits); 2675 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2676 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2677 ctx->me_E_len = htole16(nbits); 2678 ctx->me_N_len = htole16(nbits); 2679 2680 #ifdef UBSEC_DEBUG 2681 if (ubsec_debug) { 2682 ubsec_dump_mcr(mcr); 2683 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2684 } 2685 #endif 2686 2687 /* 2688 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2689 * everything else. 2690 */ 2691 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2692 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2693 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2694 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2695 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2696 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2697 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2698 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2699 2700 /* Enqueue and we're done... */ 2701 mutex_spin_enter(&sc->sc_mtx); 2702 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2703 ubsec_feed2(sc); 2704 ubsecstats.hst_modexp++; 2705 mutex_spin_exit(&sc->sc_mtx); 2706 2707 return (0); 2708 2709 errout: 2710 if (me != NULL) { 2711 if (me->me_q.q_mcr.dma_map != NULL) 2712 ubsec_dma_free(sc, &me->me_q.q_mcr); 2713 if (me->me_q.q_ctx.dma_map != NULL) { 2714 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2715 ubsec_dma_free(sc, &me->me_q.q_ctx); 2716 } 2717 if (me->me_M.dma_map != NULL) { 2718 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2719 ubsec_dma_free(sc, &me->me_M); 2720 } 2721 if (me->me_E.dma_map != NULL) { 2722 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2723 ubsec_dma_free(sc, &me->me_E); 2724 } 2725 if (me->me_C.dma_map != NULL) { 2726 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2727 ubsec_dma_free(sc, &me->me_C); 2728 } 2729 if (me->me_epb.dma_map != NULL) 2730 ubsec_dma_free(sc, &me->me_epb); 2731 free(me, M_DEVBUF); 2732 } 2733 krp->krp_status = err; 2734 crypto_kdone(krp); 2735 return (0); 2736 } 2737 2738 /* 2739 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2740 */ 2741 static int 2742 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, 2743 int hint) 2744 { 2745 struct ubsec_q2_modexp *me; 2746 struct ubsec_mcr *mcr; 2747 struct ubsec_ctx_modexp *ctx; 2748 struct ubsec_pktbuf *epb; 2749 int err = 0; 2750 u_int nbits, normbits, mbits, shiftbits, ebits; 2751 2752 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2753 if (me == NULL) { 2754 err = ENOMEM; 2755 goto errout; 2756 } 2757 memset(me, 0, sizeof *me); 2758 me->me_krp = krp; 2759 me->me_q.q_type = UBS_CTXOP_MODEXP; 2760 2761 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2762 if (nbits <= 512) 2763 normbits = 512; 2764 else if (nbits <= 768) 2765 normbits = 768; 2766 else if (nbits <= 1024) 2767 normbits = 1024; 2768 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2769 normbits = 1536; 2770 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2771 normbits = 2048; 2772 else { 2773 err = E2BIG; 2774 goto errout; 2775 } 2776 2777 shiftbits = normbits - nbits; 2778 2779 /* XXX ??? */ 2780 me->me_modbits = nbits; 2781 me->me_shiftbits = shiftbits; 2782 me->me_normbits = normbits; 2783 2784 /* Sanity check: result bits must be >= true modulus bits. */ 2785 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2786 err = ERANGE; 2787 goto errout; 2788 } 2789 2790 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2791 &me->me_q.q_mcr, 0)) { 2792 err = ENOMEM; 2793 goto errout; 2794 } 2795 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2796 2797 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2798 &me->me_q.q_ctx, 0)) { 2799 err = ENOMEM; 2800 goto errout; 2801 } 2802 2803 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2804 if (mbits > nbits) { 2805 err = E2BIG; 2806 goto errout; 2807 } 2808 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2809 err = ENOMEM; 2810 goto errout; 2811 } 2812 memset(me->me_M.dma_vaddr, 0, normbits / 8); 2813 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2814 me->me_M.dma_vaddr, (mbits + 7) / 8); 2815 2816 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2817 err = ENOMEM; 2818 goto errout; 2819 } 2820 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2821 2822 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2823 if (ebits > nbits) { 2824 err = E2BIG; 2825 goto errout; 2826 } 2827 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2828 err = ENOMEM; 2829 goto errout; 2830 } 2831 memset(me->me_E.dma_vaddr, 0, normbits / 8); 2832 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2833 me->me_E.dma_vaddr, (ebits + 7) / 8); 2834 2835 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2836 &me->me_epb, 0)) { 2837 err = ENOMEM; 2838 goto errout; 2839 } 2840 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2841 epb->pb_addr = htole32(me->me_E.dma_paddr); 2842 epb->pb_next = 0; 2843 epb->pb_len = htole32((ebits + 7) / 8); 2844 2845 #ifdef UBSEC_DEBUG 2846 if (ubsec_debug) { 2847 printf("Epb "); 2848 ubsec_dump_pb(epb); 2849 } 2850 #endif 2851 2852 mcr->mcr_pkts = htole16(1); 2853 mcr->mcr_flags = 0; 2854 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2855 mcr->mcr_reserved = 0; 2856 mcr->mcr_pktlen = 0; 2857 2858 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2859 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2860 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2861 2862 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2863 mcr->mcr_opktbuf.pb_next = 0; 2864 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2865 2866 #ifdef DIAGNOSTIC 2867 /* Misaligned output buffer will hang the chip. */ 2868 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2869 panic("%s: modexp invalid addr 0x%x", 2870 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_addr)); 2871 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2872 panic("%s: modexp invalid len 0x%x", 2873 device_xname(sc->sc_dev), letoh32(mcr->mcr_opktbuf.pb_len)); 2874 #endif 2875 2876 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2877 memset(ctx, 0, sizeof(*ctx)); 2878 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p, 2879 (nbits + 7) / 8); 2880 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2881 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2882 ctx->me_E_len = htole16(ebits); 2883 ctx->me_N_len = htole16(nbits); 2884 2885 #ifdef UBSEC_DEBUG 2886 if (ubsec_debug) { 2887 ubsec_dump_mcr(mcr); 2888 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2889 } 2890 #endif 2891 2892 /* 2893 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2894 * everything else. 2895 */ 2896 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2897 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2898 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2899 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2900 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2901 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2902 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2903 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2904 2905 /* Enqueue and we're done... */ 2906 mutex_spin_enter(&sc->sc_mtx); 2907 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2908 ubsec_feed2(sc); 2909 mutex_spin_exit(&sc->sc_mtx); 2910 2911 return (0); 2912 2913 errout: 2914 if (me != NULL) { 2915 if (me->me_q.q_mcr.dma_map != NULL) 2916 ubsec_dma_free(sc, &me->me_q.q_mcr); 2917 if (me->me_q.q_ctx.dma_map != NULL) { 2918 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2919 ubsec_dma_free(sc, &me->me_q.q_ctx); 2920 } 2921 if (me->me_M.dma_map != NULL) { 2922 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2923 ubsec_dma_free(sc, &me->me_M); 2924 } 2925 if (me->me_E.dma_map != NULL) { 2926 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2927 ubsec_dma_free(sc, &me->me_E); 2928 } 2929 if (me->me_C.dma_map != NULL) { 2930 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2931 ubsec_dma_free(sc, &me->me_C); 2932 } 2933 if (me->me_epb.dma_map != NULL) 2934 ubsec_dma_free(sc, &me->me_epb); 2935 free(me, M_DEVBUF); 2936 } 2937 krp->krp_status = err; 2938 crypto_kdone(krp); 2939 return (0); 2940 } 2941 2942 static int 2943 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, 2944 int hint) 2945 { 2946 struct ubsec_q2_rsapriv *rp = NULL; 2947 struct ubsec_mcr *mcr; 2948 struct ubsec_ctx_rsapriv *ctx; 2949 int err = 0; 2950 u_int padlen, msglen; 2951 2952 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2953 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2954 if (msglen > padlen) 2955 padlen = msglen; 2956 2957 if (padlen <= 256) 2958 padlen = 256; 2959 else if (padlen <= 384) 2960 padlen = 384; 2961 else if (padlen <= 512) 2962 padlen = 512; 2963 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2964 padlen = 768; 2965 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2966 padlen = 1024; 2967 else { 2968 err = E2BIG; 2969 goto errout; 2970 } 2971 2972 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2973 err = E2BIG; 2974 goto errout; 2975 } 2976 2977 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2978 err = E2BIG; 2979 goto errout; 2980 } 2981 2982 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2983 err = E2BIG; 2984 goto errout; 2985 } 2986 2987 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO); 2988 if (rp == NULL) 2989 return (ENOMEM); 2990 rp->rpr_krp = krp; 2991 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2992 2993 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2994 &rp->rpr_q.q_mcr, 0)) { 2995 err = ENOMEM; 2996 goto errout; 2997 } 2998 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2999 3000 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 3001 &rp->rpr_q.q_ctx, 0)) { 3002 err = ENOMEM; 3003 goto errout; 3004 } 3005 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 3006 memset(ctx, 0, sizeof *ctx); 3007 3008 /* Copy in p */ 3009 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 3010 &ctx->rpr_buf[0 * (padlen / 8)], 3011 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 3012 3013 /* Copy in q */ 3014 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 3015 &ctx->rpr_buf[1 * (padlen / 8)], 3016 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 3017 3018 /* Copy in dp */ 3019 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 3020 &ctx->rpr_buf[2 * (padlen / 8)], 3021 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 3022 3023 /* Copy in dq */ 3024 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 3025 &ctx->rpr_buf[3 * (padlen / 8)], 3026 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 3027 3028 /* Copy in pinv */ 3029 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 3030 &ctx->rpr_buf[4 * (padlen / 8)], 3031 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 3032 3033 msglen = padlen * 2; 3034 3035 /* Copy in input message (aligned buffer/length). */ 3036 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 3037 /* Is this likely? */ 3038 err = E2BIG; 3039 goto errout; 3040 } 3041 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 3042 err = ENOMEM; 3043 goto errout; 3044 } 3045 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8); 3046 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 3047 rp->rpr_msgin.dma_vaddr, 3048 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 3049 3050 /* Prepare space for output message (aligned buffer/length). */ 3051 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 3052 /* Is this likely? */ 3053 err = E2BIG; 3054 goto errout; 3055 } 3056 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 3057 err = ENOMEM; 3058 goto errout; 3059 } 3060 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8); 3061 3062 mcr->mcr_pkts = htole16(1); 3063 mcr->mcr_flags = 0; 3064 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 3065 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 3066 mcr->mcr_ipktbuf.pb_next = 0; 3067 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 3068 mcr->mcr_reserved = 0; 3069 mcr->mcr_pktlen = htole16(msglen); 3070 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 3071 mcr->mcr_opktbuf.pb_next = 0; 3072 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 3073 3074 #ifdef DIAGNOSTIC 3075 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 3076 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)", 3077 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr, 3078 (u_long) rp->rpr_msgin.dma_size); 3079 } 3080 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 3081 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)", 3082 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr, 3083 (u_long) rp->rpr_msgout.dma_size); 3084 } 3085 #endif 3086 3087 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 3088 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 3089 ctx->rpr_q_len = htole16(padlen); 3090 ctx->rpr_p_len = htole16(padlen); 3091 3092 /* 3093 * ubsec_feed2 will sync mcr and ctx, we just need to sync 3094 * everything else. 3095 */ 3096 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 3097 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 3098 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 3099 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 3100 3101 /* Enqueue and we're done... */ 3102 mutex_spin_enter(&sc->sc_mtx); 3103 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 3104 ubsec_feed2(sc); 3105 ubsecstats.hst_modexpcrt++; 3106 mutex_spin_exit(&sc->sc_mtx); 3107 return (0); 3108 3109 errout: 3110 if (rp != NULL) { 3111 if (rp->rpr_q.q_mcr.dma_map != NULL) 3112 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 3113 if (rp->rpr_msgin.dma_map != NULL) { 3114 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 3115 ubsec_dma_free(sc, &rp->rpr_msgin); 3116 } 3117 if (rp->rpr_msgout.dma_map != NULL) { 3118 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 3119 ubsec_dma_free(sc, &rp->rpr_msgout); 3120 } 3121 free(rp, M_DEVBUF); 3122 } 3123 krp->krp_status = err; 3124 crypto_kdone(krp); 3125 return (0); 3126 } 3127 3128 #ifdef UBSEC_DEBUG 3129 static void 3130 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 3131 { 3132 printf("addr 0x%x (0x%x) next 0x%x\n", 3133 pb->pb_addr, pb->pb_len, pb->pb_next); 3134 } 3135 3136 static void 3137 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c) 3138 { 3139 printf("CTX (0x%x):\n", c->ctx_len); 3140 switch (letoh16(c->ctx_op)) { 3141 case UBS_CTXOP_RNGBYPASS: 3142 case UBS_CTXOP_RNGSHA1: 3143 break; 3144 case UBS_CTXOP_MODEXP: 3145 { 3146 struct ubsec_ctx_modexp *cx = (void *)c; 3147 int i, len; 3148 3149 printf(" Elen %u, Nlen %u\n", 3150 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 3151 len = (cx->me_N_len + 7)/8; 3152 for (i = 0; i < len; i++) 3153 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 3154 printf("\n"); 3155 break; 3156 } 3157 default: 3158 printf("unknown context: %x\n", c->ctx_op); 3159 } 3160 printf("END CTX\n"); 3161 } 3162 3163 static void 3164 ubsec_dump_mcr(struct ubsec_mcr *mcr) 3165 { 3166 volatile struct ubsec_mcr_add *ma; 3167 int i; 3168 3169 printf("MCR:\n"); 3170 printf(" pkts: %u, flags 0x%x\n", 3171 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 3172 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 3173 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 3174 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 3175 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 3176 letoh16(ma->mcr_reserved)); 3177 printf(" %d: ipkt ", i); 3178 ubsec_dump_pb(&ma->mcr_ipktbuf); 3179 printf(" %d: opkt ", i); 3180 ubsec_dump_pb(&ma->mcr_opktbuf); 3181 ma++; 3182 } 3183 printf("END MCR\n"); 3184 } 3185 #endif /* UBSEC_DEBUG */ 3186 3187 /* 3188 * Return the number of significant bits of a big number. 3189 */ 3190 static int 3191 ubsec_ksigbits(struct crparam *cr) 3192 { 3193 u_int plen = (cr->crp_nbits + 7) / 8; 3194 int i, sig = plen * 8; 3195 u_int8_t c, *p = cr->crp_p; 3196 3197 for (i = plen - 1; i >= 0; i--) { 3198 c = p[i]; 3199 if (c != 0) { 3200 while ((c & 0x80) == 0) { 3201 sig--; 3202 c <<= 1; 3203 } 3204 break; 3205 } 3206 sig -= 8; 3207 } 3208 return (sig); 3209 } 3210 3211 static void 3212 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits, 3213 u_int8_t *dst, u_int dstbits) 3214 { 3215 u_int slen, dlen; 3216 int i, si, di, n; 3217 3218 slen = (srcbits + 7) / 8; 3219 dlen = (dstbits + 7) / 8; 3220 3221 for (i = 0; i < slen; i++) 3222 dst[i] = src[i]; 3223 for (i = 0; i < dlen - slen; i++) 3224 dst[slen + i] = 0; 3225 3226 n = shiftbits / 8; 3227 if (n != 0) { 3228 si = dlen - n - 1; 3229 di = dlen - 1; 3230 while (si >= 0) 3231 dst[di--] = dst[si--]; 3232 while (di >= 0) 3233 dst[di--] = 0; 3234 } 3235 3236 n = shiftbits % 8; 3237 if (n != 0) { 3238 for (i = dlen - 1; i > 0; i--) 3239 dst[i] = (dst[i] << n) | 3240 (dst[i - 1] >> (8 - n)); 3241 dst[0] = dst[0] << n; 3242 } 3243 } 3244 3245 static void 3246 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits, 3247 u_int8_t *dst, u_int dstbits) 3248 { 3249 int slen, dlen, i, n; 3250 3251 slen = (srcbits + 7) / 8; 3252 dlen = (dstbits + 7) / 8; 3253 3254 n = shiftbits / 8; 3255 for (i = 0; i < slen; i++) 3256 dst[i] = src[i + n]; 3257 for (i = 0; i < dlen - slen; i++) 3258 dst[slen + i] = 0; 3259 3260 n = shiftbits % 8; 3261 if (n != 0) { 3262 for (i = 0; i < (dlen - 1); i++) 3263 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 3264 dst[dlen - 1] = dst[dlen - 1] >> n; 3265 } 3266 } 3267
Indexes created Wed Oct 01 19:09:53 GMT 2025