ubsec.c revision 1.55
1 /* $NetBSD: ubsec.c,v 1.55 2022/05/22 11:30:58 riastradh Exp $ */ 2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Effort sponsored in part by the Defense Advanced Research Projects 32 * Agency (DARPA) and Air Force Research Laboratory, Air Force 33 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 34 * 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.55 2022/05/22 11:30:58 riastradh Exp $"); 39 40 #undef UBSEC_DEBUG 41 42 /* 43 * uBsec 5[56]01, 58xx hardware crypto accelerator 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/endian.h> 50 #include <sys/errno.h> 51 #include <sys/malloc.h> 52 #include <sys/kernel.h> 53 #include <sys/mbuf.h> 54 #include <sys/device.h> 55 #include <sys/module.h> 56 #include <sys/queue.h> 57 #include <sys/sysctl.h> 58 59 #include <opencrypto/cryptodev.h> 60 #include <opencrypto/xform.h> 61 #include <sys/cprng.h> 62 #include <sys/md5.h> 63 #include <sys/rndsource.h> 64 #include <sys/sha1.h> 65 66 #include <dev/pci/pcireg.h> 67 #include <dev/pci/pcivar.h> 68 #include <dev/pci/pcidevs.h> 69 70 #include <dev/pci/ubsecreg.h> 71 #include <dev/pci/ubsecvar.h> 72 73 #define UBSEC_NO_RNG /* hangs on attach */ 74 #define letoh16 htole16 75 #define letoh32 htole32 76 77 /* 78 * Prototypes and count for the pci_device structure 79 */ 80 static int ubsec_probe(device_t, cfdata_t, void *); 81 static void ubsec_attach(device_t, device_t, void *); 82 static int ubsec_detach(device_t, int); 83 static void ubsec_reset_board(struct ubsec_softc *); 84 static void ubsec_init_board(struct ubsec_softc *); 85 static void ubsec_init_pciregs(struct pci_attach_args *pa); 86 static void ubsec_cleanchip(struct ubsec_softc *); 87 static void ubsec_totalreset(struct ubsec_softc *); 88 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *); 89 90 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 91 ubsec_detach, NULL); 92 extern struct cfdriver ubsec_cd; 93 94 /* patchable */ 95 #ifdef UBSEC_DEBUG 96 extern int ubsec_debug; 97 int ubsec_debug=1; 98 #endif 99 100 static int ubsec_intr(void *); 101 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *); 102 static int ubsec_freesession(void*, u_int64_t); 103 static int ubsec_process(void*, struct cryptop *, int hint); 104 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 105 static void ubsec_feed(struct ubsec_softc *); 106 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 107 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 108 static void ubsec_feed2(struct ubsec_softc *); 109 static void ubsec_feed4(struct ubsec_softc *); 110 #ifndef UBSEC_NO_RNG 111 static void ubsec_rng(void *); 112 static void ubsec_rng_locked(void *); 113 static void ubsec_rng_get(size_t, void *); 114 #endif /* UBSEC_NO_RNG */ 115 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 116 struct ubsec_dma_alloc *, int); 117 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 118 static int ubsec_dmamap_aligned(bus_dmamap_t); 119 120 static int ubsec_kprocess(void*, struct cryptkop *, int); 121 static void ubsec_kprocess_modexp_sw(struct ubsec_softc *, 122 struct cryptkop *, int); 123 static void ubsec_kprocess_modexp_hw(struct ubsec_softc *, 124 struct cryptkop *, int); 125 static void ubsec_kprocess_rsapriv(struct ubsec_softc *, 126 struct cryptkop *, int); 127 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 128 static int ubsec_ksigbits(struct crparam *); 129 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 130 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 131 132 #ifdef UBSEC_DEBUG 133 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 134 static void ubsec_dump_mcr(struct ubsec_mcr *); 135 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *); 136 #endif 137 138 #define READ_REG(sc,r) \ 139 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 140 141 #define WRITE_REG(sc,reg,val) \ 142 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 143 144 #define SWAP32(x) (x) = htole32(ntohl((x))) 145 #ifndef HTOLE32 146 #define HTOLE32(x) (x) = htole32(x) 147 #endif 148 149 struct ubsec_stats ubsecstats; 150 151 /* 152 * ubsec_maxbatch controls the number of crypto ops to voluntarily 153 * collect into one submission to the hardware. This batching happens 154 * when ops are dispatched from the crypto subsystem with a hint that 155 * more are to follow immediately. These ops must also not be marked 156 * with a ``no delay'' flag. 157 */ 158 static int ubsec_maxbatch = 1; 159 160 /* 161 * ubsec_maxaggr controls the number of crypto ops to submit to the 162 * hardware as a unit. This aggregation reduces the number of interrupts 163 * to the host at the expense of increased latency (for all but the last 164 * operation). For network traffic setting this to one yields the highest 165 * performance but at the expense of more interrupt processing. 166 */ 167 static int ubsec_maxaggr = 1; 168 169 static const struct ubsec_product { 170 pci_vendor_id_t ubsec_vendor; 171 pci_product_id_t ubsec_product; 172 int ubsec_flags; 173 int ubsec_statmask; 174 int ubsec_maxaggr; 175 const char *ubsec_name; 176 } ubsec_products[] = { 177 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501, 178 0, 179 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 180 UBS_MIN_AGGR, 181 "Bluesteel 5501" 182 }, 183 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601, 184 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 185 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 186 UBS_MIN_AGGR, 187 "Bluesteel 5601" 188 }, 189 190 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801, 191 0, 192 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 193 UBS_MIN_AGGR, 194 "Broadcom BCM5801" 195 }, 196 197 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802, 198 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 199 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 200 UBS_MIN_AGGR, 201 "Broadcom BCM5802" 202 }, 203 204 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805, 205 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 206 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 207 UBS_MIN_AGGR, 208 "Broadcom BCM5805" 209 }, 210 211 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820, 212 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 213 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 214 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 215 UBS_MIN_AGGR, 216 "Broadcom BCM5820" 217 }, 218 219 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821, 220 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 221 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 222 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 223 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 224 UBS_MIN_AGGR, 225 "Broadcom BCM5821" 226 }, 227 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K, 228 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 229 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 230 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 231 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 232 UBS_MIN_AGGR, 233 "Sun Crypto Accelerator 1000" 234 }, 235 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821, 236 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 237 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 238 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 239 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 240 UBS_MIN_AGGR, 241 "Broadcom BCM5821 (Sun)" 242 }, 243 244 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822, 245 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 246 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 247 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 248 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 249 UBS_MIN_AGGR, 250 "Broadcom BCM5822" 251 }, 252 253 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823, 254 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 255 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 256 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 257 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 258 UBS_MIN_AGGR, 259 "Broadcom BCM5823" 260 }, 261 262 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825, 263 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 264 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 265 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 266 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 267 UBS_MIN_AGGR, 268 "Broadcom BCM5825" 269 }, 270 271 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860, 272 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 273 UBS_FLAGS_LONGCTX | 274 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 275 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 276 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 277 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 278 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 279 UBS_MAX_AGGR, 280 "Broadcom BCM5860" 281 }, 282 283 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861, 284 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 285 UBS_FLAGS_LONGCTX | 286 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 287 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 288 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 289 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 290 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 291 UBS_MAX_AGGR, 292 "Broadcom BCM5861" 293 }, 294 295 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862, 296 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 297 UBS_FLAGS_LONGCTX | 298 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 299 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 300 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 301 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 302 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 303 UBS_MAX_AGGR, 304 "Broadcom BCM5862" 305 }, 306 307 { 0, 0, 308 0, 309 0, 310 0, 311 NULL 312 } 313 }; 314 315 static const struct ubsec_product * 316 ubsec_lookup(const struct pci_attach_args *pa) 317 { 318 const struct ubsec_product *up; 319 320 for (up = ubsec_products; up->ubsec_name != NULL; up++) { 321 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor && 322 PCI_PRODUCT(pa->pa_id) == up->ubsec_product) 323 return (up); 324 } 325 return (NULL); 326 } 327 328 static int 329 ubsec_probe(device_t parent, cfdata_t match, void *aux) 330 { 331 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 332 333 if (ubsec_lookup(pa) != NULL) 334 return (1); 335 336 return (0); 337 } 338 339 static void 340 ubsec_attach(device_t parent, device_t self, void *aux) 341 { 342 struct ubsec_softc *sc = device_private(self); 343 struct pci_attach_args *pa = aux; 344 const struct ubsec_product *up; 345 pci_chipset_tag_t pc = pa->pa_pc; 346 pci_intr_handle_t ih; 347 const char *intrstr = NULL; 348 pcireg_t memtype; 349 struct ubsec_dma *dmap; 350 u_int32_t cmd, i; 351 char intrbuf[PCI_INTRSTR_LEN]; 352 353 sc->sc_dev = self; 354 sc->sc_pct = pc; 355 356 up = ubsec_lookup(pa); 357 if (up == NULL) { 358 printf("\n"); 359 panic("ubsec_attach: impossible"); 360 } 361 362 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1); 363 364 SIMPLEQ_INIT(&sc->sc_queue); 365 SIMPLEQ_INIT(&sc->sc_qchip); 366 SIMPLEQ_INIT(&sc->sc_queue2); 367 SIMPLEQ_INIT(&sc->sc_qchip2); 368 SIMPLEQ_INIT(&sc->sc_queue4); 369 SIMPLEQ_INIT(&sc->sc_qchip4); 370 SIMPLEQ_INIT(&sc->sc_q2free); 371 372 sc->sc_flags = up->ubsec_flags; 373 sc->sc_statmask = up->ubsec_statmask; 374 sc->sc_maxaggr = up->ubsec_maxaggr; 375 376 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 377 cmd |= PCI_COMMAND_MASTER_ENABLE; 378 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 379 380 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR); 381 if (pci_mapreg_map(pa, BS_BAR, memtype, 0, 382 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) { 383 aprint_error_dev(self, "can't find mem space"); 384 return; 385 } 386 387 sc->sc_dmat = pa->pa_dmat; 388 389 if (pci_intr_map(pa, &ih)) { 390 aprint_error_dev(self, "couldn't map interrupt\n"); 391 return; 392 } 393 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 394 sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, ubsec_intr, sc, 395 device_xname(self)); 396 if (sc->sc_ih == NULL) { 397 aprint_error_dev(self, "couldn't establish interrupt"); 398 if (intrstr != NULL) 399 aprint_error(" at %s", intrstr); 400 aprint_error("\n"); 401 return; 402 } 403 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 404 405 sc->sc_cid = crypto_get_driverid(0); 406 if (sc->sc_cid < 0) { 407 aprint_error_dev(self, "couldn't get crypto driver id\n"); 408 pci_intr_disestablish(pc, sc->sc_ih); 409 return; 410 } 411 412 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 413 414 SIMPLEQ_INIT(&sc->sc_freequeue); 415 dmap = sc->sc_dmaa; 416 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 417 struct ubsec_q *q; 418 419 q = malloc(sizeof(struct ubsec_q), M_DEVBUF, M_ZERO|M_WAITOK); 420 421 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 422 &dmap->d_alloc, 0)) { 423 aprint_error_dev(self, "can't allocate dma buffers\n"); 424 free(q, M_DEVBUF); 425 break; 426 } 427 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 428 429 q->q_dma = dmap; 430 sc->sc_queuea[i] = q; 431 432 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 433 } 434 435 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 436 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 437 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 438 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 439 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 440 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 441 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 442 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 443 if (sc->sc_flags & UBS_FLAGS_AES) { 444 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 445 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 446 } 447 448 /* 449 * Reset Broadcom chip 450 */ 451 ubsec_reset_board(sc); 452 453 /* 454 * Init Broadcom specific PCI settings 455 */ 456 ubsec_init_pciregs(pa); 457 458 /* 459 * Init Broadcom chip 460 */ 461 ubsec_init_board(sc); 462 463 #ifndef UBSEC_NO_RNG 464 if (sc->sc_flags & UBS_FLAGS_RNG) { 465 if (sc->sc_flags & UBS_FLAGS_RNG4) 466 sc->sc_statmask |= BS_STAT_MCR4_DONE; 467 else 468 sc->sc_statmask |= BS_STAT_MCR2_DONE; 469 470 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 471 &sc->sc_rng.rng_q.q_mcr, 0)) 472 goto skip_rng; 473 474 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 475 &sc->sc_rng.rng_q.q_ctx, 0)) { 476 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 477 goto skip_rng; 478 } 479 480 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 481 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 482 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 483 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 484 goto skip_rng; 485 } 486 if (hz >= 100) 487 sc->sc_rnghz = hz / 100; 488 else 489 sc->sc_rnghz = 1; 490 callout_init(&sc->sc_rngto, 0); 491 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc); 492 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc); 493 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev), 494 RND_TYPE_RNG, 495 RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB); 496 497 skip_rng: 498 if (sc->sc_rnghz) 499 aprint_normal_dev(self, 500 "random number generator enabled\n"); 501 else 502 aprint_error_dev(self, 503 "WARNING: random number generator disabled\n"); 504 } 505 #endif /* UBSEC_NO_RNG */ 506 507 if (sc->sc_flags & UBS_FLAGS_KEY) { 508 sc->sc_statmask |= BS_STAT_MCR2_DONE; 509 510 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 511 ubsec_kprocess, sc); 512 #if 0 513 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 514 ubsec_kprocess, sc); 515 #endif 516 } 517 } 518 519 static int 520 ubsec_detach(device_t self, int flags) 521 { 522 struct ubsec_softc *sc = device_private(self); 523 struct ubsec_q *q, *qtmp; 524 volatile u_int32_t ctrl; 525 526 /* disable interrupts */ 527 /* XXX wait/abort current ops? where is DMAERR enabled? */ 528 ctrl = READ_REG(sc, BS_CTRL); 529 530 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR); 531 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 532 ctrl &= ~BS_CTRL_MCR4INT; 533 534 WRITE_REG(sc, BS_CTRL, ctrl); 535 536 #ifndef UBSEC_NO_RNG 537 if (sc->sc_flags & UBS_FLAGS_RNG) { 538 callout_halt(&sc->sc_rngto, NULL); 539 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 540 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 541 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 542 rnd_detach_source(&sc->sc_rnd_source); 543 } 544 #endif /* UBSEC_NO_RNG */ 545 546 crypto_unregister_all(sc->sc_cid); 547 548 mutex_spin_enter(&sc->sc_mtx); 549 550 ubsec_totalreset(sc); /* XXX leaves the chip running */ 551 552 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) { 553 ubsec_dma_free(sc, &q->q_dma->d_alloc); 554 if (q->q_src_map != NULL) 555 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 556 if (q->q_cached_dst_map != NULL) 557 bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map); 558 free(q, M_DEVBUF); 559 } 560 561 mutex_spin_exit(&sc->sc_mtx); 562 563 if (sc->sc_ih != NULL) { 564 pci_intr_disestablish(sc->sc_pct, sc->sc_ih); 565 sc->sc_ih = NULL; 566 } 567 568 if (sc->sc_memsize != 0) { 569 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize); 570 sc->sc_memsize = 0; 571 } 572 573 return 0; 574 } 575 576 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto"); 577 578 #ifdef _MODULE 579 #include "ioconf.c" 580 #endif 581 582 static int 583 ubsec_modcmd(modcmd_t cmd, void *data) 584 { 585 int error = 0; 586 587 switch (cmd) { 588 case MODULE_CMD_INIT: 589 #ifdef _MODULE 590 error = config_init_component(cfdriver_ioconf_ubsec, 591 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 592 #endif 593 return error; 594 case MODULE_CMD_FINI: 595 #ifdef _MODULE 596 error = config_fini_component(cfdriver_ioconf_ubsec, 597 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 598 #endif 599 return error; 600 default: 601 return ENOTTY; 602 } 603 } 604 605 SYSCTL_SETUP(ubsec_sysctl_init, "ubsec sysctl") 606 { 607 const struct sysctlnode *node = NULL; 608 609 sysctl_createv(clog, 0, NULL, &node, 610 CTLFLAG_PERMANENT, 611 CTLTYPE_NODE, "ubsec", 612 SYSCTL_DESCR("ubsec opetions"), 613 NULL, 0, NULL, 0, 614 CTL_HW, CTL_CREATE, CTL_EOL); 615 sysctl_createv(clog, 0, &node, NULL, 616 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 617 CTLTYPE_INT, "maxbatch", 618 SYSCTL_DESCR("max ops to batch w/o interrupt"), 619 NULL, 0, &ubsec_maxbatch, 0, 620 CTL_CREATE, CTL_EOL); 621 sysctl_createv(clog, 0, &node, NULL, 622 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 623 CTLTYPE_INT, "maxaggr", 624 SYSCTL_DESCR("max ops to aggregate under one interrupt"), 625 NULL, 0, &ubsec_maxaggr, 0, 626 CTL_CREATE, CTL_EOL); 627 628 return; 629 } 630 631 /* 632 * UBSEC Interrupt routine 633 */ 634 static int 635 ubsec_intr(void *arg) 636 { 637 struct ubsec_softc *sc = arg; 638 volatile u_int32_t stat; 639 struct ubsec_q *q; 640 struct ubsec_dma *dmap; 641 int flags; 642 int npkts = 0, i; 643 644 mutex_spin_enter(&sc->sc_mtx); 645 stat = READ_REG(sc, BS_STAT); 646 stat &= sc->sc_statmask; 647 if (stat == 0) { 648 mutex_spin_exit(&sc->sc_mtx); 649 return (0); 650 } 651 652 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 653 654 /* 655 * Check to see if we have any packets waiting for us 656 */ 657 if ((stat & BS_STAT_MCR1_DONE)) { 658 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 659 q = SIMPLEQ_FIRST(&sc->sc_qchip); 660 dmap = q->q_dma; 661 662 if ((dmap->d_dma->d_mcr.mcr_flags 663 & htole16(UBS_MCR_DONE)) == 0) 664 break; 665 666 q = SIMPLEQ_FIRST(&sc->sc_qchip); 667 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 668 669 npkts = q->q_nstacked_mcrs; 670 sc->sc_nqchip -= 1+npkts; 671 /* 672 * search for further sc_qchip ubsec_q's that share 673 * the same MCR, and complete them too, they must be 674 * at the top. 675 */ 676 for (i = 0; i < npkts; i++) { 677 if(q->q_stacked_mcr[i]) 678 ubsec_callback(sc, q->q_stacked_mcr[i]); 679 else 680 break; 681 } 682 ubsec_callback(sc, q); 683 } 684 685 /* 686 * Don't send any more packet to chip if there has been 687 * a DMAERR. 688 */ 689 if (!(stat & BS_STAT_DMAERR)) 690 ubsec_feed(sc); 691 } 692 693 /* 694 * Check to see if we have any key setups/rng's waiting for us 695 */ 696 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 697 (stat & BS_STAT_MCR2_DONE)) { 698 struct ubsec_q2 *q2; 699 struct ubsec_mcr *mcr; 700 701 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 702 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 703 704 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 705 0, q2->q_mcr.dma_map->dm_mapsize, 706 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 707 708 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 709 710 /* A bug in new devices requires to swap this field */ 711 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 712 flags = htole16(mcr->mcr_flags); 713 else 714 flags = mcr->mcr_flags; 715 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 716 bus_dmamap_sync(sc->sc_dmat, 717 q2->q_mcr.dma_map, 0, 718 q2->q_mcr.dma_map->dm_mapsize, 719 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 720 break; 721 } 722 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 723 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next); 724 ubsec_callback2(sc, q2); 725 /* 726 * Don't send any more packet to chip if there has been 727 * a DMAERR. 728 */ 729 if (!(stat & BS_STAT_DMAERR)) 730 ubsec_feed2(sc); 731 } 732 } 733 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) { 734 struct ubsec_q2 *q2; 735 struct ubsec_mcr *mcr; 736 737 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) { 738 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4); 739 740 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 741 0, q2->q_mcr.dma_map->dm_mapsize, 742 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 743 744 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 745 746 /* A bug in new devices requires to swap this field */ 747 flags = htole16(mcr->mcr_flags); 748 749 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 750 bus_dmamap_sync(sc->sc_dmat, 751 q2->q_mcr.dma_map, 0, 752 q2->q_mcr.dma_map->dm_mapsize, 753 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 754 break; 755 } 756 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next); 757 ubsec_callback2(sc, q2); 758 /* 759 * Don't send any more packet to chip if there has been 760 * a DMAERR. 761 */ 762 if (!(stat & BS_STAT_DMAERR)) 763 ubsec_feed4(sc); 764 } 765 } 766 767 /* 768 * Check to see if we got any DMA Error 769 */ 770 if (stat & BS_STAT_DMAERR) { 771 #ifdef UBSEC_DEBUG 772 if (ubsec_debug) { 773 volatile u_int32_t a = READ_REG(sc, BS_ERR); 774 775 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev), 776 (a & BS_ERR_READ) ? "read" : "write", 777 a & BS_ERR_ADDR); 778 } 779 #endif /* UBSEC_DEBUG */ 780 ubsecstats.hst_dmaerr++; 781 ubsec_totalreset(sc); 782 ubsec_feed(sc); 783 } 784 785 if (sc->sc_needwakeup) { /* XXX check high watermark */ 786 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 787 #ifdef UBSEC_DEBUG 788 if (ubsec_debug) 789 printf("%s: wakeup crypto (%x)\n", 790 device_xname(sc->sc_dev), sc->sc_needwakeup); 791 #endif /* UBSEC_DEBUG */ 792 sc->sc_needwakeup &= ~wkeup; 793 crypto_unblock(sc->sc_cid, wkeup); 794 } 795 mutex_spin_exit(&sc->sc_mtx); 796 return (1); 797 } 798 799 /* 800 * ubsec_feed() - aggregate and post requests to chip 801 * OpenBSD comments: 802 * It is assumed that the caller set splnet() 803 */ 804 static void 805 ubsec_feed(struct ubsec_softc *sc) 806 { 807 struct ubsec_q *q, *q2; 808 int npkts, i; 809 void *v; 810 u_int32_t stat; 811 #ifdef UBSEC_DEBUG 812 static int max; 813 #endif /* UBSEC_DEBUG */ 814 815 npkts = sc->sc_nqueue; 816 if (npkts > ubsecstats.hst_maxqueue) 817 ubsecstats.hst_maxqueue = npkts; 818 if (npkts < 2) 819 goto feed1; 820 821 /* 822 * Decide how many ops to combine in a single MCR. We cannot 823 * aggregate more than UBS_MAX_AGGR because this is the number 824 * of slots defined in the data structure. Otherwise we clamp 825 * based on the tunable parameter ubsec_maxaggr. Note that 826 * aggregation can happen in two ways: either by batching ops 827 * from above or because the h/w backs up and throttles us. 828 * Aggregating ops reduces the number of interrupts to the host 829 * but also (potentially) increases the latency for processing 830 * completed ops as we only get an interrupt when all aggregated 831 * ops have completed. 832 */ 833 if (npkts > sc->sc_maxaggr) 834 npkts = sc->sc_maxaggr; 835 if (npkts > ubsec_maxaggr) 836 npkts = ubsec_maxaggr; 837 if (npkts > ubsecstats.hst_maxbatch) 838 ubsecstats.hst_maxbatch = npkts; 839 if (npkts < 2) 840 goto feed1; 841 ubsecstats.hst_totbatch += npkts-1; 842 843 if ((stat = READ_REG(sc, BS_STAT)) 844 & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 845 if (stat & BS_STAT_DMAERR) { 846 ubsec_totalreset(sc); 847 ubsecstats.hst_dmaerr++; 848 } else { 849 ubsecstats.hst_mcr1full++; 850 } 851 return; 852 } 853 854 #ifdef UBSEC_DEBUG 855 if (ubsec_debug) 856 printf("merging %d records\n", npkts); 857 /* XXX temporary aggregation statistics reporting code */ 858 if (max < npkts) { 859 max = npkts; 860 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), 861 max); 862 } 863 #endif /* UBSEC_DEBUG */ 864 865 q = SIMPLEQ_FIRST(&sc->sc_queue); 866 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 867 --sc->sc_nqueue; 868 869 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 870 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 871 if (q->q_dst_map != NULL) 872 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 873 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 874 875 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 876 877 for (i = 0; i < q->q_nstacked_mcrs; i++) { 878 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 879 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 880 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 881 if (q2->q_dst_map != NULL) 882 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 883 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 884 q2= SIMPLEQ_FIRST(&sc->sc_queue); 885 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next); 886 --sc->sc_nqueue; 887 888 v = ((void *)&q2->q_dma->d_dma->d_mcr); 889 v = (char*)v + (sizeof(struct ubsec_mcr) - 890 sizeof(struct ubsec_mcr_add)); 891 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, 892 sizeof(struct ubsec_mcr_add)); 893 q->q_stacked_mcr[i] = q2; 894 } 895 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 896 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 897 sc->sc_nqchip += npkts; 898 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 899 ubsecstats.hst_maxqchip = sc->sc_nqchip; 900 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 901 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 902 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 903 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 904 offsetof(struct ubsec_dmachunk, d_mcr)); 905 return; 906 907 feed1: 908 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 909 if ((stat = READ_REG(sc, BS_STAT)) 910 & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 911 if (stat & BS_STAT_DMAERR) { 912 ubsec_totalreset(sc); 913 ubsecstats.hst_dmaerr++; 914 } else { 915 ubsecstats.hst_mcr1full++; 916 } 917 break; 918 } 919 920 q = SIMPLEQ_FIRST(&sc->sc_queue); 921 922 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 923 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 924 if (q->q_dst_map != NULL) 925 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 926 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 927 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 928 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 929 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 930 931 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 932 offsetof(struct ubsec_dmachunk, d_mcr)); 933 #ifdef UBSEC_DEBUG 934 if (ubsec_debug) 935 printf("feed: q->chip %p %08x stat %08x\n", 936 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr, 937 stat); 938 #endif /* UBSEC_DEBUG */ 939 q = SIMPLEQ_FIRST(&sc->sc_queue); 940 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 941 --sc->sc_nqueue; 942 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 943 sc->sc_nqchip++; 944 } 945 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 946 ubsecstats.hst_maxqchip = sc->sc_nqchip; 947 } 948 949 /* 950 * Allocate a new 'session' and return an encoded session id. 'sidp' 951 * contains our registration id, and should contain an encoded session 952 * id on successful allocation. 953 */ 954 static int 955 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 956 { 957 struct cryptoini *c, *encini = NULL, *macini = NULL; 958 struct ubsec_softc *sc; 959 struct ubsec_session *ses = NULL; 960 MD5_CTX md5ctx; 961 SHA1_CTX sha1ctx; 962 int i, sesn; 963 964 sc = arg; 965 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/); 966 967 if (sidp == NULL || cri == NULL || sc == NULL) 968 return (EINVAL); 969 970 for (c = cri; c != NULL; c = c->cri_next) { 971 if (c->cri_alg == CRYPTO_MD5_HMAC_96 || 972 c->cri_alg == CRYPTO_SHA1_HMAC_96) { 973 if (macini) 974 return (EINVAL); 975 macini = c; 976 } else if (c->cri_alg == CRYPTO_DES_CBC || 977 c->cri_alg == CRYPTO_3DES_CBC || 978 c->cri_alg == CRYPTO_AES_CBC) { 979 if (encini) 980 return (EINVAL); 981 encini = c; 982 } else 983 return (EINVAL); 984 } 985 if (encini == NULL && macini == NULL) 986 return (EINVAL); 987 988 if (encini && encini->cri_alg == CRYPTO_AES_CBC) { 989 switch (encini->cri_klen) { 990 case 128: 991 case 192: 992 case 256: 993 break; 994 default: 995 return (EINVAL); 996 } 997 } 998 999 if (sc->sc_sessions == NULL) { 1000 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 1001 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1002 if (ses == NULL) 1003 return (ENOMEM); 1004 sesn = 0; 1005 sc->sc_nsessions = 1; 1006 } else { 1007 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 1008 if (sc->sc_sessions[sesn].ses_used == 0) { 1009 ses = &sc->sc_sessions[sesn]; 1010 break; 1011 } 1012 } 1013 1014 if (ses == NULL) { 1015 sesn = sc->sc_nsessions; 1016 ses = (struct ubsec_session *)malloc((sesn + 1) * 1017 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1018 if (ses == NULL) 1019 return (ENOMEM); 1020 memcpy(ses, sc->sc_sessions, sesn * 1021 sizeof(struct ubsec_session)); 1022 memset(sc->sc_sessions, 0, sesn * 1023 sizeof(struct ubsec_session)); 1024 free(sc->sc_sessions, M_DEVBUF); 1025 sc->sc_sessions = ses; 1026 ses = &sc->sc_sessions[sesn]; 1027 sc->sc_nsessions++; 1028 } 1029 } 1030 1031 memset(ses, 0, sizeof(struct ubsec_session)); 1032 ses->ses_used = 1; 1033 if (encini) { 1034 /* Go ahead and compute key in ubsec's byte order */ 1035 if (encini->cri_alg == CRYPTO_AES_CBC) { 1036 memcpy(ses->ses_key, encini->cri_key, 1037 encini->cri_klen / 8); 1038 } 1039 if (encini->cri_alg == CRYPTO_DES_CBC) { 1040 memcpy(&ses->ses_key[0], encini->cri_key, 8); 1041 memcpy(&ses->ses_key[2], encini->cri_key, 8); 1042 memcpy(&ses->ses_key[4], encini->cri_key, 8); 1043 } else 1044 memcpy(ses->ses_key, encini->cri_key, 24); 1045 1046 SWAP32(ses->ses_key[0]); 1047 SWAP32(ses->ses_key[1]); 1048 SWAP32(ses->ses_key[2]); 1049 SWAP32(ses->ses_key[3]); 1050 SWAP32(ses->ses_key[4]); 1051 SWAP32(ses->ses_key[5]); 1052 } 1053 1054 if (macini) { 1055 for (i = 0; i < macini->cri_klen / 8; i++) 1056 macini->cri_key[i] ^= HMAC_IPAD_VAL; 1057 1058 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1059 MD5Init(&md5ctx); 1060 MD5Update(&md5ctx, macini->cri_key, 1061 macini->cri_klen / 8); 1062 MD5Update(&md5ctx, hmac_ipad_buffer, 1063 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1064 memcpy(ses->ses_hminner, md5ctx.state, 1065 sizeof(md5ctx.state)); 1066 } else { 1067 SHA1Init(&sha1ctx); 1068 SHA1Update(&sha1ctx, macini->cri_key, 1069 macini->cri_klen / 8); 1070 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1071 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1072 memcpy(ses->ses_hminner, sha1ctx.state, 1073 sizeof(sha1ctx.state)); 1074 } 1075 1076 for (i = 0; i < macini->cri_klen / 8; i++) 1077 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1078 1079 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1080 MD5Init(&md5ctx); 1081 MD5Update(&md5ctx, macini->cri_key, 1082 macini->cri_klen / 8); 1083 MD5Update(&md5ctx, hmac_opad_buffer, 1084 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1085 memcpy(ses->ses_hmouter, md5ctx.state, 1086 sizeof(md5ctx.state)); 1087 } else { 1088 SHA1Init(&sha1ctx); 1089 SHA1Update(&sha1ctx, macini->cri_key, 1090 macini->cri_klen / 8); 1091 SHA1Update(&sha1ctx, hmac_opad_buffer, 1092 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1093 memcpy(ses->ses_hmouter, sha1ctx.state, 1094 sizeof(sha1ctx.state)); 1095 } 1096 1097 for (i = 0; i < macini->cri_klen / 8; i++) 1098 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1099 } 1100 1101 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn); 1102 return (0); 1103 } 1104 1105 /* 1106 * Deallocate a session. 1107 */ 1108 static int 1109 ubsec_freesession(void *arg, u_int64_t tid) 1110 { 1111 struct ubsec_softc *sc; 1112 int session; 1113 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1114 1115 sc = arg; 1116 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/); 1117 1118 session = UBSEC_SESSION(sid); 1119 if (session >= sc->sc_nsessions) 1120 return (EINVAL); 1121 1122 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 1123 return (0); 1124 } 1125 1126 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */ 1127 static void 1128 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, 1129 int error) 1130 { 1131 struct ubsec_operand *op = arg; 1132 1133 KASSERT(nsegs <= UBS_MAX_SCATTER 1134 /*, ("Too many DMA segments returned when mapping operand")*/); 1135 #ifdef UBSEC_DEBUG 1136 if (ubsec_debug) 1137 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 1138 (u_int) mapsize, nsegs); 1139 #endif 1140 op->mapsize = mapsize; 1141 op->nsegs = nsegs; 1142 memcpy(op->segs, seg, nsegs * sizeof (seg[0])); 1143 } 1144 #endif 1145 1146 static int 1147 ubsec_process(void *arg, struct cryptop *crp, int hint) 1148 { 1149 struct ubsec_q *q = NULL; 1150 int err = 0, i, j, nicealign; 1151 struct ubsec_softc *sc = arg; 1152 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1153 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1154 int sskip, dskip, stheend, dtheend; 1155 int16_t coffset; 1156 struct ubsec_session *ses, key; 1157 struct ubsec_dma *dmap = NULL; 1158 u_int16_t flags = 0; 1159 int ivlen = 0, keylen = 0; 1160 1161 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1162 ubsecstats.hst_badsession++; 1163 return (EINVAL); 1164 } 1165 1166 mutex_spin_enter(&sc->sc_mtx); 1167 1168 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1169 ubsecstats.hst_queuefull++; 1170 sc->sc_needwakeup |= CRYPTO_SYMQ; 1171 mutex_spin_exit(&sc->sc_mtx); 1172 return(ERESTART); 1173 } 1174 1175 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1176 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next); 1177 mutex_spin_exit(&sc->sc_mtx); 1178 1179 dmap = q->q_dma; /* Save dma pointer */ 1180 /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */ 1181 memset(q, 0, offsetof(struct ubsec_q, q_src_map)); 1182 memset(&key, 0, sizeof(key)); 1183 1184 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1185 q->q_dma = dmap; 1186 ses = &sc->sc_sessions[q->q_sesn]; 1187 1188 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1189 q->q_src_m = (struct mbuf *)crp->crp_buf; 1190 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1191 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1192 q->q_src_io = (struct uio *)crp->crp_buf; 1193 q->q_dst_io = (struct uio *)crp->crp_buf; 1194 } else { 1195 ubsecstats.hst_badflags++; 1196 err = EINVAL; 1197 goto errout; /* XXX we don't handle contiguous blocks! */ 1198 } 1199 1200 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr)); 1201 1202 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1203 dmap->d_dma->d_mcr.mcr_flags = 0; 1204 q->q_crp = crp; 1205 1206 crd1 = crp->crp_desc; 1207 if (crd1 == NULL) { 1208 ubsecstats.hst_nodesc++; 1209 err = EINVAL; 1210 goto errout; 1211 } 1212 crd2 = crd1->crd_next; 1213 1214 if (crd2 == NULL) { 1215 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1216 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) { 1217 maccrd = crd1; 1218 enccrd = NULL; 1219 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1220 crd1->crd_alg == CRYPTO_3DES_CBC || 1221 crd1->crd_alg == CRYPTO_AES_CBC) { 1222 maccrd = NULL; 1223 enccrd = crd1; 1224 } else { 1225 ubsecstats.hst_badalg++; 1226 err = EINVAL; 1227 goto errout; 1228 } 1229 } else { 1230 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1231 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) && 1232 (crd2->crd_alg == CRYPTO_DES_CBC || 1233 crd2->crd_alg == CRYPTO_3DES_CBC || 1234 crd2->crd_alg == CRYPTO_AES_CBC) && 1235 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1236 maccrd = crd1; 1237 enccrd = crd2; 1238 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1239 crd1->crd_alg == CRYPTO_3DES_CBC || 1240 crd1->crd_alg == CRYPTO_AES_CBC) && 1241 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 1242 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) && 1243 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1244 enccrd = crd1; 1245 maccrd = crd2; 1246 } else { 1247 /* 1248 * We cannot order the ubsec as requested 1249 */ 1250 ubsecstats.hst_badalg++; 1251 err = EINVAL; 1252 goto errout; 1253 } 1254 } 1255 1256 if (enccrd) { 1257 if (enccrd->crd_alg == CRYPTO_AES_CBC) { 1258 if ((sc->sc_flags & UBS_FLAGS_AES) == 0) { 1259 /* 1260 * We cannot order the ubsec as requested 1261 */ 1262 ubsecstats.hst_badalg++; 1263 err = EINVAL; 1264 goto errout; 1265 } 1266 flags |= htole16(UBS_PKTCTX_ENC_AES); 1267 switch (enccrd->crd_klen) { 1268 case 128: 1269 case 192: 1270 case 256: 1271 keylen = enccrd->crd_klen / 8; 1272 break; 1273 default: 1274 err = EINVAL; 1275 goto errout; 1276 } 1277 ivlen = 16; 1278 } else { 1279 flags |= htole16(UBS_PKTCTX_ENC_3DES); 1280 ivlen = 8; 1281 keylen = 24; 1282 } 1283 1284 encoffset = enccrd->crd_skip; 1285 1286 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1287 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1288 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1289 else 1290 cprng_fast(key.ses_iv, ivlen); 1291 1292 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1293 if (crp->crp_flags & CRYPTO_F_IMBUF) 1294 m_copyback(q->q_src_m, 1295 enccrd->crd_inject, 1296 ivlen, (void *)key.ses_iv); 1297 else if (crp->crp_flags & CRYPTO_F_IOV) 1298 cuio_copyback(q->q_src_io, 1299 enccrd->crd_inject, 1300 ivlen, (void *)key.ses_iv); 1301 } 1302 } else { 1303 flags |= htole16(UBS_PKTCTX_INBOUND); 1304 1305 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1306 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1307 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1308 m_copydata(q->q_src_m, enccrd->crd_inject, 1309 ivlen, (void *)key.ses_iv); 1310 else if (crp->crp_flags & CRYPTO_F_IOV) 1311 cuio_copydata(q->q_src_io, 1312 enccrd->crd_inject, 8, 1313 (void *)key.ses_iv); 1314 } 1315 1316 for (i = 0; i < (keylen / 4); i++) 1317 key.ses_key[i] = ses->ses_key[i]; 1318 for (i = 0; i < (ivlen / 4); i++) 1319 SWAP32(key.ses_iv[i]); 1320 } 1321 1322 if (maccrd) { 1323 macoffset = maccrd->crd_skip; 1324 1325 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96) 1326 flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1327 else 1328 flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1329 1330 for (i = 0; i < 5; i++) { 1331 key.ses_hminner[i] = ses->ses_hminner[i]; 1332 key.ses_hmouter[i] = ses->ses_hmouter[i]; 1333 1334 HTOLE32(key.ses_hminner[i]); 1335 HTOLE32(key.ses_hmouter[i]); 1336 } 1337 } 1338 1339 if (enccrd && maccrd) { 1340 /* 1341 * ubsec cannot handle packets where the end of encryption 1342 * and authentication are not the same, or where the 1343 * encrypted part begins before the authenticated part. 1344 */ 1345 if ((encoffset + enccrd->crd_len) != 1346 (macoffset + maccrd->crd_len)) { 1347 ubsecstats.hst_lenmismatch++; 1348 err = EINVAL; 1349 goto errout; 1350 } 1351 if (enccrd->crd_skip < maccrd->crd_skip) { 1352 ubsecstats.hst_skipmismatch++; 1353 err = EINVAL; 1354 goto errout; 1355 } 1356 sskip = maccrd->crd_skip; 1357 cpskip = dskip = enccrd->crd_skip; 1358 stheend = maccrd->crd_len; 1359 dtheend = enccrd->crd_len; 1360 coffset = enccrd->crd_skip - maccrd->crd_skip; 1361 cpoffset = cpskip + dtheend; 1362 #ifdef UBSEC_DEBUG 1363 if (ubsec_debug) { 1364 printf("mac: skip %d, len %d, inject %d\n", 1365 maccrd->crd_skip, maccrd->crd_len, 1366 maccrd->crd_inject); 1367 printf("enc: skip %d, len %d, inject %d\n", 1368 enccrd->crd_skip, enccrd->crd_len, 1369 enccrd->crd_inject); 1370 printf("src: skip %d, len %d\n", sskip, stheend); 1371 printf("dst: skip %d, len %d\n", dskip, dtheend); 1372 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1373 coffset, stheend, cpskip, cpoffset); 1374 } 1375 #endif 1376 } else { 1377 cpskip = dskip = sskip = macoffset + encoffset; 1378 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1379 cpoffset = cpskip + dtheend; 1380 coffset = 0; 1381 } 1382 1383 if (q->q_src_map == NULL) { 1384 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */ 1385 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER, 1386 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) { 1387 err = ENOMEM; 1388 goto errout; 1389 } 1390 } 1391 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1392 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1393 q->q_src_m, BUS_DMA_NOWAIT) != 0) { 1394 ubsecstats.hst_noload++; 1395 err = ENOMEM; 1396 goto errout; 1397 } 1398 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1399 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1400 q->q_src_io, BUS_DMA_NOWAIT) != 0) { 1401 ubsecstats.hst_noload++; 1402 err = ENOMEM; 1403 goto errout; 1404 } 1405 } 1406 nicealign = ubsec_dmamap_aligned(q->q_src_map); 1407 1408 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1409 1410 #ifdef UBSEC_DEBUG 1411 if (ubsec_debug) 1412 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1413 #endif 1414 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) { 1415 struct ubsec_pktbuf *pb; 1416 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len; 1417 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr; 1418 1419 if (sskip >= packl) { 1420 sskip -= packl; 1421 continue; 1422 } 1423 1424 packl -= sskip; 1425 packp += sskip; 1426 sskip = 0; 1427 1428 if (packl > 0xfffc) { 1429 err = EIO; 1430 goto errout; 1431 } 1432 1433 if (j == 0) 1434 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1435 else 1436 pb = &dmap->d_dma->d_sbuf[j - 1]; 1437 1438 pb->pb_addr = htole32(packp); 1439 1440 if (stheend) { 1441 if (packl > stheend) { 1442 pb->pb_len = htole32(stheend); 1443 stheend = 0; 1444 } else { 1445 pb->pb_len = htole32(packl); 1446 stheend -= packl; 1447 } 1448 } else 1449 pb->pb_len = htole32(packl); 1450 1451 if ((i + 1) == q->q_src_map->dm_nsegs) 1452 pb->pb_next = 0; 1453 else 1454 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1455 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1456 j++; 1457 } 1458 1459 if (enccrd == NULL && maccrd != NULL) { 1460 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1461 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1462 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1463 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1464 #ifdef UBSEC_DEBUG 1465 if (ubsec_debug) 1466 printf("opkt: %x %x %x\n", 1467 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1468 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1469 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1470 1471 #endif 1472 } else { 1473 if (crp->crp_flags & CRYPTO_F_IOV) { 1474 if (!nicealign) { 1475 ubsecstats.hst_iovmisaligned++; 1476 err = EINVAL; 1477 goto errout; 1478 } 1479 if (q->q_dst_map == NULL) { 1480 if (q->q_cached_dst_map == NULL) { 1481 /* 1482 * XXX: ``what the heck's that'' 1483 * 0xfff0? 1484 */ 1485 if (bus_dmamap_create(sc->sc_dmat, 1486 0xfff0, UBS_MAX_SCATTER, 0xfff0, 0, 1487 BUS_DMA_NOWAIT, 1488 &q->q_cached_dst_map) != 0) { 1489 ubsecstats.hst_nomap++; 1490 err = ENOMEM; 1491 goto errout; 1492 } 1493 } 1494 q->q_dst_map = q->q_cached_dst_map; 1495 } 1496 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1497 q->q_dst_io, BUS_DMA_NOWAIT) != 0) { 1498 ubsecstats.hst_noload++; 1499 err = ENOMEM; 1500 goto errout; 1501 } 1502 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1503 if (nicealign) { 1504 q->q_dst_m = q->q_src_m; 1505 q->q_dst_map = q->q_src_map; 1506 } else { 1507 int totlen, len; 1508 struct mbuf *m, *top, **mp; 1509 1510 ubsecstats.hst_unaligned++; 1511 totlen = q->q_src_map->dm_mapsize; 1512 if (q->q_src_m->m_flags & M_PKTHDR) { 1513 len = MHLEN; 1514 MGETHDR(m, M_DONTWAIT, MT_DATA); 1515 /*XXX FIXME: m_dup_pkthdr */ 1516 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) { 1517 m_free(m); 1518 m = NULL; 1519 } 1520 } else { 1521 len = MLEN; 1522 MGET(m, M_DONTWAIT, MT_DATA); 1523 } 1524 if (m == NULL) { 1525 ubsecstats.hst_nombuf++; 1526 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1527 goto errout; 1528 } 1529 if (len == MHLEN) 1530 /*XXX was M_DUP_PKTHDR*/ 1531 m_copy_pkthdr(m, q->q_src_m); 1532 if (totlen >= MINCLSIZE) { 1533 MCLGET(m, M_DONTWAIT); 1534 if ((m->m_flags & M_EXT) == 0) { 1535 m_free(m); 1536 ubsecstats.hst_nomcl++; 1537 err = sc->sc_nqueue 1538 ? ERESTART : ENOMEM; 1539 goto errout; 1540 } 1541 len = MCLBYTES; 1542 } 1543 m->m_len = len; 1544 top = NULL; 1545 mp = ⊤ 1546 1547 while (totlen > 0) { 1548 if (top) { 1549 MGET(m, M_DONTWAIT, MT_DATA); 1550 if (m == NULL) { 1551 m_freem(top); 1552 ubsecstats.hst_nombuf++; 1553 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1554 goto errout; 1555 } 1556 len = MLEN; 1557 } 1558 if (top && totlen >= MINCLSIZE) { 1559 MCLGET(m, M_DONTWAIT); 1560 if ((m->m_flags & M_EXT) == 0) { 1561 *mp = m; 1562 m_freem(top); 1563 ubsecstats.hst_nomcl++; 1564 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1565 goto errout; 1566 } 1567 len = MCLBYTES; 1568 } 1569 m->m_len = len = uimin(totlen, len); 1570 totlen -= len; 1571 *mp = m; 1572 mp = &m->m_next; 1573 } 1574 q->q_dst_m = top; 1575 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1576 cpskip, cpoffset); 1577 if (q->q_dst_map == NULL) { 1578 if (q->q_cached_dst_map == NULL) { 1579 /* XXX again, what the heck is that 0xfff0? */ 1580 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1581 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1582 &q->q_cached_dst_map) != 0) { 1583 ubsecstats.hst_nomap++; 1584 err = ENOMEM; 1585 goto errout; 1586 } 1587 } 1588 q->q_dst_map = q->q_cached_dst_map; 1589 } 1590 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1591 q->q_dst_map, q->q_dst_m, 1592 BUS_DMA_NOWAIT) != 0) { 1593 ubsecstats.hst_noload++; 1594 err = ENOMEM; 1595 goto errout; 1596 } 1597 } 1598 } else { 1599 ubsecstats.hst_badflags++; 1600 err = EINVAL; 1601 goto errout; 1602 } 1603 1604 #ifdef UBSEC_DEBUG 1605 if (ubsec_debug) 1606 printf("dst skip: %d\n", dskip); 1607 #endif 1608 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) { 1609 struct ubsec_pktbuf *pb; 1610 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len; 1611 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr; 1612 1613 if (dskip >= packl) { 1614 dskip -= packl; 1615 continue; 1616 } 1617 1618 packl -= dskip; 1619 packp += dskip; 1620 dskip = 0; 1621 1622 if (packl > 0xfffc) { 1623 err = EIO; 1624 goto errout; 1625 } 1626 1627 if (j == 0) 1628 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1629 else 1630 pb = &dmap->d_dma->d_dbuf[j - 1]; 1631 1632 pb->pb_addr = htole32(packp); 1633 1634 if (dtheend) { 1635 if (packl > dtheend) { 1636 pb->pb_len = htole32(dtheend); 1637 dtheend = 0; 1638 } else { 1639 pb->pb_len = htole32(packl); 1640 dtheend -= packl; 1641 } 1642 } else 1643 pb->pb_len = htole32(packl); 1644 1645 if ((i + 1) == q->q_dst_map->dm_nsegs) { 1646 if (maccrd) 1647 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1648 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1649 else 1650 pb->pb_next = 0; 1651 } else 1652 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1653 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1654 j++; 1655 } 1656 } 1657 1658 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1659 offsetof(struct ubsec_dmachunk, d_ctx)); 1660 1661 if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) { 1662 struct ubsec_pktctx_aes128 *aes128; 1663 struct ubsec_pktctx_aes192 *aes192; 1664 struct ubsec_pktctx_aes256 *aes256; 1665 struct ubsec_pktctx_hdr *ph; 1666 u_int8_t *ctx; 1667 1668 ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1669 offsetof(struct ubsec_dmachunk, d_ctx); 1670 1671 ph = (struct ubsec_pktctx_hdr *)ctx; 1672 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES); 1673 ph->ph_flags = flags; 1674 ph->ph_offset = htole16(coffset >> 2); 1675 1676 switch (enccrd->crd_klen) { 1677 case 128: 1678 aes128 = (struct ubsec_pktctx_aes128 *)ctx; 1679 ph->ph_len = htole16(sizeof(*aes128)); 1680 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128); 1681 for (i = 0; i < 4; i++) 1682 aes128->pc_aeskey[i] = key.ses_key[i]; 1683 for (i = 0; i < 5; i++) 1684 aes128->pc_hminner[i] = key.ses_hminner[i]; 1685 for (i = 0; i < 5; i++) 1686 aes128->pc_hmouter[i] = key.ses_hmouter[i]; 1687 for (i = 0; i < 4; i++) 1688 aes128->pc_iv[i] = key.ses_iv[i]; 1689 break; 1690 case 192: 1691 aes192 = (struct ubsec_pktctx_aes192 *)ctx; 1692 ph->ph_len = htole16(sizeof(*aes192)); 1693 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192); 1694 for (i = 0; i < 6; i++) 1695 aes192->pc_aeskey[i] = key.ses_key[i]; 1696 for (i = 0; i < 5; i++) 1697 aes192->pc_hminner[i] = key.ses_hminner[i]; 1698 for (i = 0; i < 5; i++) 1699 aes192->pc_hmouter[i] = key.ses_hmouter[i]; 1700 for (i = 0; i < 4; i++) 1701 aes192->pc_iv[i] = key.ses_iv[i]; 1702 break; 1703 case 256: 1704 aes256 = (struct ubsec_pktctx_aes256 *)ctx; 1705 ph->ph_len = htole16(sizeof(*aes256)); 1706 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256); 1707 for (i = 0; i < 8; i++) 1708 aes256->pc_aeskey[i] = key.ses_key[i]; 1709 for (i = 0; i < 5; i++) 1710 aes256->pc_hminner[i] = key.ses_hminner[i]; 1711 for (i = 0; i < 5; i++) 1712 aes256->pc_hmouter[i] = key.ses_hmouter[i]; 1713 for (i = 0; i < 4; i++) 1714 aes256->pc_iv[i] = key.ses_iv[i]; 1715 break; 1716 } 1717 } else if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1718 struct ubsec_pktctx_3des *ctx; 1719 struct ubsec_pktctx_hdr *ph; 1720 1721 ctx = (struct ubsec_pktctx_3des *) 1722 ((u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1723 offsetof(struct ubsec_dmachunk, d_ctx)); 1724 1725 ph = (struct ubsec_pktctx_hdr *)ctx; 1726 ph->ph_len = htole16(sizeof(*ctx)); 1727 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES); 1728 ph->ph_flags = flags; 1729 ph->ph_offset = htole16(coffset >> 2); 1730 1731 for (i = 0; i < 6; i++) 1732 ctx->pc_deskey[i] = key.ses_key[i]; 1733 for (i = 0; i < 5; i++) 1734 ctx->pc_hminner[i] = key.ses_hminner[i]; 1735 for (i = 0; i < 5; i++) 1736 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1737 for (i = 0; i < 2; i++) 1738 ctx->pc_iv[i] = key.ses_iv[i]; 1739 } else { 1740 struct ubsec_pktctx *ctx = (struct ubsec_pktctx *) 1741 ((u_int8_t *)dmap->d_alloc.dma_vaddr + 1742 offsetof(struct ubsec_dmachunk, d_ctx)); 1743 1744 ctx->pc_flags = flags; 1745 ctx->pc_offset = htole16(coffset >> 2); 1746 for (i = 0; i < 6; i++) 1747 ctx->pc_deskey[i] = key.ses_key[i]; 1748 for (i = 0; i < 5; i++) 1749 ctx->pc_hminner[i] = key.ses_hminner[i]; 1750 for (i = 0; i < 5; i++) 1751 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1752 for (i = 0; i < 2; i++) 1753 ctx->pc_iv[i] = key.ses_iv[i]; 1754 } 1755 1756 mutex_spin_enter(&sc->sc_mtx); 1757 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1758 sc->sc_nqueue++; 1759 ubsecstats.hst_ipackets++; 1760 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize; 1761 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1762 ubsec_feed(sc); 1763 mutex_spin_exit(&sc->sc_mtx); 1764 return (0); 1765 1766 errout: 1767 if (q != NULL) { 1768 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1769 m_freem(q->q_dst_m); 1770 1771 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1772 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1773 } 1774 if (q->q_src_map != NULL) { 1775 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1776 } 1777 1778 mutex_spin_enter(&sc->sc_mtx); 1779 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1780 mutex_spin_exit(&sc->sc_mtx); 1781 } 1782 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */ 1783 if (err == EINVAL) 1784 ubsecstats.hst_invalid++; 1785 else 1786 ubsecstats.hst_nomem++; 1787 #endif 1788 if (err != ERESTART) { 1789 crp->crp_etype = err; 1790 crypto_done(crp); 1791 } else { 1792 sc->sc_needwakeup |= CRYPTO_SYMQ; 1793 } 1794 return (err); 1795 } 1796 1797 static void 1798 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1799 { 1800 struct cryptop *crp = (struct cryptop *)q->q_crp; 1801 struct cryptodesc *crd; 1802 struct ubsec_dma *dmap = q->q_dma; 1803 1804 ubsecstats.hst_opackets++; 1805 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1806 1807 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0, 1808 dmap->d_alloc.dma_map->dm_mapsize, 1809 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1810 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1811 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1812 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1813 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1814 } 1815 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 1816 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1817 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1818 1819 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1820 m_freem(q->q_src_m); 1821 crp->crp_buf = (void *)q->q_dst_m; 1822 } 1823 1824 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1825 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 && 1826 crd->crd_alg != CRYPTO_SHA1_HMAC_96) 1827 continue; 1828 if (crp->crp_flags & CRYPTO_F_IMBUF) 1829 m_copyback((struct mbuf *)crp->crp_buf, 1830 crd->crd_inject, 12, 1831 (void *)dmap->d_dma->d_macbuf); 1832 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1833 bcopy((void *)dmap->d_dma->d_macbuf, 1834 crp->crp_mac, 12); 1835 break; 1836 } 1837 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1838 crypto_done(crp); 1839 } 1840 1841 static void 1842 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1843 { 1844 int i, j, dlen, slen; 1845 char *dptr, *sptr; 1846 1847 j = 0; 1848 sptr = srcm->m_data; 1849 slen = srcm->m_len; 1850 dptr = dstm->m_data; 1851 dlen = dstm->m_len; 1852 1853 while (1) { 1854 for (i = 0; i < uimin(slen, dlen); i++) { 1855 if (j < hoffset || j >= toffset) 1856 *dptr++ = *sptr++; 1857 slen--; 1858 dlen--; 1859 j++; 1860 } 1861 if (slen == 0) { 1862 srcm = srcm->m_next; 1863 if (srcm == NULL) 1864 return; 1865 sptr = srcm->m_data; 1866 slen = srcm->m_len; 1867 } 1868 if (dlen == 0) { 1869 dstm = dstm->m_next; 1870 if (dstm == NULL) 1871 return; 1872 dptr = dstm->m_data; 1873 dlen = dstm->m_len; 1874 } 1875 } 1876 } 1877 1878 /* 1879 * feed the key generator, must be called at splnet() or higher. 1880 */ 1881 static void 1882 ubsec_feed2(struct ubsec_softc *sc) 1883 { 1884 struct ubsec_q2 *q; 1885 1886 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1887 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1888 break; 1889 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1890 1891 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1892 q->q_mcr.dma_map->dm_mapsize, 1893 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1894 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1895 q->q_ctx.dma_map->dm_mapsize, 1896 BUS_DMASYNC_PREWRITE); 1897 1898 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1899 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1900 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next); 1901 --sc->sc_nqueue2; 1902 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1903 } 1904 } 1905 1906 /* 1907 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices) 1908 */ 1909 void 1910 ubsec_feed4(struct ubsec_softc *sc) 1911 { 1912 struct ubsec_q2 *q; 1913 1914 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) { 1915 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL) 1916 break; 1917 q = SIMPLEQ_FIRST(&sc->sc_queue4); 1918 1919 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1920 q->q_mcr.dma_map->dm_mapsize, 1921 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1922 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1923 q->q_ctx.dma_map->dm_mapsize, 1924 BUS_DMASYNC_PREWRITE); 1925 1926 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr); 1927 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next); 1928 --sc->sc_nqueue4; 1929 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next); 1930 } 1931 } 1932 1933 /* 1934 * Callback for handling random numbers 1935 */ 1936 static void 1937 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1938 { 1939 struct cryptkop *krp; 1940 struct ubsec_ctx_keyop *ctx; 1941 1942 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1943 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1944 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1945 1946 switch (q->q_type) { 1947 #ifndef UBSEC_NO_RNG 1948 case UBS_CTXOP_RNGSHA1: 1949 case UBS_CTXOP_RNGBYPASS: { 1950 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1951 u_int32_t *p; 1952 int i; 1953 1954 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 1955 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1956 p = (u_int32_t *)rng->rng_buf.dma_vaddr; 1957 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t); 1958 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY); 1959 sc->sc_rng_need -= i; 1960 rng->rng_used = 0; 1961 if (sc->sc_rng_need > 0) { 1962 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 1963 } 1964 break; 1965 } 1966 #endif 1967 case UBS_CTXOP_MODEXP: { 1968 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1969 u_int rlen, clen; 1970 1971 krp = me->me_krp; 1972 rlen = (me->me_modbits + 7) / 8; 1973 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1974 1975 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 1976 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1977 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 1978 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1979 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 1980 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1981 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 1982 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1983 1984 if (clen < rlen) 1985 krp->krp_status = E2BIG; 1986 else { 1987 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1988 memset(krp->krp_param[krp->krp_iparams].crp_p, 0, 1989 (krp->krp_param[krp->krp_iparams].crp_nbits 1990 + 7) / 8); 1991 bcopy(me->me_C.dma_vaddr, 1992 krp->krp_param[krp->krp_iparams].crp_p, 1993 (me->me_modbits + 7) / 8); 1994 } else 1995 ubsec_kshift_l(me->me_shiftbits, 1996 me->me_C.dma_vaddr, me->me_normbits, 1997 krp->krp_param[krp->krp_iparams].crp_p, 1998 krp->krp_param[krp->krp_iparams].crp_nbits); 1999 } 2000 2001 crypto_kdone(krp); 2002 2003 /* bzero all potentially sensitive data */ 2004 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2005 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2006 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2007 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2008 2009 /* Can't free here, so put us on the free list. */ 2010 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 2011 break; 2012 } 2013 case UBS_CTXOP_RSAPRIV: { 2014 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2015 u_int len; 2016 2017 krp = rp->rpr_krp; 2018 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0, 2019 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2020 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0, 2021 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2022 2023 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) 2024 / 8; 2025 bcopy(rp->rpr_msgout.dma_vaddr, 2026 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 2027 2028 crypto_kdone(krp); 2029 2030 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 2031 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 2032 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size); 2033 2034 /* Can't free here, so put us on the free list. */ 2035 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 2036 break; 2037 } 2038 default: 2039 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev), 2040 letoh16(ctx->ctx_op)); 2041 break; 2042 } 2043 } 2044 2045 #ifndef UBSEC_NO_RNG 2046 2047 static void 2048 ubsec_rng_get(size_t bytes, void *vsc) 2049 { 2050 struct ubsec_softc *sc = vsc; 2051 2052 mutex_spin_enter(&sc->sc_mtx); 2053 sc->sc_rng_need = bytes; 2054 ubsec_rng_locked(sc); 2055 mutex_spin_exit(&sc->sc_mtx); 2056 2057 } 2058 2059 static void 2060 ubsec_rng(void *vsc) 2061 { 2062 struct ubsec_softc *sc = vsc; 2063 mutex_spin_enter(&sc->sc_mtx); 2064 ubsec_rng_locked(sc); 2065 mutex_spin_exit(&sc->sc_mtx); 2066 } 2067 2068 static void 2069 ubsec_rng_locked(void *vsc) 2070 { 2071 struct ubsec_softc *sc = vsc; 2072 struct ubsec_q2_rng *rng = &sc->sc_rng; 2073 struct ubsec_mcr *mcr; 2074 struct ubsec_ctx_rngbypass *ctx; 2075 int *nqueue; 2076 2077 /* Caller is responsible to lock and release sc_mtx. */ 2078 KASSERT(mutex_owned(&sc->sc_mtx)); 2079 2080 if (rng->rng_used) { 2081 return; 2082 } 2083 2084 if (sc->sc_rng_need < 1) { 2085 callout_stop(&sc->sc_rngto); 2086 return; 2087 } 2088 2089 if (sc->sc_flags & UBS_FLAGS_RNG4) 2090 nqueue = &sc->sc_nqueue4; 2091 else 2092 nqueue = &sc->sc_nqueue2; 2093 2094 (*nqueue)++; 2095 if (*nqueue >= UBS_MAX_NQUEUE) 2096 goto out; 2097 2098 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 2099 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 2100 2101 mcr->mcr_pkts = htole16(1); 2102 mcr->mcr_flags = 0; 2103 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 2104 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 2105 mcr->mcr_ipktbuf.pb_len = 0; 2106 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 2107 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 2108 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 2109 UBS_PKTBUF_LEN); 2110 mcr->mcr_opktbuf.pb_next = 0; 2111 2112 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 2113 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1); 2114 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1; 2115 2116 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2117 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2118 2119 if (sc->sc_flags & UBS_FLAGS_RNG4) { 2120 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next); 2121 ubsec_feed4(sc); 2122 } else { 2123 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 2124 ubsec_feed2(sc); 2125 } 2126 rng->rng_used = 1; 2127 ubsecstats.hst_rng++; 2128 2129 return; 2130 2131 out: 2132 /* 2133 * Something weird happened, generate our own call back. 2134 */ 2135 (*nqueue)--; 2136 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2137 } 2138 #endif /* UBSEC_NO_RNG */ 2139 2140 static int 2141 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size, 2142 struct ubsec_dma_alloc *dma,int mapflags) 2143 { 2144 int r; 2145 2146 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, 2147 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0) 2148 goto fail_0; 2149 2150 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, 2151 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0) 2152 goto fail_1; 2153 2154 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2155 BUS_DMA_NOWAIT, &dma->dma_map)) != 0) 2156 goto fail_2; 2157 2158 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 2159 size, NULL, BUS_DMA_NOWAIT)) != 0) 2160 goto fail_3; 2161 2162 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2163 dma->dma_size = size; 2164 return (0); 2165 2166 fail_3: 2167 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2168 fail_2: 2169 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size); 2170 fail_1: 2171 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2172 fail_0: 2173 dma->dma_map = NULL; 2174 return (r); 2175 } 2176 2177 static void 2178 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 2179 { 2180 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 2181 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size); 2182 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2183 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2184 } 2185 2186 /* 2187 * Resets the board. Values in the regesters are left as is 2188 * from the reset (i.e. initial values are assigned elsewhere). 2189 */ 2190 static void 2191 ubsec_reset_board(struct ubsec_softc *sc) 2192 { 2193 volatile u_int32_t ctrl; 2194 2195 ctrl = READ_REG(sc, BS_CTRL); 2196 ctrl |= BS_CTRL_RESET; 2197 WRITE_REG(sc, BS_CTRL, ctrl); 2198 2199 /* 2200 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2201 */ 2202 DELAY(10); 2203 2204 /* Enable RNG and interrupts on newer devices */ 2205 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2206 #ifndef UBSEC_NO_RNG 2207 WRITE_REG(sc, BS_CFG, BS_CFG_RNG); 2208 #endif 2209 WRITE_REG(sc, BS_INT, BS_INT_DMAINT); 2210 } 2211 } 2212 2213 /* 2214 * Init Broadcom registers 2215 */ 2216 static void 2217 ubsec_init_board(struct ubsec_softc *sc) 2218 { 2219 u_int32_t ctrl; 2220 2221 ctrl = READ_REG(sc, BS_CTRL); 2222 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2223 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2224 2225 /* 2226 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)). 2227 * anyone got hw docs? 2228 */ 2229 if (sc->sc_flags & UBS_FLAGS_KEY) 2230 ctrl |= BS_CTRL_MCR2INT; 2231 else 2232 ctrl &= ~BS_CTRL_MCR2INT; 2233 2234 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2235 ctrl &= ~BS_CTRL_SWNORM; 2236 2237 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2238 ctrl |= BS_CTRL_BSIZE240; 2239 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */ 2240 2241 if (sc->sc_flags & UBS_FLAGS_RNG4) 2242 ctrl |= BS_CTRL_MCR4INT; 2243 else 2244 ctrl &= ~BS_CTRL_MCR4INT; 2245 } 2246 2247 WRITE_REG(sc, BS_CTRL, ctrl); 2248 } 2249 2250 /* 2251 * Init Broadcom PCI registers 2252 */ 2253 static void 2254 ubsec_init_pciregs(struct pci_attach_args *pa) 2255 { 2256 pci_chipset_tag_t pc = pa->pa_pc; 2257 u_int32_t misc; 2258 2259 /* 2260 * This will set the cache line size to 1, this will 2261 * force the BCM58xx chip just to do burst read/writes. 2262 * Cache line read/writes are to slow 2263 */ 2264 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG); 2265 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT)) 2266 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT); 2267 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc); 2268 } 2269 2270 /* 2271 * Clean up after a chip crash. 2272 * It is assumed that the caller in splnet() 2273 */ 2274 static void 2275 ubsec_cleanchip(struct ubsec_softc *sc) 2276 { 2277 struct ubsec_q *q; 2278 2279 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2280 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2281 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 2282 ubsec_free_q(sc, q); 2283 } 2284 sc->sc_nqchip = 0; 2285 } 2286 2287 /* 2288 * free a ubsec_q 2289 * It is assumed that the caller is within splnet() 2290 */ 2291 static int 2292 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2293 { 2294 struct ubsec_q *q2; 2295 struct cryptop *crp; 2296 int npkts; 2297 int i; 2298 2299 npkts = q->q_nstacked_mcrs; 2300 2301 for (i = 0; i < npkts; i++) { 2302 if(q->q_stacked_mcr[i]) { 2303 q2 = q->q_stacked_mcr[i]; 2304 2305 if ((q2->q_dst_m != NULL) 2306 && (q2->q_src_m != q2->q_dst_m)) 2307 m_freem(q2->q_dst_m); 2308 2309 crp = (struct cryptop *)q2->q_crp; 2310 2311 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2312 2313 crp->crp_etype = EFAULT; 2314 crypto_done(crp); 2315 } else { 2316 break; 2317 } 2318 } 2319 2320 /* 2321 * Free header MCR 2322 */ 2323 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2324 m_freem(q->q_dst_m); 2325 2326 crp = (struct cryptop *)q->q_crp; 2327 2328 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2329 2330 crp->crp_etype = EFAULT; 2331 crypto_done(crp); 2332 return(0); 2333 } 2334 2335 /* 2336 * Routine to reset the chip and clean up. 2337 * It is assumed that the caller is in splnet() 2338 */ 2339 static void 2340 ubsec_totalreset(struct ubsec_softc *sc) 2341 { 2342 ubsec_reset_board(sc); 2343 ubsec_init_board(sc); 2344 ubsec_cleanchip(sc); 2345 } 2346 2347 static int 2348 ubsec_dmamap_aligned(bus_dmamap_t map) 2349 { 2350 int i; 2351 2352 for (i = 0; i < map->dm_nsegs; i++) { 2353 if (map->dm_segs[i].ds_addr & 3) 2354 return (0); 2355 if ((i != (map->dm_nsegs - 1)) && 2356 (map->dm_segs[i].ds_len & 3)) 2357 return (0); 2358 } 2359 return (1); 2360 } 2361 2362 static void 2363 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2364 { 2365 switch (q->q_type) { 2366 case UBS_CTXOP_MODEXP: { 2367 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2368 2369 ubsec_dma_free(sc, &me->me_q.q_mcr); 2370 ubsec_dma_free(sc, &me->me_q.q_ctx); 2371 ubsec_dma_free(sc, &me->me_M); 2372 ubsec_dma_free(sc, &me->me_E); 2373 ubsec_dma_free(sc, &me->me_C); 2374 ubsec_dma_free(sc, &me->me_epb); 2375 free(me, M_DEVBUF); 2376 break; 2377 } 2378 case UBS_CTXOP_RSAPRIV: { 2379 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2380 2381 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2382 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2383 ubsec_dma_free(sc, &rp->rpr_msgin); 2384 ubsec_dma_free(sc, &rp->rpr_msgout); 2385 free(rp, M_DEVBUF); 2386 break; 2387 } 2388 default: 2389 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev), 2390 q->q_type); 2391 break; 2392 } 2393 } 2394 2395 static int 2396 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2397 { 2398 struct ubsec_softc *sc = arg; 2399 2400 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2401 struct ubsec_q2 *q; 2402 2403 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2404 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next); 2405 ubsec_kfree(sc, q); 2406 } 2407 2408 switch (krp->krp_op) { 2409 case CRK_MOD_EXP: 2410 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2411 ubsec_kprocess_modexp_hw(sc, krp, hint); 2412 else 2413 ubsec_kprocess_modexp_sw(sc, krp, hint); 2414 break; 2415 case CRK_MOD_EXP_CRT: 2416 ubsec_kprocess_rsapriv(sc, krp, hint); 2417 break; 2418 default: 2419 printf("%s: kprocess: invalid op 0x%x\n", 2420 device_xname(sc->sc_dev), krp->krp_op); 2421 krp->krp_status = EOPNOTSUPP; 2422 crypto_kdone(krp); 2423 } 2424 return 0; 2425 } 2426 2427 /* 2428 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2429 */ 2430 static void 2431 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, 2432 int hint) 2433 { 2434 struct ubsec_q2_modexp *me; 2435 struct ubsec_mcr *mcr; 2436 struct ubsec_ctx_modexp *ctx; 2437 struct ubsec_pktbuf *epb; 2438 int err = 0; 2439 u_int nbits, normbits, mbits, shiftbits, ebits; 2440 2441 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2442 if (me == NULL) { 2443 err = ENOMEM; 2444 goto errout; 2445 } 2446 memset(me, 0, sizeof *me); 2447 me->me_krp = krp; 2448 me->me_q.q_type = UBS_CTXOP_MODEXP; 2449 2450 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2451 if (nbits <= 512) 2452 normbits = 512; 2453 else if (nbits <= 768) 2454 normbits = 768; 2455 else if (nbits <= 1024) 2456 normbits = 1024; 2457 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2458 normbits = 1536; 2459 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2460 normbits = 2048; 2461 else { 2462 err = E2BIG; 2463 goto errout; 2464 } 2465 2466 shiftbits = normbits - nbits; 2467 2468 me->me_modbits = nbits; 2469 me->me_shiftbits = shiftbits; 2470 me->me_normbits = normbits; 2471 2472 /* Sanity check: result bits must be >= true modulus bits. */ 2473 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2474 err = ERANGE; 2475 goto errout; 2476 } 2477 2478 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2479 &me->me_q.q_mcr, 0)) { 2480 err = ENOMEM; 2481 goto errout; 2482 } 2483 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2484 2485 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2486 &me->me_q.q_ctx, 0)) { 2487 err = ENOMEM; 2488 goto errout; 2489 } 2490 2491 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2492 if (mbits > nbits) { 2493 err = E2BIG; 2494 goto errout; 2495 } 2496 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2497 err = ENOMEM; 2498 goto errout; 2499 } 2500 ubsec_kshift_r(shiftbits, 2501 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2502 me->me_M.dma_vaddr, normbits); 2503 2504 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2505 err = ENOMEM; 2506 goto errout; 2507 } 2508 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2509 2510 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2511 if (ebits > nbits) { 2512 err = E2BIG; 2513 goto errout; 2514 } 2515 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2516 err = ENOMEM; 2517 goto errout; 2518 } 2519 ubsec_kshift_r(shiftbits, 2520 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2521 me->me_E.dma_vaddr, normbits); 2522 2523 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2524 &me->me_epb, 0)) { 2525 err = ENOMEM; 2526 goto errout; 2527 } 2528 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2529 epb->pb_addr = htole32(me->me_E.dma_paddr); 2530 epb->pb_next = 0; 2531 epb->pb_len = htole32(normbits / 8); 2532 2533 #ifdef UBSEC_DEBUG 2534 if (ubsec_debug) { 2535 printf("Epb "); 2536 ubsec_dump_pb(epb); 2537 } 2538 #endif 2539 2540 mcr->mcr_pkts = htole16(1); 2541 mcr->mcr_flags = 0; 2542 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2543 mcr->mcr_reserved = 0; 2544 mcr->mcr_pktlen = 0; 2545 2546 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2547 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2548 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2549 2550 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2551 mcr->mcr_opktbuf.pb_next = 0; 2552 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2553 2554 #ifdef DIAGNOSTIC 2555 /* Misaligned output buffer will hang the chip. */ 2556 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2557 panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev), 2558 letoh32(mcr->mcr_opktbuf.pb_addr)); 2559 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2560 panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev), 2561 letoh32(mcr->mcr_opktbuf.pb_len)); 2562 #endif 2563 2564 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2565 memset(ctx, 0, sizeof(*ctx)); 2566 ubsec_kshift_r(shiftbits, 2567 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2568 ctx->me_N, normbits); 2569 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2570 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2571 ctx->me_E_len = htole16(nbits); 2572 ctx->me_N_len = htole16(nbits); 2573 2574 #ifdef UBSEC_DEBUG 2575 if (ubsec_debug) { 2576 ubsec_dump_mcr(mcr); 2577 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2578 } 2579 #endif 2580 2581 /* 2582 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2583 * everything else. 2584 */ 2585 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2586 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2587 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2588 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2589 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2590 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2591 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2592 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2593 2594 /* Enqueue and we're done... */ 2595 mutex_spin_enter(&sc->sc_mtx); 2596 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2597 ubsec_feed2(sc); 2598 ubsecstats.hst_modexp++; 2599 mutex_spin_exit(&sc->sc_mtx); 2600 2601 return; 2602 2603 errout: 2604 if (me != NULL) { 2605 if (me->me_q.q_mcr.dma_map != NULL) 2606 ubsec_dma_free(sc, &me->me_q.q_mcr); 2607 if (me->me_q.q_ctx.dma_map != NULL) { 2608 memset(me->me_q.q_ctx.dma_vaddr, 0, 2609 me->me_q.q_ctx.dma_size); 2610 ubsec_dma_free(sc, &me->me_q.q_ctx); 2611 } 2612 if (me->me_M.dma_map != NULL) { 2613 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2614 ubsec_dma_free(sc, &me->me_M); 2615 } 2616 if (me->me_E.dma_map != NULL) { 2617 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2618 ubsec_dma_free(sc, &me->me_E); 2619 } 2620 if (me->me_C.dma_map != NULL) { 2621 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2622 ubsec_dma_free(sc, &me->me_C); 2623 } 2624 if (me->me_epb.dma_map != NULL) 2625 ubsec_dma_free(sc, &me->me_epb); 2626 free(me, M_DEVBUF); 2627 } 2628 krp->krp_status = err; 2629 crypto_kdone(krp); 2630 } 2631 2632 /* 2633 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2634 */ 2635 static void 2636 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, 2637 int hint) 2638 { 2639 struct ubsec_q2_modexp *me; 2640 struct ubsec_mcr *mcr; 2641 struct ubsec_ctx_modexp *ctx; 2642 struct ubsec_pktbuf *epb; 2643 int err = 0; 2644 u_int nbits, normbits, mbits, shiftbits, ebits; 2645 2646 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2647 if (me == NULL) { 2648 err = ENOMEM; 2649 goto errout; 2650 } 2651 memset(me, 0, sizeof *me); 2652 me->me_krp = krp; 2653 me->me_q.q_type = UBS_CTXOP_MODEXP; 2654 2655 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2656 if (nbits <= 512) 2657 normbits = 512; 2658 else if (nbits <= 768) 2659 normbits = 768; 2660 else if (nbits <= 1024) 2661 normbits = 1024; 2662 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2663 normbits = 1536; 2664 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2665 normbits = 2048; 2666 else { 2667 err = E2BIG; 2668 goto errout; 2669 } 2670 2671 shiftbits = normbits - nbits; 2672 2673 /* XXX ??? */ 2674 me->me_modbits = nbits; 2675 me->me_shiftbits = shiftbits; 2676 me->me_normbits = normbits; 2677 2678 /* Sanity check: result bits must be >= true modulus bits. */ 2679 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2680 err = ERANGE; 2681 goto errout; 2682 } 2683 2684 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2685 &me->me_q.q_mcr, 0)) { 2686 err = ENOMEM; 2687 goto errout; 2688 } 2689 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2690 2691 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2692 &me->me_q.q_ctx, 0)) { 2693 err = ENOMEM; 2694 goto errout; 2695 } 2696 2697 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2698 if (mbits > nbits) { 2699 err = E2BIG; 2700 goto errout; 2701 } 2702 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2703 err = ENOMEM; 2704 goto errout; 2705 } 2706 memset(me->me_M.dma_vaddr, 0, normbits / 8); 2707 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2708 me->me_M.dma_vaddr, (mbits + 7) / 8); 2709 2710 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2711 err = ENOMEM; 2712 goto errout; 2713 } 2714 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2715 2716 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2717 if (ebits > nbits) { 2718 err = E2BIG; 2719 goto errout; 2720 } 2721 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2722 err = ENOMEM; 2723 goto errout; 2724 } 2725 memset(me->me_E.dma_vaddr, 0, normbits / 8); 2726 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2727 me->me_E.dma_vaddr, (ebits + 7) / 8); 2728 2729 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2730 &me->me_epb, 0)) { 2731 err = ENOMEM; 2732 goto errout; 2733 } 2734 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2735 epb->pb_addr = htole32(me->me_E.dma_paddr); 2736 epb->pb_next = 0; 2737 epb->pb_len = htole32((ebits + 7) / 8); 2738 2739 #ifdef UBSEC_DEBUG 2740 if (ubsec_debug) { 2741 printf("Epb "); 2742 ubsec_dump_pb(epb); 2743 } 2744 #endif 2745 2746 mcr->mcr_pkts = htole16(1); 2747 mcr->mcr_flags = 0; 2748 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2749 mcr->mcr_reserved = 0; 2750 mcr->mcr_pktlen = 0; 2751 2752 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2753 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2754 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2755 2756 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2757 mcr->mcr_opktbuf.pb_next = 0; 2758 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2759 2760 #ifdef DIAGNOSTIC 2761 /* Misaligned output buffer will hang the chip. */ 2762 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2763 panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev), 2764 letoh32(mcr->mcr_opktbuf.pb_addr)); 2765 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2766 panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev), 2767 letoh32(mcr->mcr_opktbuf.pb_len)); 2768 #endif 2769 2770 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2771 memset(ctx, 0, sizeof(*ctx)); 2772 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p, 2773 (nbits + 7) / 8); 2774 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2775 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2776 ctx->me_E_len = htole16(ebits); 2777 ctx->me_N_len = htole16(nbits); 2778 2779 #ifdef UBSEC_DEBUG 2780 if (ubsec_debug) { 2781 ubsec_dump_mcr(mcr); 2782 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2783 } 2784 #endif 2785 2786 /* 2787 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2788 * everything else. 2789 */ 2790 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2791 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2792 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2793 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2794 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2795 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2796 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2797 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2798 2799 /* Enqueue and we're done... */ 2800 mutex_spin_enter(&sc->sc_mtx); 2801 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2802 ubsec_feed2(sc); 2803 mutex_spin_exit(&sc->sc_mtx); 2804 2805 return; 2806 2807 errout: 2808 if (me != NULL) { 2809 if (me->me_q.q_mcr.dma_map != NULL) 2810 ubsec_dma_free(sc, &me->me_q.q_mcr); 2811 if (me->me_q.q_ctx.dma_map != NULL) { 2812 memset(me->me_q.q_ctx.dma_vaddr, 0, 2813 me->me_q.q_ctx.dma_size); 2814 ubsec_dma_free(sc, &me->me_q.q_ctx); 2815 } 2816 if (me->me_M.dma_map != NULL) { 2817 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2818 ubsec_dma_free(sc, &me->me_M); 2819 } 2820 if (me->me_E.dma_map != NULL) { 2821 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2822 ubsec_dma_free(sc, &me->me_E); 2823 } 2824 if (me->me_C.dma_map != NULL) { 2825 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2826 ubsec_dma_free(sc, &me->me_C); 2827 } 2828 if (me->me_epb.dma_map != NULL) 2829 ubsec_dma_free(sc, &me->me_epb); 2830 free(me, M_DEVBUF); 2831 } 2832 krp->krp_status = err; 2833 crypto_kdone(krp); 2834 } 2835 2836 static void 2837 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, 2838 int hint) 2839 { 2840 struct ubsec_q2_rsapriv *rp = NULL; 2841 struct ubsec_mcr *mcr; 2842 struct ubsec_ctx_rsapriv *ctx; 2843 int err = 0; 2844 u_int padlen, msglen; 2845 2846 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2847 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2848 if (msglen > padlen) 2849 padlen = msglen; 2850 2851 if (padlen <= 256) 2852 padlen = 256; 2853 else if (padlen <= 384) 2854 padlen = 384; 2855 else if (padlen <= 512) 2856 padlen = 512; 2857 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2858 padlen = 768; 2859 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2860 padlen = 1024; 2861 else { 2862 err = E2BIG; 2863 goto errout; 2864 } 2865 2866 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2867 err = E2BIG; 2868 goto errout; 2869 } 2870 2871 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2872 err = E2BIG; 2873 goto errout; 2874 } 2875 2876 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2877 err = E2BIG; 2878 goto errout; 2879 } 2880 2881 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO); 2882 if (rp == NULL) { 2883 err = ENOMEM; 2884 goto errout; 2885 } 2886 rp->rpr_krp = krp; 2887 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2888 2889 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2890 &rp->rpr_q.q_mcr, 0)) { 2891 err = ENOMEM; 2892 goto errout; 2893 } 2894 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2895 2896 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2897 &rp->rpr_q.q_ctx, 0)) { 2898 err = ENOMEM; 2899 goto errout; 2900 } 2901 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2902 memset(ctx, 0, sizeof *ctx); 2903 2904 /* Copy in p */ 2905 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2906 &ctx->rpr_buf[0 * (padlen / 8)], 2907 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2908 2909 /* Copy in q */ 2910 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2911 &ctx->rpr_buf[1 * (padlen / 8)], 2912 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2913 2914 /* Copy in dp */ 2915 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2916 &ctx->rpr_buf[2 * (padlen / 8)], 2917 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2918 2919 /* Copy in dq */ 2920 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2921 &ctx->rpr_buf[3 * (padlen / 8)], 2922 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2923 2924 /* Copy in pinv */ 2925 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2926 &ctx->rpr_buf[4 * (padlen / 8)], 2927 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2928 2929 msglen = padlen * 2; 2930 2931 /* Copy in input message (aligned buffer/length). */ 2932 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2933 /* Is this likely? */ 2934 err = E2BIG; 2935 goto errout; 2936 } 2937 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2938 err = ENOMEM; 2939 goto errout; 2940 } 2941 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8); 2942 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2943 rp->rpr_msgin.dma_vaddr, 2944 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2945 2946 /* Prepare space for output message (aligned buffer/length). */ 2947 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2948 /* Is this likely? */ 2949 err = E2BIG; 2950 goto errout; 2951 } 2952 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2953 err = ENOMEM; 2954 goto errout; 2955 } 2956 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8); 2957 2958 mcr->mcr_pkts = htole16(1); 2959 mcr->mcr_flags = 0; 2960 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2961 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2962 mcr->mcr_ipktbuf.pb_next = 0; 2963 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2964 mcr->mcr_reserved = 0; 2965 mcr->mcr_pktlen = htole16(msglen); 2966 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2967 mcr->mcr_opktbuf.pb_next = 0; 2968 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2969 2970 #ifdef DIAGNOSTIC 2971 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2972 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)", 2973 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr, 2974 (u_long) rp->rpr_msgin.dma_size); 2975 } 2976 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2977 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)", 2978 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr, 2979 (u_long) rp->rpr_msgout.dma_size); 2980 } 2981 #endif 2982 2983 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2984 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2985 ctx->rpr_q_len = htole16(padlen); 2986 ctx->rpr_p_len = htole16(padlen); 2987 2988 /* 2989 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2990 * everything else. 2991 */ 2992 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 2993 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2994 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 2995 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2996 2997 /* Enqueue and we're done... */ 2998 mutex_spin_enter(&sc->sc_mtx); 2999 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 3000 ubsec_feed2(sc); 3001 ubsecstats.hst_modexpcrt++; 3002 mutex_spin_exit(&sc->sc_mtx); 3003 return; 3004 3005 errout: 3006 if (rp != NULL) { 3007 if (rp->rpr_q.q_mcr.dma_map != NULL) 3008 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 3009 if (rp->rpr_msgin.dma_map != NULL) { 3010 memset(rp->rpr_msgin.dma_vaddr, 0, 3011 rp->rpr_msgin.dma_size); 3012 ubsec_dma_free(sc, &rp->rpr_msgin); 3013 } 3014 if (rp->rpr_msgout.dma_map != NULL) { 3015 memset(rp->rpr_msgout.dma_vaddr, 0, 3016 rp->rpr_msgout.dma_size); 3017 ubsec_dma_free(sc, &rp->rpr_msgout); 3018 } 3019 free(rp, M_DEVBUF); 3020 } 3021 krp->krp_status = err; 3022 crypto_kdone(krp); 3023 } 3024 3025 #ifdef UBSEC_DEBUG 3026 static void 3027 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 3028 { 3029 printf("addr 0x%x (0x%x) next 0x%x\n", 3030 pb->pb_addr, pb->pb_len, pb->pb_next); 3031 } 3032 3033 static void 3034 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c) 3035 { 3036 printf("CTX (0x%x):\n", c->ctx_len); 3037 switch (letoh16(c->ctx_op)) { 3038 case UBS_CTXOP_RNGBYPASS: 3039 case UBS_CTXOP_RNGSHA1: 3040 break; 3041 case UBS_CTXOP_MODEXP: 3042 { 3043 struct ubsec_ctx_modexp *cx = (void *)c; 3044 int i, len; 3045 3046 printf(" Elen %u, Nlen %u\n", 3047 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 3048 len = (cx->me_N_len + 7)/8; 3049 for (i = 0; i < len; i++) 3050 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 3051 printf("\n"); 3052 break; 3053 } 3054 default: 3055 printf("unknown context: %x\n", c->ctx_op); 3056 } 3057 printf("END CTX\n"); 3058 } 3059 3060 static void 3061 ubsec_dump_mcr(struct ubsec_mcr *mcr) 3062 { 3063 volatile struct ubsec_mcr_add *ma; 3064 int i; 3065 3066 printf("MCR:\n"); 3067 printf(" pkts: %u, flags 0x%x\n", 3068 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 3069 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 3070 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 3071 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 3072 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 3073 letoh16(ma->mcr_reserved)); 3074 printf(" %d: ipkt ", i); 3075 ubsec_dump_pb(&ma->mcr_ipktbuf); 3076 printf(" %d: opkt ", i); 3077 ubsec_dump_pb(&ma->mcr_opktbuf); 3078 ma++; 3079 } 3080 printf("END MCR\n"); 3081 } 3082 #endif /* UBSEC_DEBUG */ 3083 3084 /* 3085 * Return the number of significant bits of a big number. 3086 */ 3087 static int 3088 ubsec_ksigbits(struct crparam *cr) 3089 { 3090 u_int plen = (cr->crp_nbits + 7) / 8; 3091 int i, sig = plen * 8; 3092 u_int8_t c, *p = cr->crp_p; 3093 3094 for (i = plen - 1; i >= 0; i--) { 3095 c = p[i]; 3096 if (c != 0) { 3097 while ((c & 0x80) == 0) { 3098 sig--; 3099 c <<= 1; 3100 } 3101 break; 3102 } 3103 sig -= 8; 3104 } 3105 return (sig); 3106 } 3107 3108 static void 3109 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits, 3110 u_int8_t *dst, u_int dstbits) 3111 { 3112 u_int slen, dlen; 3113 int i, si, di, n; 3114 3115 slen = (srcbits + 7) / 8; 3116 dlen = (dstbits + 7) / 8; 3117 3118 for (i = 0; i < slen; i++) 3119 dst[i] = src[i]; 3120 for (i = 0; i < dlen - slen; i++) 3121 dst[slen + i] = 0; 3122 3123 n = shiftbits / 8; 3124 if (n != 0) { 3125 si = dlen - n - 1; 3126 di = dlen - 1; 3127 while (si >= 0) 3128 dst[di--] = dst[si--]; 3129 while (di >= 0) 3130 dst[di--] = 0; 3131 } 3132 3133 n = shiftbits % 8; 3134 if (n != 0) { 3135 for (i = dlen - 1; i > 0; i--) 3136 dst[i] = (dst[i] << n) | 3137 (dst[i - 1] >> (8 - n)); 3138 dst[0] = dst[0] << n; 3139 } 3140 } 3141 3142 static void 3143 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits, 3144 u_int8_t *dst, u_int dstbits) 3145 { 3146 int slen, dlen, i, n; 3147 3148 slen = (srcbits + 7) / 8; 3149 dlen = (dstbits + 7) / 8; 3150 3151 n = shiftbits / 8; 3152 for (i = 0; i < slen; i++) 3153 dst[i] = src[i + n]; 3154 for (i = 0; i < dlen - slen; i++) 3155 dst[slen + i] = 0; 3156 3157 n = shiftbits % 8; 3158 if (n != 0) { 3159 for (i = 0; i < (dlen - 1); i++) 3160 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 3161 dst[dlen - 1] = dst[dlen - 1] >> n; 3162 } 3163 } 3164
Indexes created Tue Sep 23 02:09:52 GMT 2025