ubsec.c revision 1.43.18.3
1 /* $NetBSD: ubsec.c,v 1.43.18.3 2020/04/13 08:04:45 martin Exp $ */ 2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Effort sponsored in part by the Defense Advanced Research Projects 32 * Agency (DARPA) and Air Force Research Laboratory, Air Force 33 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 34 * 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.43.18.3 2020/04/13 08:04:45 martin Exp $"); 39 40 #undef UBSEC_DEBUG 41 42 /* 43 * uBsec 5[56]01, 58xx hardware crypto accelerator 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/endian.h> 50 #ifdef __NetBSD__ 51 #define UBSEC_NO_RNG /* hangs on attach */ 52 #define letoh16 htole16 53 #define letoh32 htole32 54 #endif 55 #include <sys/errno.h> 56 #include <sys/malloc.h> 57 #include <sys/kernel.h> 58 #include <sys/mbuf.h> 59 #include <sys/device.h> 60 #include <sys/module.h> 61 #include <sys/queue.h> 62 #include <sys/sysctl.h> 63 64 #include <opencrypto/cryptodev.h> 65 #include <opencrypto/xform.h> 66 #ifdef __OpenBSD__ 67 #include <dev/rndvar.h> 68 #include <sys/md5k.h> 69 #else 70 #include <sys/cprng.h> 71 #include <sys/md5.h> 72 #include <sys/rndpool.h> 73 #include <sys/rndsource.h> 74 #endif 75 #include <sys/sha1.h> 76 77 #include <dev/pci/pcireg.h> 78 #include <dev/pci/pcivar.h> 79 #include <dev/pci/pcidevs.h> 80 81 #include <dev/pci/ubsecreg.h> 82 #include <dev/pci/ubsecvar.h> 83 84 /* 85 * Prototypes and count for the pci_device structure 86 */ 87 static int ubsec_probe(device_t, cfdata_t, void *); 88 static void ubsec_attach(device_t, device_t, void *); 89 static int ubsec_detach(device_t, int); 90 static void ubsec_reset_board(struct ubsec_softc *); 91 static void ubsec_init_board(struct ubsec_softc *); 92 static void ubsec_init_pciregs(struct pci_attach_args *pa); 93 static void ubsec_cleanchip(struct ubsec_softc *); 94 static void ubsec_totalreset(struct ubsec_softc *); 95 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *); 96 97 #ifdef __OpenBSD__ 98 struct cfattach ubsec_ca = { 99 sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 100 }; 101 102 struct cfdriver ubsec_cd = { 103 0, "ubsec", DV_DULL 104 }; 105 #else 106 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 107 ubsec_detach, NULL); 108 extern struct cfdriver ubsec_cd; 109 #endif 110 111 /* patchable */ 112 #ifdef UBSEC_DEBUG 113 extern int ubsec_debug; 114 int ubsec_debug=1; 115 #endif 116 117 static int ubsec_intr(void *); 118 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *); 119 static int ubsec_freesession(void*, u_int64_t); 120 static int ubsec_process(void*, struct cryptop *, int hint); 121 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 122 static void ubsec_feed(struct ubsec_softc *); 123 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 124 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 125 static void ubsec_feed2(struct ubsec_softc *); 126 static void ubsec_feed4(struct ubsec_softc *); 127 #ifndef UBSEC_NO_RNG 128 static void ubsec_rng(void *); 129 static void ubsec_rng_locked(void *); 130 static void ubsec_rng_get(size_t, void *); 131 #endif /* UBSEC_NO_RNG */ 132 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 133 struct ubsec_dma_alloc *, int); 134 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 135 static int ubsec_dmamap_aligned(bus_dmamap_t); 136 137 static int ubsec_kprocess(void*, struct cryptkop *, int); 138 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, 139 struct cryptkop *, int); 140 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, 141 struct cryptkop *, int); 142 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, 143 struct cryptkop *, int); 144 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 145 static int ubsec_ksigbits(struct crparam *); 146 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 147 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 148 149 #ifdef UBSEC_DEBUG 150 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 151 static void ubsec_dump_mcr(struct ubsec_mcr *); 152 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *); 153 #endif 154 155 #define READ_REG(sc,r) \ 156 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 157 158 #define WRITE_REG(sc,reg,val) \ 159 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 160 161 #define SWAP32(x) (x) = htole32(ntohl((x))) 162 #ifndef HTOLE32 163 #define HTOLE32(x) (x) = htole32(x) 164 #endif 165 166 struct ubsec_stats ubsecstats; 167 168 /* 169 * ubsec_maxbatch controls the number of crypto ops to voluntarily 170 * collect into one submission to the hardware. This batching happens 171 * when ops are dispatched from the crypto subsystem with a hint that 172 * more are to follow immediately. These ops must also not be marked 173 * with a ``no delay'' flag. 174 */ 175 static int ubsec_maxbatch = 1; 176 177 /* 178 * ubsec_maxaggr controls the number of crypto ops to submit to the 179 * hardware as a unit. This aggregation reduces the number of interrupts 180 * to the host at the expense of increased latency (for all but the last 181 * operation). For network traffic setting this to one yields the highest 182 * performance but at the expense of more interrupt processing. 183 */ 184 static int ubsec_maxaggr = 1; 185 186 static const struct ubsec_product { 187 pci_vendor_id_t ubsec_vendor; 188 pci_product_id_t ubsec_product; 189 int ubsec_flags; 190 int ubsec_statmask; 191 int ubsec_maxaggr; 192 const char *ubsec_name; 193 } ubsec_products[] = { 194 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501, 195 0, 196 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 197 UBS_MIN_AGGR, 198 "Bluesteel 5501" 199 }, 200 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601, 201 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 202 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 203 UBS_MIN_AGGR, 204 "Bluesteel 5601" 205 }, 206 207 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801, 208 0, 209 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 210 UBS_MIN_AGGR, 211 "Broadcom BCM5801" 212 }, 213 214 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802, 215 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 216 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 217 UBS_MIN_AGGR, 218 "Broadcom BCM5802" 219 }, 220 221 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805, 222 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 223 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 224 UBS_MIN_AGGR, 225 "Broadcom BCM5805" 226 }, 227 228 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820, 229 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 230 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 231 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 232 UBS_MIN_AGGR, 233 "Broadcom BCM5820" 234 }, 235 236 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821, 237 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 238 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 239 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 240 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 241 UBS_MIN_AGGR, 242 "Broadcom BCM5821" 243 }, 244 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K, 245 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 246 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 247 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 248 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 249 UBS_MIN_AGGR, 250 "Sun Crypto Accelerator 1000" 251 }, 252 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821, 253 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 254 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 255 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 256 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 257 UBS_MIN_AGGR, 258 "Broadcom BCM5821 (Sun)" 259 }, 260 261 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822, 262 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 263 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 264 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 265 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 266 UBS_MIN_AGGR, 267 "Broadcom BCM5822" 268 }, 269 270 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823, 271 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 272 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 273 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 274 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 275 UBS_MIN_AGGR, 276 "Broadcom BCM5823" 277 }, 278 279 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825, 280 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 281 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 282 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 283 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 284 UBS_MIN_AGGR, 285 "Broadcom BCM5825" 286 }, 287 288 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860, 289 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 290 UBS_FLAGS_LONGCTX | 291 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 292 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 293 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 294 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 295 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 296 UBS_MAX_AGGR, 297 "Broadcom BCM5860" 298 }, 299 300 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861, 301 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 302 UBS_FLAGS_LONGCTX | 303 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 304 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 305 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 306 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 307 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 308 UBS_MAX_AGGR, 309 "Broadcom BCM5861" 310 }, 311 312 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862, 313 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 314 UBS_FLAGS_LONGCTX | 315 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 316 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 317 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 318 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 319 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 320 UBS_MAX_AGGR, 321 "Broadcom BCM5862" 322 }, 323 324 { 0, 0, 325 0, 326 0, 327 0, 328 NULL 329 } 330 }; 331 332 static const struct ubsec_product * 333 ubsec_lookup(const struct pci_attach_args *pa) 334 { 335 const struct ubsec_product *up; 336 337 for (up = ubsec_products; up->ubsec_name != NULL; up++) { 338 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor && 339 PCI_PRODUCT(pa->pa_id) == up->ubsec_product) 340 return (up); 341 } 342 return (NULL); 343 } 344 345 static int 346 ubsec_probe(device_t parent, cfdata_t match, void *aux) 347 { 348 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 349 350 if (ubsec_lookup(pa) != NULL) 351 return (1); 352 353 return (0); 354 } 355 356 static void 357 ubsec_attach(device_t parent, device_t self, void *aux) 358 { 359 struct ubsec_softc *sc = device_private(self); 360 struct pci_attach_args *pa = aux; 361 const struct ubsec_product *up; 362 pci_chipset_tag_t pc = pa->pa_pc; 363 pci_intr_handle_t ih; 364 const char *intrstr = NULL; 365 pcireg_t memtype; 366 struct ubsec_dma *dmap; 367 u_int32_t cmd, i; 368 char intrbuf[PCI_INTRSTR_LEN]; 369 370 sc->sc_dev = self; 371 sc->sc_pct = pc; 372 373 up = ubsec_lookup(pa); 374 if (up == NULL) { 375 printf("\n"); 376 panic("ubsec_attach: impossible"); 377 } 378 379 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1); 380 381 SIMPLEQ_INIT(&sc->sc_queue); 382 SIMPLEQ_INIT(&sc->sc_qchip); 383 SIMPLEQ_INIT(&sc->sc_queue2); 384 SIMPLEQ_INIT(&sc->sc_qchip2); 385 SIMPLEQ_INIT(&sc->sc_queue4); 386 SIMPLEQ_INIT(&sc->sc_qchip4); 387 SIMPLEQ_INIT(&sc->sc_q2free); 388 389 sc->sc_flags = up->ubsec_flags; 390 sc->sc_statmask = up->ubsec_statmask; 391 sc->sc_maxaggr = up->ubsec_maxaggr; 392 393 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 394 cmd |= PCI_COMMAND_MASTER_ENABLE; 395 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 396 397 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR); 398 if (pci_mapreg_map(pa, BS_BAR, memtype, 0, 399 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) { 400 aprint_error_dev(self, "can't find mem space"); 401 return; 402 } 403 404 sc->sc_dmat = pa->pa_dmat; 405 406 if (pci_intr_map(pa, &ih)) { 407 aprint_error_dev(self, "couldn't map interrupt\n"); 408 return; 409 } 410 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 411 sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, ubsec_intr, sc, 412 device_xname(self)); 413 if (sc->sc_ih == NULL) { 414 aprint_error_dev(self, "couldn't establish interrupt"); 415 if (intrstr != NULL) 416 aprint_error(" at %s", intrstr); 417 aprint_error("\n"); 418 return; 419 } 420 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 421 422 sc->sc_cid = crypto_get_driverid(0); 423 if (sc->sc_cid < 0) { 424 aprint_error_dev(self, "couldn't get crypto driver id\n"); 425 pci_intr_disestablish(pc, sc->sc_ih); 426 return; 427 } 428 429 sc->sc_rng_need = RND_POOLBITS / NBBY; 430 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 431 432 SIMPLEQ_INIT(&sc->sc_freequeue); 433 dmap = sc->sc_dmaa; 434 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 435 struct ubsec_q *q; 436 437 q = malloc(sizeof(struct ubsec_q), M_DEVBUF, M_ZERO|M_WAITOK); 438 439 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 440 &dmap->d_alloc, 0)) { 441 aprint_error_dev(self, "can't allocate dma buffers\n"); 442 free(q, M_DEVBUF); 443 break; 444 } 445 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 446 447 q->q_dma = dmap; 448 sc->sc_queuea[i] = q; 449 450 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 451 } 452 453 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 454 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 455 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 456 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 457 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 458 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 459 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 460 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 461 if (sc->sc_flags & UBS_FLAGS_AES) { 462 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 463 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 464 } 465 466 /* 467 * Reset Broadcom chip 468 */ 469 ubsec_reset_board(sc); 470 471 /* 472 * Init Broadcom specific PCI settings 473 */ 474 ubsec_init_pciregs(pa); 475 476 /* 477 * Init Broadcom chip 478 */ 479 ubsec_init_board(sc); 480 481 #ifndef UBSEC_NO_RNG 482 if (sc->sc_flags & UBS_FLAGS_RNG) { 483 if (sc->sc_flags & UBS_FLAGS_RNG4) 484 sc->sc_statmask |= BS_STAT_MCR4_DONE; 485 else 486 sc->sc_statmask |= BS_STAT_MCR2_DONE; 487 488 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 489 &sc->sc_rng.rng_q.q_mcr, 0)) 490 goto skip_rng; 491 492 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 493 &sc->sc_rng.rng_q.q_ctx, 0)) { 494 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 495 goto skip_rng; 496 } 497 498 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 499 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 500 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 501 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 502 goto skip_rng; 503 } 504 505 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc); 506 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev), 507 RND_TYPE_RNG, 508 RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB); 509 if (hz >= 100) 510 sc->sc_rnghz = hz / 100; 511 else 512 sc->sc_rnghz = 1; 513 #ifdef __OpenBSD__ 514 timeout_set(&sc->sc_rngto, ubsec_rng, sc); 515 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 516 #else 517 callout_init(&sc->sc_rngto, 0); 518 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc); 519 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 520 #endif 521 skip_rng: 522 if (sc->sc_rnghz) 523 aprint_normal_dev(self, 524 "random number generator enabled\n"); 525 else 526 aprint_error_dev(self, 527 "WARNING: random number generator disabled\n"); 528 } 529 #endif /* UBSEC_NO_RNG */ 530 531 if (sc->sc_flags & UBS_FLAGS_KEY) { 532 sc->sc_statmask |= BS_STAT_MCR2_DONE; 533 534 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 535 ubsec_kprocess, sc); 536 #if 0 537 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 538 ubsec_kprocess, sc); 539 #endif 540 } 541 } 542 543 static int 544 ubsec_detach(device_t self, int flags) 545 { 546 struct ubsec_softc *sc = device_private(self); 547 struct ubsec_q *q, *qtmp; 548 volatile u_int32_t ctrl; 549 550 /* disable interrupts */ 551 /* XXX wait/abort current ops? where is DMAERR enabled? */ 552 ctrl = READ_REG(sc, BS_CTRL); 553 554 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR); 555 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 556 ctrl &= ~BS_CTRL_MCR4INT; 557 558 WRITE_REG(sc, BS_CTRL, ctrl); 559 560 #ifndef UBSEC_NO_RNG 561 if (sc->sc_flags & UBS_FLAGS_RNG) { 562 callout_halt(&sc->sc_rngto, NULL); 563 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 564 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 565 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 566 rnd_detach_source(&sc->sc_rnd_source); 567 } 568 #endif /* UBSEC_NO_RNG */ 569 570 crypto_unregister_all(sc->sc_cid); 571 572 mutex_spin_enter(&sc->sc_mtx); 573 574 ubsec_totalreset(sc); /* XXX leaves the chip running */ 575 576 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) { 577 ubsec_dma_free(sc, &q->q_dma->d_alloc); 578 if (q->q_src_map != NULL) 579 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 580 if (q->q_cached_dst_map != NULL) 581 bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map); 582 free(q, M_DEVBUF); 583 } 584 585 mutex_spin_exit(&sc->sc_mtx); 586 587 if (sc->sc_ih != NULL) { 588 pci_intr_disestablish(sc->sc_pct, sc->sc_ih); 589 sc->sc_ih = NULL; 590 } 591 592 if (sc->sc_memsize != 0) { 593 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize); 594 sc->sc_memsize = 0; 595 } 596 597 return 0; 598 } 599 600 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto"); 601 602 #ifdef _MODULE 603 #include "ioconf.c" 604 #endif 605 606 static int 607 ubsec_modcmd(modcmd_t cmd, void *data) 608 { 609 int error = 0; 610 611 switch (cmd) { 612 case MODULE_CMD_INIT: 613 #ifdef _MODULE 614 error = config_init_component(cfdriver_ioconf_ubsec, 615 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 616 #endif 617 return error; 618 case MODULE_CMD_FINI: 619 #ifdef _MODULE 620 error = config_fini_component(cfdriver_ioconf_ubsec, 621 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 622 #endif 623 return error; 624 default: 625 return ENOTTY; 626 } 627 } 628 629 SYSCTL_SETUP(ubsec_sysctl_init, "ubsec sysctl") 630 { 631 const struct sysctlnode *node = NULL; 632 633 sysctl_createv(clog, 0, NULL, &node, 634 CTLFLAG_PERMANENT, 635 CTLTYPE_NODE, "ubsec", 636 SYSCTL_DESCR("ubsec opetions"), 637 NULL, 0, NULL, 0, 638 CTL_HW, CTL_CREATE, CTL_EOL); 639 sysctl_createv(clog, 0, &node, NULL, 640 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 641 CTLTYPE_INT, "maxbatch", 642 SYSCTL_DESCR("max ops to batch w/o interrupt"), 643 NULL, 0, &ubsec_maxbatch, 0, 644 CTL_CREATE, CTL_EOL); 645 sysctl_createv(clog, 0, &node, NULL, 646 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 647 CTLTYPE_INT, "maxaggr", 648 SYSCTL_DESCR("max ops to aggregate under one interrupt"), 649 NULL, 0, &ubsec_maxaggr, 0, 650 CTL_CREATE, CTL_EOL); 651 652 return; 653 } 654 655 /* 656 * UBSEC Interrupt routine 657 */ 658 static int 659 ubsec_intr(void *arg) 660 { 661 struct ubsec_softc *sc = arg; 662 volatile u_int32_t stat; 663 struct ubsec_q *q; 664 struct ubsec_dma *dmap; 665 int flags; 666 int npkts = 0, i; 667 668 mutex_spin_enter(&sc->sc_mtx); 669 stat = READ_REG(sc, BS_STAT); 670 stat &= sc->sc_statmask; 671 if (stat == 0) { 672 mutex_spin_exit(&sc->sc_mtx); 673 return (0); 674 } 675 676 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 677 678 /* 679 * Check to see if we have any packets waiting for us 680 */ 681 if ((stat & BS_STAT_MCR1_DONE)) { 682 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 683 q = SIMPLEQ_FIRST(&sc->sc_qchip); 684 dmap = q->q_dma; 685 686 if ((dmap->d_dma->d_mcr.mcr_flags 687 & htole16(UBS_MCR_DONE)) == 0) 688 break; 689 690 q = SIMPLEQ_FIRST(&sc->sc_qchip); 691 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 692 693 npkts = q->q_nstacked_mcrs; 694 sc->sc_nqchip -= 1+npkts; 695 /* 696 * search for further sc_qchip ubsec_q's that share 697 * the same MCR, and complete them too, they must be 698 * at the top. 699 */ 700 for (i = 0; i < npkts; i++) { 701 if(q->q_stacked_mcr[i]) 702 ubsec_callback(sc, q->q_stacked_mcr[i]); 703 else 704 break; 705 } 706 ubsec_callback(sc, q); 707 } 708 709 /* 710 * Don't send any more packet to chip if there has been 711 * a DMAERR. 712 */ 713 if (!(stat & BS_STAT_DMAERR)) 714 ubsec_feed(sc); 715 } 716 717 /* 718 * Check to see if we have any key setups/rng's waiting for us 719 */ 720 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 721 (stat & BS_STAT_MCR2_DONE)) { 722 struct ubsec_q2 *q2; 723 struct ubsec_mcr *mcr; 724 725 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 726 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 727 728 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 729 0, q2->q_mcr.dma_map->dm_mapsize, 730 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 731 732 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 733 734 /* A bug in new devices requires to swap this field */ 735 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 736 flags = htole16(mcr->mcr_flags); 737 else 738 flags = mcr->mcr_flags; 739 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 740 bus_dmamap_sync(sc->sc_dmat, 741 q2->q_mcr.dma_map, 0, 742 q2->q_mcr.dma_map->dm_mapsize, 743 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 744 break; 745 } 746 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 747 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next); 748 ubsec_callback2(sc, q2); 749 /* 750 * Don't send any more packet to chip if there has been 751 * a DMAERR. 752 */ 753 if (!(stat & BS_STAT_DMAERR)) 754 ubsec_feed2(sc); 755 } 756 } 757 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) { 758 struct ubsec_q2 *q2; 759 struct ubsec_mcr *mcr; 760 761 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) { 762 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4); 763 764 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 765 0, q2->q_mcr.dma_map->dm_mapsize, 766 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 767 768 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 769 770 /* A bug in new devices requires to swap this field */ 771 flags = htole16(mcr->mcr_flags); 772 773 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 774 bus_dmamap_sync(sc->sc_dmat, 775 q2->q_mcr.dma_map, 0, 776 q2->q_mcr.dma_map->dm_mapsize, 777 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 778 break; 779 } 780 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next); 781 ubsec_callback2(sc, q2); 782 /* 783 * Don't send any more packet to chip if there has been 784 * a DMAERR. 785 */ 786 if (!(stat & BS_STAT_DMAERR)) 787 ubsec_feed4(sc); 788 } 789 } 790 791 /* 792 * Check to see if we got any DMA Error 793 */ 794 if (stat & BS_STAT_DMAERR) { 795 #ifdef UBSEC_DEBUG 796 if (ubsec_debug) { 797 volatile u_int32_t a = READ_REG(sc, BS_ERR); 798 799 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev), 800 (a & BS_ERR_READ) ? "read" : "write", 801 a & BS_ERR_ADDR); 802 } 803 #endif /* UBSEC_DEBUG */ 804 ubsecstats.hst_dmaerr++; 805 ubsec_totalreset(sc); 806 ubsec_feed(sc); 807 } 808 809 if (sc->sc_needwakeup) { /* XXX check high watermark */ 810 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 811 #ifdef UBSEC_DEBUG 812 if (ubsec_debug) 813 printf("%s: wakeup crypto (%x)\n", 814 device_xname(sc->sc_dev), sc->sc_needwakeup); 815 #endif /* UBSEC_DEBUG */ 816 sc->sc_needwakeup &= ~wkeup; 817 crypto_unblock(sc->sc_cid, wkeup); 818 } 819 mutex_spin_exit(&sc->sc_mtx); 820 return (1); 821 } 822 823 /* 824 * ubsec_feed() - aggregate and post requests to chip 825 * OpenBSD comments: 826 * It is assumed that the caller set splnet() 827 */ 828 static void 829 ubsec_feed(struct ubsec_softc *sc) 830 { 831 struct ubsec_q *q, *q2; 832 int npkts, i; 833 void *v; 834 u_int32_t stat; 835 #ifdef UBSEC_DEBUG 836 static int max; 837 #endif /* UBSEC_DEBUG */ 838 839 npkts = sc->sc_nqueue; 840 if (npkts > ubsecstats.hst_maxqueue) 841 ubsecstats.hst_maxqueue = npkts; 842 if (npkts < 2) 843 goto feed1; 844 845 /* 846 * Decide how many ops to combine in a single MCR. We cannot 847 * aggregate more than UBS_MAX_AGGR because this is the number 848 * of slots defined in the data structure. Otherwise we clamp 849 * based on the tunable parameter ubsec_maxaggr. Note that 850 * aggregation can happen in two ways: either by batching ops 851 * from above or because the h/w backs up and throttles us. 852 * Aggregating ops reduces the number of interrupts to the host 853 * but also (potentially) increases the latency for processing 854 * completed ops as we only get an interrupt when all aggregated 855 * ops have completed. 856 */ 857 if (npkts > sc->sc_maxaggr) 858 npkts = sc->sc_maxaggr; 859 if (npkts > ubsec_maxaggr) 860 npkts = ubsec_maxaggr; 861 if (npkts > ubsecstats.hst_maxbatch) 862 ubsecstats.hst_maxbatch = npkts; 863 if (npkts < 2) 864 goto feed1; 865 ubsecstats.hst_totbatch += npkts-1; 866 867 if ((stat = READ_REG(sc, BS_STAT)) 868 & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 869 if (stat & BS_STAT_DMAERR) { 870 ubsec_totalreset(sc); 871 ubsecstats.hst_dmaerr++; 872 } else { 873 ubsecstats.hst_mcr1full++; 874 } 875 return; 876 } 877 878 #ifdef UBSEC_DEBUG 879 if (ubsec_debug) 880 printf("merging %d records\n", npkts); 881 /* XXX temporary aggregation statistics reporting code */ 882 if (max < npkts) { 883 max = npkts; 884 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), 885 max); 886 } 887 #endif /* UBSEC_DEBUG */ 888 889 q = SIMPLEQ_FIRST(&sc->sc_queue); 890 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 891 --sc->sc_nqueue; 892 893 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 894 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 895 if (q->q_dst_map != NULL) 896 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 897 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 898 899 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 900 901 for (i = 0; i < q->q_nstacked_mcrs; i++) { 902 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 903 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 904 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 905 if (q2->q_dst_map != NULL) 906 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 907 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 908 q2= SIMPLEQ_FIRST(&sc->sc_queue); 909 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next); 910 --sc->sc_nqueue; 911 912 v = ((void *)&q2->q_dma->d_dma->d_mcr); 913 v = (char*)v + (sizeof(struct ubsec_mcr) - 914 sizeof(struct ubsec_mcr_add)); 915 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, 916 sizeof(struct ubsec_mcr_add)); 917 q->q_stacked_mcr[i] = q2; 918 } 919 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 920 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 921 sc->sc_nqchip += npkts; 922 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 923 ubsecstats.hst_maxqchip = sc->sc_nqchip; 924 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 925 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 926 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 927 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 928 offsetof(struct ubsec_dmachunk, d_mcr)); 929 return; 930 931 feed1: 932 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 933 if ((stat = READ_REG(sc, BS_STAT)) 934 & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 935 if (stat & BS_STAT_DMAERR) { 936 ubsec_totalreset(sc); 937 ubsecstats.hst_dmaerr++; 938 } else { 939 ubsecstats.hst_mcr1full++; 940 } 941 break; 942 } 943 944 q = SIMPLEQ_FIRST(&sc->sc_queue); 945 946 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 947 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 948 if (q->q_dst_map != NULL) 949 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 950 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 951 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 952 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 953 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 954 955 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 956 offsetof(struct ubsec_dmachunk, d_mcr)); 957 #ifdef UBSEC_DEBUG 958 if (ubsec_debug) 959 printf("feed: q->chip %p %08x stat %08x\n", 960 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr, 961 stat); 962 #endif /* UBSEC_DEBUG */ 963 q = SIMPLEQ_FIRST(&sc->sc_queue); 964 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 965 --sc->sc_nqueue; 966 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 967 sc->sc_nqchip++; 968 } 969 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 970 ubsecstats.hst_maxqchip = sc->sc_nqchip; 971 } 972 973 /* 974 * Allocate a new 'session' and return an encoded session id. 'sidp' 975 * contains our registration id, and should contain an encoded session 976 * id on successful allocation. 977 */ 978 static int 979 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 980 { 981 struct cryptoini *c, *encini = NULL, *macini = NULL; 982 struct ubsec_softc *sc; 983 struct ubsec_session *ses = NULL; 984 MD5_CTX md5ctx; 985 SHA1_CTX sha1ctx; 986 int i, sesn; 987 988 sc = arg; 989 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/); 990 991 if (sidp == NULL || cri == NULL || sc == NULL) 992 return (EINVAL); 993 994 for (c = cri; c != NULL; c = c->cri_next) { 995 if (c->cri_alg == CRYPTO_MD5_HMAC_96 || 996 c->cri_alg == CRYPTO_SHA1_HMAC_96) { 997 if (macini) 998 return (EINVAL); 999 macini = c; 1000 } else if (c->cri_alg == CRYPTO_DES_CBC || 1001 c->cri_alg == CRYPTO_3DES_CBC || 1002 c->cri_alg == CRYPTO_AES_CBC) { 1003 if (encini) 1004 return (EINVAL); 1005 encini = c; 1006 } else 1007 return (EINVAL); 1008 } 1009 if (encini == NULL && macini == NULL) 1010 return (EINVAL); 1011 1012 if (encini && encini->cri_alg == CRYPTO_AES_CBC) { 1013 switch (encini->cri_klen) { 1014 case 128: 1015 case 192: 1016 case 256: 1017 break; 1018 default: 1019 return (EINVAL); 1020 } 1021 } 1022 1023 if (sc->sc_sessions == NULL) { 1024 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 1025 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1026 if (ses == NULL) 1027 return (ENOMEM); 1028 sesn = 0; 1029 sc->sc_nsessions = 1; 1030 } else { 1031 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 1032 if (sc->sc_sessions[sesn].ses_used == 0) { 1033 ses = &sc->sc_sessions[sesn]; 1034 break; 1035 } 1036 } 1037 1038 if (ses == NULL) { 1039 sesn = sc->sc_nsessions; 1040 ses = (struct ubsec_session *)malloc((sesn + 1) * 1041 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1042 if (ses == NULL) 1043 return (ENOMEM); 1044 memcpy(ses, sc->sc_sessions, sesn * 1045 sizeof(struct ubsec_session)); 1046 memset(sc->sc_sessions, 0, sesn * 1047 sizeof(struct ubsec_session)); 1048 free(sc->sc_sessions, M_DEVBUF); 1049 sc->sc_sessions = ses; 1050 ses = &sc->sc_sessions[sesn]; 1051 sc->sc_nsessions++; 1052 } 1053 } 1054 1055 memset(ses, 0, sizeof(struct ubsec_session)); 1056 ses->ses_used = 1; 1057 if (encini) { 1058 /* get an IV, network byte order */ 1059 #ifdef __NetBSD__ 1060 cprng_fast(ses->ses_iv, sizeof(ses->ses_iv)); 1061 #else 1062 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv)); 1063 #endif 1064 1065 /* Go ahead and compute key in ubsec's byte order */ 1066 if (encini->cri_alg == CRYPTO_AES_CBC) { 1067 memcpy(ses->ses_key, encini->cri_key, 1068 encini->cri_klen / 8); 1069 } 1070 if (encini->cri_alg == CRYPTO_DES_CBC) { 1071 memcpy(&ses->ses_key[0], encini->cri_key, 8); 1072 memcpy(&ses->ses_key[2], encini->cri_key, 8); 1073 memcpy(&ses->ses_key[4], encini->cri_key, 8); 1074 } else 1075 memcpy(ses->ses_key, encini->cri_key, 24); 1076 1077 SWAP32(ses->ses_key[0]); 1078 SWAP32(ses->ses_key[1]); 1079 SWAP32(ses->ses_key[2]); 1080 SWAP32(ses->ses_key[3]); 1081 SWAP32(ses->ses_key[4]); 1082 SWAP32(ses->ses_key[5]); 1083 } 1084 1085 if (macini) { 1086 for (i = 0; i < macini->cri_klen / 8; i++) 1087 macini->cri_key[i] ^= HMAC_IPAD_VAL; 1088 1089 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1090 MD5Init(&md5ctx); 1091 MD5Update(&md5ctx, macini->cri_key, 1092 macini->cri_klen / 8); 1093 MD5Update(&md5ctx, hmac_ipad_buffer, 1094 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1095 memcpy(ses->ses_hminner, md5ctx.state, 1096 sizeof(md5ctx.state)); 1097 } else { 1098 SHA1Init(&sha1ctx); 1099 SHA1Update(&sha1ctx, macini->cri_key, 1100 macini->cri_klen / 8); 1101 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1102 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1103 memcpy(ses->ses_hminner, sha1ctx.state, 1104 sizeof(sha1ctx.state)); 1105 } 1106 1107 for (i = 0; i < macini->cri_klen / 8; i++) 1108 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1109 1110 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1111 MD5Init(&md5ctx); 1112 MD5Update(&md5ctx, macini->cri_key, 1113 macini->cri_klen / 8); 1114 MD5Update(&md5ctx, hmac_opad_buffer, 1115 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1116 memcpy(ses->ses_hmouter, md5ctx.state, 1117 sizeof(md5ctx.state)); 1118 } else { 1119 SHA1Init(&sha1ctx); 1120 SHA1Update(&sha1ctx, macini->cri_key, 1121 macini->cri_klen / 8); 1122 SHA1Update(&sha1ctx, hmac_opad_buffer, 1123 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1124 memcpy(ses->ses_hmouter, sha1ctx.state, 1125 sizeof(sha1ctx.state)); 1126 } 1127 1128 for (i = 0; i < macini->cri_klen / 8; i++) 1129 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1130 } 1131 1132 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn); 1133 return (0); 1134 } 1135 1136 /* 1137 * Deallocate a session. 1138 */ 1139 static int 1140 ubsec_freesession(void *arg, u_int64_t tid) 1141 { 1142 struct ubsec_softc *sc; 1143 int session; 1144 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1145 1146 sc = arg; 1147 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/); 1148 1149 session = UBSEC_SESSION(sid); 1150 if (session >= sc->sc_nsessions) 1151 return (EINVAL); 1152 1153 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 1154 return (0); 1155 } 1156 1157 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */ 1158 static void 1159 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, 1160 int error) 1161 { 1162 struct ubsec_operand *op = arg; 1163 1164 KASSERT(nsegs <= UBS_MAX_SCATTER 1165 /*, ("Too many DMA segments returned when mapping operand")*/); 1166 #ifdef UBSEC_DEBUG 1167 if (ubsec_debug) 1168 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 1169 (u_int) mapsize, nsegs); 1170 #endif 1171 op->mapsize = mapsize; 1172 op->nsegs = nsegs; 1173 memcpy(op->segs, seg, nsegs * sizeof (seg[0])); 1174 } 1175 #endif 1176 1177 static int 1178 ubsec_process(void *arg, struct cryptop *crp, int hint) 1179 { 1180 struct ubsec_q *q = NULL; 1181 #ifdef __OpenBSD__ 1182 int card; 1183 #endif 1184 int err = 0, i, j, nicealign; 1185 struct ubsec_softc *sc; 1186 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1187 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1188 int sskip, dskip, stheend, dtheend; 1189 int16_t coffset; 1190 struct ubsec_session *ses, key; 1191 struct ubsec_dma *dmap = NULL; 1192 u_int16_t flags = 0; 1193 int ivlen = 0, keylen = 0; 1194 1195 sc = arg; 1196 KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/); 1197 1198 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1199 ubsecstats.hst_invalid++; 1200 return (EINVAL); 1201 } 1202 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1203 ubsecstats.hst_badsession++; 1204 return (EINVAL); 1205 } 1206 1207 mutex_spin_enter(&sc->sc_mtx); 1208 1209 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1210 ubsecstats.hst_queuefull++; 1211 sc->sc_needwakeup |= CRYPTO_SYMQ; 1212 mutex_spin_exit(&sc->sc_mtx); 1213 return(ERESTART); 1214 } 1215 1216 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1217 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next); 1218 mutex_spin_exit(&sc->sc_mtx); 1219 1220 dmap = q->q_dma; /* Save dma pointer */ 1221 /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */ 1222 memset(q, 0, offsetof(struct ubsec_q, q_src_map)); 1223 memset(&key, 0, sizeof(key)); 1224 1225 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1226 q->q_dma = dmap; 1227 ses = &sc->sc_sessions[q->q_sesn]; 1228 1229 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1230 q->q_src_m = (struct mbuf *)crp->crp_buf; 1231 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1232 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1233 q->q_src_io = (struct uio *)crp->crp_buf; 1234 q->q_dst_io = (struct uio *)crp->crp_buf; 1235 } else { 1236 ubsecstats.hst_badflags++; 1237 err = EINVAL; 1238 goto errout; /* XXX we don't handle contiguous blocks! */ 1239 } 1240 1241 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr)); 1242 1243 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1244 dmap->d_dma->d_mcr.mcr_flags = 0; 1245 q->q_crp = crp; 1246 1247 crd1 = crp->crp_desc; 1248 if (crd1 == NULL) { 1249 ubsecstats.hst_nodesc++; 1250 err = EINVAL; 1251 goto errout; 1252 } 1253 crd2 = crd1->crd_next; 1254 1255 if (crd2 == NULL) { 1256 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1257 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) { 1258 maccrd = crd1; 1259 enccrd = NULL; 1260 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1261 crd1->crd_alg == CRYPTO_3DES_CBC || 1262 crd1->crd_alg == CRYPTO_AES_CBC) { 1263 maccrd = NULL; 1264 enccrd = crd1; 1265 } else { 1266 ubsecstats.hst_badalg++; 1267 err = EINVAL; 1268 goto errout; 1269 } 1270 } else { 1271 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1272 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) && 1273 (crd2->crd_alg == CRYPTO_DES_CBC || 1274 crd2->crd_alg == CRYPTO_3DES_CBC || 1275 crd2->crd_alg == CRYPTO_AES_CBC) && 1276 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1277 maccrd = crd1; 1278 enccrd = crd2; 1279 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1280 crd1->crd_alg == CRYPTO_3DES_CBC || 1281 crd1->crd_alg == CRYPTO_AES_CBC) && 1282 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 1283 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) && 1284 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1285 enccrd = crd1; 1286 maccrd = crd2; 1287 } else { 1288 /* 1289 * We cannot order the ubsec as requested 1290 */ 1291 ubsecstats.hst_badalg++; 1292 err = EINVAL; 1293 goto errout; 1294 } 1295 } 1296 1297 if (enccrd) { 1298 if (enccrd->crd_alg == CRYPTO_AES_CBC) { 1299 if ((sc->sc_flags & UBS_FLAGS_AES) == 0) { 1300 /* 1301 * We cannot order the ubsec as requested 1302 */ 1303 ubsecstats.hst_badalg++; 1304 err = EINVAL; 1305 goto errout; 1306 } 1307 flags |= htole16(UBS_PKTCTX_ENC_AES); 1308 switch (enccrd->crd_klen) { 1309 case 128: 1310 case 192: 1311 case 256: 1312 keylen = enccrd->crd_klen / 8; 1313 break; 1314 default: 1315 err = EINVAL; 1316 goto errout; 1317 } 1318 ivlen = 16; 1319 } else { 1320 flags |= htole16(UBS_PKTCTX_ENC_3DES); 1321 ivlen = 8; 1322 keylen = 24; 1323 } 1324 1325 encoffset = enccrd->crd_skip; 1326 1327 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1328 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1329 1330 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1331 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1332 else { 1333 for (i = 0; i < (ivlen / 4); i++) 1334 key.ses_iv[i] = ses->ses_iv[i]; 1335 } 1336 1337 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1338 if (crp->crp_flags & CRYPTO_F_IMBUF) 1339 m_copyback(q->q_src_m, 1340 enccrd->crd_inject, 1341 ivlen, (void *)key.ses_iv); 1342 else if (crp->crp_flags & CRYPTO_F_IOV) 1343 cuio_copyback(q->q_src_io, 1344 enccrd->crd_inject, 1345 ivlen, (void *)key.ses_iv); 1346 } 1347 } else { 1348 flags |= htole16(UBS_PKTCTX_INBOUND); 1349 1350 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1351 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1352 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1353 m_copydata(q->q_src_m, enccrd->crd_inject, 1354 ivlen, (void *)key.ses_iv); 1355 else if (crp->crp_flags & CRYPTO_F_IOV) 1356 cuio_copydata(q->q_src_io, 1357 enccrd->crd_inject, 8, 1358 (void *)key.ses_iv); 1359 } 1360 1361 for (i = 0; i < (keylen / 4); i++) 1362 key.ses_key[i] = ses->ses_key[i]; 1363 for (i = 0; i < (ivlen / 4); i++) 1364 SWAP32(key.ses_iv[i]); 1365 } 1366 1367 if (maccrd) { 1368 macoffset = maccrd->crd_skip; 1369 1370 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96) 1371 flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1372 else 1373 flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1374 1375 for (i = 0; i < 5; i++) { 1376 key.ses_hminner[i] = ses->ses_hminner[i]; 1377 key.ses_hmouter[i] = ses->ses_hmouter[i]; 1378 1379 HTOLE32(key.ses_hminner[i]); 1380 HTOLE32(key.ses_hmouter[i]); 1381 } 1382 } 1383 1384 if (enccrd && maccrd) { 1385 /* 1386 * ubsec cannot handle packets where the end of encryption 1387 * and authentication are not the same, or where the 1388 * encrypted part begins before the authenticated part. 1389 */ 1390 if ((encoffset + enccrd->crd_len) != 1391 (macoffset + maccrd->crd_len)) { 1392 ubsecstats.hst_lenmismatch++; 1393 err = EINVAL; 1394 goto errout; 1395 } 1396 if (enccrd->crd_skip < maccrd->crd_skip) { 1397 ubsecstats.hst_skipmismatch++; 1398 err = EINVAL; 1399 goto errout; 1400 } 1401 sskip = maccrd->crd_skip; 1402 cpskip = dskip = enccrd->crd_skip; 1403 stheend = maccrd->crd_len; 1404 dtheend = enccrd->crd_len; 1405 coffset = enccrd->crd_skip - maccrd->crd_skip; 1406 cpoffset = cpskip + dtheend; 1407 #ifdef UBSEC_DEBUG 1408 if (ubsec_debug) { 1409 printf("mac: skip %d, len %d, inject %d\n", 1410 maccrd->crd_skip, maccrd->crd_len, 1411 maccrd->crd_inject); 1412 printf("enc: skip %d, len %d, inject %d\n", 1413 enccrd->crd_skip, enccrd->crd_len, 1414 enccrd->crd_inject); 1415 printf("src: skip %d, len %d\n", sskip, stheend); 1416 printf("dst: skip %d, len %d\n", dskip, dtheend); 1417 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1418 coffset, stheend, cpskip, cpoffset); 1419 } 1420 #endif 1421 } else { 1422 cpskip = dskip = sskip = macoffset + encoffset; 1423 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1424 cpoffset = cpskip + dtheend; 1425 coffset = 0; 1426 } 1427 1428 if (q->q_src_map == NULL) { 1429 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */ 1430 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER, 1431 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) { 1432 err = ENOMEM; 1433 goto errout; 1434 } 1435 } 1436 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1437 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1438 q->q_src_m, BUS_DMA_NOWAIT) != 0) { 1439 ubsecstats.hst_noload++; 1440 err = ENOMEM; 1441 goto errout; 1442 } 1443 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1444 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1445 q->q_src_io, BUS_DMA_NOWAIT) != 0) { 1446 ubsecstats.hst_noload++; 1447 err = ENOMEM; 1448 goto errout; 1449 } 1450 } 1451 nicealign = ubsec_dmamap_aligned(q->q_src_map); 1452 1453 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1454 1455 #ifdef UBSEC_DEBUG 1456 if (ubsec_debug) 1457 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1458 #endif 1459 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) { 1460 struct ubsec_pktbuf *pb; 1461 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len; 1462 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr; 1463 1464 if (sskip >= packl) { 1465 sskip -= packl; 1466 continue; 1467 } 1468 1469 packl -= sskip; 1470 packp += sskip; 1471 sskip = 0; 1472 1473 if (packl > 0xfffc) { 1474 err = EIO; 1475 goto errout; 1476 } 1477 1478 if (j == 0) 1479 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1480 else 1481 pb = &dmap->d_dma->d_sbuf[j - 1]; 1482 1483 pb->pb_addr = htole32(packp); 1484 1485 if (stheend) { 1486 if (packl > stheend) { 1487 pb->pb_len = htole32(stheend); 1488 stheend = 0; 1489 } else { 1490 pb->pb_len = htole32(packl); 1491 stheend -= packl; 1492 } 1493 } else 1494 pb->pb_len = htole32(packl); 1495 1496 if ((i + 1) == q->q_src_map->dm_nsegs) 1497 pb->pb_next = 0; 1498 else 1499 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1500 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1501 j++; 1502 } 1503 1504 if (enccrd == NULL && maccrd != NULL) { 1505 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1506 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1507 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1508 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1509 #ifdef UBSEC_DEBUG 1510 if (ubsec_debug) 1511 printf("opkt: %x %x %x\n", 1512 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1513 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1514 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1515 1516 #endif 1517 } else { 1518 if (crp->crp_flags & CRYPTO_F_IOV) { 1519 if (!nicealign) { 1520 ubsecstats.hst_iovmisaligned++; 1521 err = EINVAL; 1522 goto errout; 1523 } 1524 if (q->q_dst_map == NULL) { 1525 if (q->q_cached_dst_map == NULL) { 1526 /* 1527 * XXX: ``what the heck's that'' 1528 * 0xfff0? 1529 */ 1530 if (bus_dmamap_create(sc->sc_dmat, 1531 0xfff0, UBS_MAX_SCATTER, 0xfff0, 0, 1532 BUS_DMA_NOWAIT, 1533 &q->q_cached_dst_map) != 0) { 1534 ubsecstats.hst_nomap++; 1535 err = ENOMEM; 1536 goto errout; 1537 } 1538 } 1539 q->q_dst_map = q->q_cached_dst_map; 1540 } 1541 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1542 q->q_dst_io, BUS_DMA_NOWAIT) != 0) { 1543 ubsecstats.hst_noload++; 1544 err = ENOMEM; 1545 goto errout; 1546 } 1547 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1548 if (nicealign) { 1549 q->q_dst_m = q->q_src_m; 1550 q->q_dst_map = q->q_src_map; 1551 } else { 1552 int totlen, len; 1553 struct mbuf *m, *top, **mp; 1554 1555 ubsecstats.hst_unaligned++; 1556 totlen = q->q_src_map->dm_mapsize; 1557 if (q->q_src_m->m_flags & M_PKTHDR) { 1558 len = MHLEN; 1559 MGETHDR(m, M_DONTWAIT, MT_DATA); 1560 /*XXX FIXME: m_dup_pkthdr */ 1561 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) { 1562 m_free(m); 1563 m = NULL; 1564 } 1565 } else { 1566 len = MLEN; 1567 MGET(m, M_DONTWAIT, MT_DATA); 1568 } 1569 if (m == NULL) { 1570 ubsecstats.hst_nombuf++; 1571 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1572 goto errout; 1573 } 1574 if (len == MHLEN) 1575 /*XXX was M_DUP_PKTHDR*/ 1576 m_copy_pkthdr(m, q->q_src_m); 1577 if (totlen >= MINCLSIZE) { 1578 MCLGET(m, M_DONTWAIT); 1579 if ((m->m_flags & M_EXT) == 0) { 1580 m_free(m); 1581 ubsecstats.hst_nomcl++; 1582 err = sc->sc_nqueue 1583 ? ERESTART : ENOMEM; 1584 goto errout; 1585 } 1586 len = MCLBYTES; 1587 } 1588 m->m_len = len; 1589 top = NULL; 1590 mp = ⊤ 1591 1592 while (totlen > 0) { 1593 if (top) { 1594 MGET(m, M_DONTWAIT, MT_DATA); 1595 if (m == NULL) { 1596 m_freem(top); 1597 ubsecstats.hst_nombuf++; 1598 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1599 goto errout; 1600 } 1601 len = MLEN; 1602 } 1603 if (top && totlen >= MINCLSIZE) { 1604 MCLGET(m, M_DONTWAIT); 1605 if ((m->m_flags & M_EXT) == 0) { 1606 *mp = m; 1607 m_freem(top); 1608 ubsecstats.hst_nomcl++; 1609 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1610 goto errout; 1611 } 1612 len = MCLBYTES; 1613 } 1614 m->m_len = len = uimin(totlen, len); 1615 totlen -= len; 1616 *mp = m; 1617 mp = &m->m_next; 1618 } 1619 q->q_dst_m = top; 1620 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1621 cpskip, cpoffset); 1622 if (q->q_dst_map == NULL) { 1623 if (q->q_cached_dst_map == NULL) { 1624 /* XXX again, what the heck is that 0xfff0? */ 1625 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1626 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1627 &q->q_cached_dst_map) != 0) { 1628 ubsecstats.hst_nomap++; 1629 err = ENOMEM; 1630 goto errout; 1631 } 1632 } 1633 q->q_dst_map = q->q_cached_dst_map; 1634 } 1635 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1636 q->q_dst_map, q->q_dst_m, 1637 BUS_DMA_NOWAIT) != 0) { 1638 ubsecstats.hst_noload++; 1639 err = ENOMEM; 1640 goto errout; 1641 } 1642 } 1643 } else { 1644 ubsecstats.hst_badflags++; 1645 err = EINVAL; 1646 goto errout; 1647 } 1648 1649 #ifdef UBSEC_DEBUG 1650 if (ubsec_debug) 1651 printf("dst skip: %d\n", dskip); 1652 #endif 1653 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) { 1654 struct ubsec_pktbuf *pb; 1655 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len; 1656 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr; 1657 1658 if (dskip >= packl) { 1659 dskip -= packl; 1660 continue; 1661 } 1662 1663 packl -= dskip; 1664 packp += dskip; 1665 dskip = 0; 1666 1667 if (packl > 0xfffc) { 1668 err = EIO; 1669 goto errout; 1670 } 1671 1672 if (j == 0) 1673 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1674 else 1675 pb = &dmap->d_dma->d_dbuf[j - 1]; 1676 1677 pb->pb_addr = htole32(packp); 1678 1679 if (dtheend) { 1680 if (packl > dtheend) { 1681 pb->pb_len = htole32(dtheend); 1682 dtheend = 0; 1683 } else { 1684 pb->pb_len = htole32(packl); 1685 dtheend -= packl; 1686 } 1687 } else 1688 pb->pb_len = htole32(packl); 1689 1690 if ((i + 1) == q->q_dst_map->dm_nsegs) { 1691 if (maccrd) 1692 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1693 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1694 else 1695 pb->pb_next = 0; 1696 } else 1697 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1698 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1699 j++; 1700 } 1701 } 1702 1703 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1704 offsetof(struct ubsec_dmachunk, d_ctx)); 1705 1706 if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) { 1707 struct ubsec_pktctx_aes128 *aes128; 1708 struct ubsec_pktctx_aes192 *aes192; 1709 struct ubsec_pktctx_aes256 *aes256; 1710 struct ubsec_pktctx_hdr *ph; 1711 u_int8_t *ctx; 1712 1713 ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1714 offsetof(struct ubsec_dmachunk, d_ctx); 1715 1716 ph = (struct ubsec_pktctx_hdr *)ctx; 1717 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES); 1718 ph->ph_flags = flags; 1719 ph->ph_offset = htole16(coffset >> 2); 1720 1721 switch (enccrd->crd_klen) { 1722 case 128: 1723 aes128 = (struct ubsec_pktctx_aes128 *)ctx; 1724 ph->ph_len = htole16(sizeof(*aes128)); 1725 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128); 1726 for (i = 0; i < 4; i++) 1727 aes128->pc_aeskey[i] = key.ses_key[i]; 1728 for (i = 0; i < 5; i++) 1729 aes128->pc_hminner[i] = key.ses_hminner[i]; 1730 for (i = 0; i < 5; i++) 1731 aes128->pc_hmouter[i] = key.ses_hmouter[i]; 1732 for (i = 0; i < 4; i++) 1733 aes128->pc_iv[i] = key.ses_iv[i]; 1734 break; 1735 case 192: 1736 aes192 = (struct ubsec_pktctx_aes192 *)ctx; 1737 ph->ph_len = htole16(sizeof(*aes192)); 1738 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192); 1739 for (i = 0; i < 6; i++) 1740 aes192->pc_aeskey[i] = key.ses_key[i]; 1741 for (i = 0; i < 5; i++) 1742 aes192->pc_hminner[i] = key.ses_hminner[i]; 1743 for (i = 0; i < 5; i++) 1744 aes192->pc_hmouter[i] = key.ses_hmouter[i]; 1745 for (i = 0; i < 4; i++) 1746 aes192->pc_iv[i] = key.ses_iv[i]; 1747 break; 1748 case 256: 1749 aes256 = (struct ubsec_pktctx_aes256 *)ctx; 1750 ph->ph_len = htole16(sizeof(*aes256)); 1751 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256); 1752 for (i = 0; i < 8; i++) 1753 aes256->pc_aeskey[i] = key.ses_key[i]; 1754 for (i = 0; i < 5; i++) 1755 aes256->pc_hminner[i] = key.ses_hminner[i]; 1756 for (i = 0; i < 5; i++) 1757 aes256->pc_hmouter[i] = key.ses_hmouter[i]; 1758 for (i = 0; i < 4; i++) 1759 aes256->pc_iv[i] = key.ses_iv[i]; 1760 break; 1761 } 1762 } else if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1763 struct ubsec_pktctx_3des *ctx; 1764 struct ubsec_pktctx_hdr *ph; 1765 1766 ctx = (struct ubsec_pktctx_3des *) 1767 ((u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1768 offsetof(struct ubsec_dmachunk, d_ctx)); 1769 1770 ph = (struct ubsec_pktctx_hdr *)ctx; 1771 ph->ph_len = htole16(sizeof(*ctx)); 1772 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES); 1773 ph->ph_flags = flags; 1774 ph->ph_offset = htole16(coffset >> 2); 1775 1776 for (i = 0; i < 6; i++) 1777 ctx->pc_deskey[i] = key.ses_key[i]; 1778 for (i = 0; i < 5; i++) 1779 ctx->pc_hminner[i] = key.ses_hminner[i]; 1780 for (i = 0; i < 5; i++) 1781 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1782 for (i = 0; i < 2; i++) 1783 ctx->pc_iv[i] = key.ses_iv[i]; 1784 } else { 1785 struct ubsec_pktctx *ctx = (struct ubsec_pktctx *) 1786 ((u_int8_t *)dmap->d_alloc.dma_vaddr + 1787 offsetof(struct ubsec_dmachunk, d_ctx)); 1788 1789 ctx->pc_flags = flags; 1790 ctx->pc_offset = htole16(coffset >> 2); 1791 for (i = 0; i < 6; i++) 1792 ctx->pc_deskey[i] = key.ses_key[i]; 1793 for (i = 0; i < 5; i++) 1794 ctx->pc_hminner[i] = key.ses_hminner[i]; 1795 for (i = 0; i < 5; i++) 1796 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1797 for (i = 0; i < 2; i++) 1798 ctx->pc_iv[i] = key.ses_iv[i]; 1799 } 1800 1801 mutex_spin_enter(&sc->sc_mtx); 1802 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1803 sc->sc_nqueue++; 1804 ubsecstats.hst_ipackets++; 1805 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize; 1806 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1807 ubsec_feed(sc); 1808 mutex_spin_exit(&sc->sc_mtx); 1809 return (0); 1810 1811 errout: 1812 if (q != NULL) { 1813 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1814 m_freem(q->q_dst_m); 1815 1816 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1817 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1818 } 1819 if (q->q_src_map != NULL) { 1820 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1821 } 1822 1823 mutex_spin_enter(&sc->sc_mtx); 1824 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1825 mutex_spin_exit(&sc->sc_mtx); 1826 } 1827 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */ 1828 if (err == EINVAL) 1829 ubsecstats.hst_invalid++; 1830 else 1831 ubsecstats.hst_nomem++; 1832 #endif 1833 if (err != ERESTART) { 1834 crp->crp_etype = err; 1835 crypto_done(crp); 1836 } else { 1837 sc->sc_needwakeup |= CRYPTO_SYMQ; 1838 } 1839 return (err); 1840 } 1841 1842 static void 1843 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1844 { 1845 struct cryptop *crp = (struct cryptop *)q->q_crp; 1846 struct cryptodesc *crd; 1847 struct ubsec_dma *dmap = q->q_dma; 1848 1849 ubsecstats.hst_opackets++; 1850 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1851 1852 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0, 1853 dmap->d_alloc.dma_map->dm_mapsize, 1854 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1855 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1856 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1857 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1858 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1859 } 1860 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 1861 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1862 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1863 1864 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1865 m_freem(q->q_src_m); 1866 crp->crp_buf = (void *)q->q_dst_m; 1867 } 1868 1869 /* copy out IV for future use */ 1870 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1871 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1872 if (crd->crd_alg != CRYPTO_DES_CBC && 1873 crd->crd_alg != CRYPTO_3DES_CBC && 1874 crd->crd_alg != CRYPTO_AES_CBC) 1875 continue; 1876 if (crp->crp_flags & CRYPTO_F_IMBUF) 1877 m_copydata((struct mbuf *)crp->crp_buf, 1878 crd->crd_skip + crd->crd_len - 8, 8, 1879 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1880 else if (crp->crp_flags & CRYPTO_F_IOV) { 1881 cuio_copydata((struct uio *)crp->crp_buf, 1882 crd->crd_skip + crd->crd_len - 8, 8, 1883 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1884 } 1885 break; 1886 } 1887 } 1888 1889 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1890 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 && 1891 crd->crd_alg != CRYPTO_SHA1_HMAC_96) 1892 continue; 1893 if (crp->crp_flags & CRYPTO_F_IMBUF) 1894 m_copyback((struct mbuf *)crp->crp_buf, 1895 crd->crd_inject, 12, 1896 (void *)dmap->d_dma->d_macbuf); 1897 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1898 bcopy((void *)dmap->d_dma->d_macbuf, 1899 crp->crp_mac, 12); 1900 break; 1901 } 1902 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1903 crypto_done(crp); 1904 } 1905 1906 static void 1907 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1908 { 1909 int i, j, dlen, slen; 1910 char *dptr, *sptr; 1911 1912 j = 0; 1913 sptr = srcm->m_data; 1914 slen = srcm->m_len; 1915 dptr = dstm->m_data; 1916 dlen = dstm->m_len; 1917 1918 while (1) { 1919 for (i = 0; i < uimin(slen, dlen); i++) { 1920 if (j < hoffset || j >= toffset) 1921 *dptr++ = *sptr++; 1922 slen--; 1923 dlen--; 1924 j++; 1925 } 1926 if (slen == 0) { 1927 srcm = srcm->m_next; 1928 if (srcm == NULL) 1929 return; 1930 sptr = srcm->m_data; 1931 slen = srcm->m_len; 1932 } 1933 if (dlen == 0) { 1934 dstm = dstm->m_next; 1935 if (dstm == NULL) 1936 return; 1937 dptr = dstm->m_data; 1938 dlen = dstm->m_len; 1939 } 1940 } 1941 } 1942 1943 /* 1944 * feed the key generator, must be called at splnet() or higher. 1945 */ 1946 static void 1947 ubsec_feed2(struct ubsec_softc *sc) 1948 { 1949 struct ubsec_q2 *q; 1950 1951 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1952 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1953 break; 1954 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1955 1956 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1957 q->q_mcr.dma_map->dm_mapsize, 1958 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1959 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1960 q->q_ctx.dma_map->dm_mapsize, 1961 BUS_DMASYNC_PREWRITE); 1962 1963 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1964 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1965 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next); 1966 --sc->sc_nqueue2; 1967 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1968 } 1969 } 1970 1971 /* 1972 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices) 1973 */ 1974 void 1975 ubsec_feed4(struct ubsec_softc *sc) 1976 { 1977 struct ubsec_q2 *q; 1978 1979 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) { 1980 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL) 1981 break; 1982 q = SIMPLEQ_FIRST(&sc->sc_queue4); 1983 1984 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1985 q->q_mcr.dma_map->dm_mapsize, 1986 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1987 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1988 q->q_ctx.dma_map->dm_mapsize, 1989 BUS_DMASYNC_PREWRITE); 1990 1991 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr); 1992 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next); 1993 --sc->sc_nqueue4; 1994 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next); 1995 } 1996 } 1997 1998 /* 1999 * Callback for handling random numbers 2000 */ 2001 static void 2002 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 2003 { 2004 struct cryptkop *krp; 2005 struct ubsec_ctx_keyop *ctx; 2006 2007 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 2008 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 2009 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2010 2011 switch (q->q_type) { 2012 #ifndef UBSEC_NO_RNG 2013 case UBS_CTXOP_RNGSHA1: 2014 case UBS_CTXOP_RNGBYPASS: { 2015 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 2016 u_int32_t *p; 2017 int i; 2018 2019 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2020 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2021 p = (u_int32_t *)rng->rng_buf.dma_vaddr; 2022 #ifndef __NetBSD__ 2023 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++) 2024 add_true_randomness(letoh32(*p)); 2025 #else 2026 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t); 2027 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY); 2028 sc->sc_rng_need -= i; 2029 #endif 2030 rng->rng_used = 0; 2031 #ifdef __OpenBSD__ 2032 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2033 #else 2034 if (sc->sc_rng_need > 0) { 2035 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2036 } 2037 #endif 2038 break; 2039 } 2040 #endif 2041 case UBS_CTXOP_MODEXP: { 2042 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2043 u_int rlen, clen; 2044 2045 krp = me->me_krp; 2046 rlen = (me->me_modbits + 7) / 8; 2047 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 2048 2049 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2050 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2051 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2052 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2053 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2054 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2055 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2056 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2057 2058 if (clen < rlen) 2059 krp->krp_status = E2BIG; 2060 else { 2061 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 2062 memset(krp->krp_param[krp->krp_iparams].crp_p, 0, 2063 (krp->krp_param[krp->krp_iparams].crp_nbits 2064 + 7) / 8); 2065 bcopy(me->me_C.dma_vaddr, 2066 krp->krp_param[krp->krp_iparams].crp_p, 2067 (me->me_modbits + 7) / 8); 2068 } else 2069 ubsec_kshift_l(me->me_shiftbits, 2070 me->me_C.dma_vaddr, me->me_normbits, 2071 krp->krp_param[krp->krp_iparams].crp_p, 2072 krp->krp_param[krp->krp_iparams].crp_nbits); 2073 } 2074 2075 crypto_kdone(krp); 2076 2077 /* bzero all potentially sensitive data */ 2078 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2079 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2080 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2081 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2082 2083 /* Can't free here, so put us on the free list. */ 2084 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 2085 break; 2086 } 2087 case UBS_CTXOP_RSAPRIV: { 2088 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2089 u_int len; 2090 2091 krp = rp->rpr_krp; 2092 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0, 2093 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2094 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0, 2095 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2096 2097 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) 2098 / 8; 2099 bcopy(rp->rpr_msgout.dma_vaddr, 2100 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 2101 2102 crypto_kdone(krp); 2103 2104 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 2105 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 2106 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size); 2107 2108 /* Can't free here, so put us on the free list. */ 2109 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 2110 break; 2111 } 2112 default: 2113 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev), 2114 letoh16(ctx->ctx_op)); 2115 break; 2116 } 2117 } 2118 2119 #ifndef UBSEC_NO_RNG 2120 2121 static void 2122 ubsec_rng_get(size_t bytes, void *vsc) 2123 { 2124 struct ubsec_softc *sc = vsc; 2125 2126 mutex_spin_enter(&sc->sc_mtx); 2127 sc->sc_rng_need = bytes; 2128 ubsec_rng_locked(sc); 2129 mutex_spin_exit(&sc->sc_mtx); 2130 2131 } 2132 2133 static void 2134 ubsec_rng(void *vsc) 2135 { 2136 struct ubsec_softc *sc = vsc; 2137 mutex_spin_enter(&sc->sc_mtx); 2138 ubsec_rng_locked(sc); 2139 mutex_spin_exit(&sc->sc_mtx); 2140 } 2141 2142 static void 2143 ubsec_rng_locked(void *vsc) 2144 { 2145 struct ubsec_softc *sc = vsc; 2146 struct ubsec_q2_rng *rng = &sc->sc_rng; 2147 struct ubsec_mcr *mcr; 2148 struct ubsec_ctx_rngbypass *ctx; 2149 int *nqueue; 2150 2151 /* Caller is responsible to lock and release sc_mtx. */ 2152 KASSERT(mutex_owned(&sc->sc_mtx)); 2153 2154 if (rng->rng_used) { 2155 return; 2156 } 2157 2158 if (sc->sc_rng_need < 1) { 2159 callout_stop(&sc->sc_rngto); 2160 return; 2161 } 2162 2163 if (sc->sc_flags & UBS_FLAGS_RNG4) 2164 nqueue = &sc->sc_nqueue4; 2165 else 2166 nqueue = &sc->sc_nqueue2; 2167 2168 (*nqueue)++; 2169 if (*nqueue >= UBS_MAX_NQUEUE) 2170 goto out; 2171 2172 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 2173 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 2174 2175 mcr->mcr_pkts = htole16(1); 2176 mcr->mcr_flags = 0; 2177 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 2178 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 2179 mcr->mcr_ipktbuf.pb_len = 0; 2180 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 2181 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 2182 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 2183 UBS_PKTBUF_LEN); 2184 mcr->mcr_opktbuf.pb_next = 0; 2185 2186 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 2187 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1); 2188 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1; 2189 2190 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2191 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2192 2193 if (sc->sc_flags & UBS_FLAGS_RNG4) { 2194 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next); 2195 ubsec_feed4(sc); 2196 } else { 2197 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 2198 ubsec_feed2(sc); 2199 } 2200 rng->rng_used = 1; 2201 ubsecstats.hst_rng++; 2202 2203 return; 2204 2205 out: 2206 /* 2207 * Something weird happened, generate our own call back. 2208 */ 2209 (*nqueue)--; 2210 #ifdef __OpenBSD__ 2211 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2212 #else 2213 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2214 #endif 2215 } 2216 #endif /* UBSEC_NO_RNG */ 2217 2218 static int 2219 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size, 2220 struct ubsec_dma_alloc *dma,int mapflags) 2221 { 2222 int r; 2223 2224 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, 2225 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0) 2226 goto fail_0; 2227 2228 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, 2229 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0) 2230 goto fail_1; 2231 2232 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2233 BUS_DMA_NOWAIT, &dma->dma_map)) != 0) 2234 goto fail_2; 2235 2236 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 2237 size, NULL, BUS_DMA_NOWAIT)) != 0) 2238 goto fail_3; 2239 2240 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2241 dma->dma_size = size; 2242 return (0); 2243 2244 fail_3: 2245 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2246 fail_2: 2247 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size); 2248 fail_1: 2249 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2250 fail_0: 2251 dma->dma_map = NULL; 2252 return (r); 2253 } 2254 2255 static void 2256 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 2257 { 2258 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 2259 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size); 2260 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2261 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2262 } 2263 2264 /* 2265 * Resets the board. Values in the regesters are left as is 2266 * from the reset (i.e. initial values are assigned elsewhere). 2267 */ 2268 static void 2269 ubsec_reset_board(struct ubsec_softc *sc) 2270 { 2271 volatile u_int32_t ctrl; 2272 2273 ctrl = READ_REG(sc, BS_CTRL); 2274 ctrl |= BS_CTRL_RESET; 2275 WRITE_REG(sc, BS_CTRL, ctrl); 2276 2277 /* 2278 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2279 */ 2280 DELAY(10); 2281 2282 /* Enable RNG and interrupts on newer devices */ 2283 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2284 #ifndef UBSEC_NO_RNG 2285 WRITE_REG(sc, BS_CFG, BS_CFG_RNG); 2286 #endif 2287 WRITE_REG(sc, BS_INT, BS_INT_DMAINT); 2288 } 2289 } 2290 2291 /* 2292 * Init Broadcom registers 2293 */ 2294 static void 2295 ubsec_init_board(struct ubsec_softc *sc) 2296 { 2297 u_int32_t ctrl; 2298 2299 ctrl = READ_REG(sc, BS_CTRL); 2300 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2301 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2302 2303 /* 2304 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)). 2305 * anyone got hw docs? 2306 */ 2307 if (sc->sc_flags & UBS_FLAGS_KEY) 2308 ctrl |= BS_CTRL_MCR2INT; 2309 else 2310 ctrl &= ~BS_CTRL_MCR2INT; 2311 2312 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2313 ctrl &= ~BS_CTRL_SWNORM; 2314 2315 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2316 ctrl |= BS_CTRL_BSIZE240; 2317 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */ 2318 2319 if (sc->sc_flags & UBS_FLAGS_RNG4) 2320 ctrl |= BS_CTRL_MCR4INT; 2321 else 2322 ctrl &= ~BS_CTRL_MCR4INT; 2323 } 2324 2325 WRITE_REG(sc, BS_CTRL, ctrl); 2326 } 2327 2328 /* 2329 * Init Broadcom PCI registers 2330 */ 2331 static void 2332 ubsec_init_pciregs(struct pci_attach_args *pa) 2333 { 2334 pci_chipset_tag_t pc = pa->pa_pc; 2335 u_int32_t misc; 2336 2337 /* 2338 * This will set the cache line size to 1, this will 2339 * force the BCM58xx chip just to do burst read/writes. 2340 * Cache line read/writes are to slow 2341 */ 2342 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG); 2343 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT)) 2344 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT); 2345 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc); 2346 } 2347 2348 /* 2349 * Clean up after a chip crash. 2350 * It is assumed that the caller in splnet() 2351 */ 2352 static void 2353 ubsec_cleanchip(struct ubsec_softc *sc) 2354 { 2355 struct ubsec_q *q; 2356 2357 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2358 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2359 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 2360 ubsec_free_q(sc, q); 2361 } 2362 sc->sc_nqchip = 0; 2363 } 2364 2365 /* 2366 * free a ubsec_q 2367 * It is assumed that the caller is within splnet() 2368 */ 2369 static int 2370 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2371 { 2372 struct ubsec_q *q2; 2373 struct cryptop *crp; 2374 int npkts; 2375 int i; 2376 2377 npkts = q->q_nstacked_mcrs; 2378 2379 for (i = 0; i < npkts; i++) { 2380 if(q->q_stacked_mcr[i]) { 2381 q2 = q->q_stacked_mcr[i]; 2382 2383 if ((q2->q_dst_m != NULL) 2384 && (q2->q_src_m != q2->q_dst_m)) 2385 m_freem(q2->q_dst_m); 2386 2387 crp = (struct cryptop *)q2->q_crp; 2388 2389 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2390 2391 crp->crp_etype = EFAULT; 2392 crypto_done(crp); 2393 } else { 2394 break; 2395 } 2396 } 2397 2398 /* 2399 * Free header MCR 2400 */ 2401 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2402 m_freem(q->q_dst_m); 2403 2404 crp = (struct cryptop *)q->q_crp; 2405 2406 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2407 2408 crp->crp_etype = EFAULT; 2409 crypto_done(crp); 2410 return(0); 2411 } 2412 2413 /* 2414 * Routine to reset the chip and clean up. 2415 * It is assumed that the caller is in splnet() 2416 */ 2417 static void 2418 ubsec_totalreset(struct ubsec_softc *sc) 2419 { 2420 ubsec_reset_board(sc); 2421 ubsec_init_board(sc); 2422 ubsec_cleanchip(sc); 2423 } 2424 2425 static int 2426 ubsec_dmamap_aligned(bus_dmamap_t map) 2427 { 2428 int i; 2429 2430 for (i = 0; i < map->dm_nsegs; i++) { 2431 if (map->dm_segs[i].ds_addr & 3) 2432 return (0); 2433 if ((i != (map->dm_nsegs - 1)) && 2434 (map->dm_segs[i].ds_len & 3)) 2435 return (0); 2436 } 2437 return (1); 2438 } 2439 2440 #ifdef __OpenBSD__ 2441 struct ubsec_softc * 2442 ubsec_kfind(struct cryptkop *krp) 2443 { 2444 struct ubsec_softc *sc; 2445 int i; 2446 2447 for (i = 0; i < ubsec_cd.cd_ndevs; i++) { 2448 sc = ubsec_cd.cd_devs[i]; 2449 if (sc == NULL) 2450 continue; 2451 if (sc->sc_cid == krp->krp_hid) 2452 return (sc); 2453 } 2454 return (NULL); 2455 } 2456 #endif 2457 2458 static void 2459 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2460 { 2461 switch (q->q_type) { 2462 case UBS_CTXOP_MODEXP: { 2463 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2464 2465 ubsec_dma_free(sc, &me->me_q.q_mcr); 2466 ubsec_dma_free(sc, &me->me_q.q_ctx); 2467 ubsec_dma_free(sc, &me->me_M); 2468 ubsec_dma_free(sc, &me->me_E); 2469 ubsec_dma_free(sc, &me->me_C); 2470 ubsec_dma_free(sc, &me->me_epb); 2471 free(me, M_DEVBUF); 2472 break; 2473 } 2474 case UBS_CTXOP_RSAPRIV: { 2475 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2476 2477 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2478 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2479 ubsec_dma_free(sc, &rp->rpr_msgin); 2480 ubsec_dma_free(sc, &rp->rpr_msgout); 2481 free(rp, M_DEVBUF); 2482 break; 2483 } 2484 default: 2485 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev), 2486 q->q_type); 2487 break; 2488 } 2489 } 2490 2491 static int 2492 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2493 { 2494 struct ubsec_softc *sc; 2495 int r; 2496 2497 if (krp == NULL || krp->krp_callback == NULL) 2498 return (EINVAL); 2499 #ifdef __OpenBSD__ 2500 if ((sc = ubsec_kfind(krp)) == NULL) 2501 return (EINVAL); 2502 #else 2503 sc = arg; 2504 KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/); 2505 #endif 2506 2507 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2508 struct ubsec_q2 *q; 2509 2510 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2511 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next); 2512 ubsec_kfree(sc, q); 2513 } 2514 2515 switch (krp->krp_op) { 2516 case CRK_MOD_EXP: 2517 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2518 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2519 else 2520 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2521 break; 2522 case CRK_MOD_EXP_CRT: 2523 r = ubsec_kprocess_rsapriv(sc, krp, hint); 2524 break; 2525 default: 2526 printf("%s: kprocess: invalid op 0x%x\n", 2527 device_xname(sc->sc_dev), krp->krp_op); 2528 krp->krp_status = EOPNOTSUPP; 2529 crypto_kdone(krp); 2530 r = 0; 2531 } 2532 return (r); 2533 } 2534 2535 /* 2536 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2537 */ 2538 static int 2539 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, 2540 int hint) 2541 { 2542 struct ubsec_q2_modexp *me; 2543 struct ubsec_mcr *mcr; 2544 struct ubsec_ctx_modexp *ctx; 2545 struct ubsec_pktbuf *epb; 2546 int err = 0; 2547 u_int nbits, normbits, mbits, shiftbits, ebits; 2548 2549 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2550 if (me == NULL) { 2551 err = ENOMEM; 2552 goto errout; 2553 } 2554 memset(me, 0, sizeof *me); 2555 me->me_krp = krp; 2556 me->me_q.q_type = UBS_CTXOP_MODEXP; 2557 2558 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2559 if (nbits <= 512) 2560 normbits = 512; 2561 else if (nbits <= 768) 2562 normbits = 768; 2563 else if (nbits <= 1024) 2564 normbits = 1024; 2565 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2566 normbits = 1536; 2567 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2568 normbits = 2048; 2569 else { 2570 err = E2BIG; 2571 goto errout; 2572 } 2573 2574 shiftbits = normbits - nbits; 2575 2576 me->me_modbits = nbits; 2577 me->me_shiftbits = shiftbits; 2578 me->me_normbits = normbits; 2579 2580 /* Sanity check: result bits must be >= true modulus bits. */ 2581 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2582 err = ERANGE; 2583 goto errout; 2584 } 2585 2586 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2587 &me->me_q.q_mcr, 0)) { 2588 err = ENOMEM; 2589 goto errout; 2590 } 2591 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2592 2593 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2594 &me->me_q.q_ctx, 0)) { 2595 err = ENOMEM; 2596 goto errout; 2597 } 2598 2599 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2600 if (mbits > nbits) { 2601 err = E2BIG; 2602 goto errout; 2603 } 2604 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2605 err = ENOMEM; 2606 goto errout; 2607 } 2608 ubsec_kshift_r(shiftbits, 2609 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2610 me->me_M.dma_vaddr, normbits); 2611 2612 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2613 err = ENOMEM; 2614 goto errout; 2615 } 2616 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2617 2618 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2619 if (ebits > nbits) { 2620 err = E2BIG; 2621 goto errout; 2622 } 2623 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2624 err = ENOMEM; 2625 goto errout; 2626 } 2627 ubsec_kshift_r(shiftbits, 2628 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2629 me->me_E.dma_vaddr, normbits); 2630 2631 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2632 &me->me_epb, 0)) { 2633 err = ENOMEM; 2634 goto errout; 2635 } 2636 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2637 epb->pb_addr = htole32(me->me_E.dma_paddr); 2638 epb->pb_next = 0; 2639 epb->pb_len = htole32(normbits / 8); 2640 2641 #ifdef UBSEC_DEBUG 2642 if (ubsec_debug) { 2643 printf("Epb "); 2644 ubsec_dump_pb(epb); 2645 } 2646 #endif 2647 2648 mcr->mcr_pkts = htole16(1); 2649 mcr->mcr_flags = 0; 2650 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2651 mcr->mcr_reserved = 0; 2652 mcr->mcr_pktlen = 0; 2653 2654 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2655 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2656 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2657 2658 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2659 mcr->mcr_opktbuf.pb_next = 0; 2660 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2661 2662 #ifdef DIAGNOSTIC 2663 /* Misaligned output buffer will hang the chip. */ 2664 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2665 panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev), 2666 letoh32(mcr->mcr_opktbuf.pb_addr)); 2667 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2668 panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev), 2669 letoh32(mcr->mcr_opktbuf.pb_len)); 2670 #endif 2671 2672 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2673 memset(ctx, 0, sizeof(*ctx)); 2674 ubsec_kshift_r(shiftbits, 2675 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2676 ctx->me_N, normbits); 2677 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2678 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2679 ctx->me_E_len = htole16(nbits); 2680 ctx->me_N_len = htole16(nbits); 2681 2682 #ifdef UBSEC_DEBUG 2683 if (ubsec_debug) { 2684 ubsec_dump_mcr(mcr); 2685 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2686 } 2687 #endif 2688 2689 /* 2690 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2691 * everything else. 2692 */ 2693 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2694 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2695 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2696 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2697 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2698 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2699 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2700 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2701 2702 /* Enqueue and we're done... */ 2703 mutex_spin_enter(&sc->sc_mtx); 2704 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2705 ubsec_feed2(sc); 2706 ubsecstats.hst_modexp++; 2707 mutex_spin_exit(&sc->sc_mtx); 2708 2709 return (0); 2710 2711 errout: 2712 if (me != NULL) { 2713 if (me->me_q.q_mcr.dma_map != NULL) 2714 ubsec_dma_free(sc, &me->me_q.q_mcr); 2715 if (me->me_q.q_ctx.dma_map != NULL) { 2716 memset(me->me_q.q_ctx.dma_vaddr, 0, 2717 me->me_q.q_ctx.dma_size); 2718 ubsec_dma_free(sc, &me->me_q.q_ctx); 2719 } 2720 if (me->me_M.dma_map != NULL) { 2721 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2722 ubsec_dma_free(sc, &me->me_M); 2723 } 2724 if (me->me_E.dma_map != NULL) { 2725 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2726 ubsec_dma_free(sc, &me->me_E); 2727 } 2728 if (me->me_C.dma_map != NULL) { 2729 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2730 ubsec_dma_free(sc, &me->me_C); 2731 } 2732 if (me->me_epb.dma_map != NULL) 2733 ubsec_dma_free(sc, &me->me_epb); 2734 free(me, M_DEVBUF); 2735 } 2736 krp->krp_status = err; 2737 crypto_kdone(krp); 2738 return (0); 2739 } 2740 2741 /* 2742 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2743 */ 2744 static int 2745 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, 2746 int hint) 2747 { 2748 struct ubsec_q2_modexp *me; 2749 struct ubsec_mcr *mcr; 2750 struct ubsec_ctx_modexp *ctx; 2751 struct ubsec_pktbuf *epb; 2752 int err = 0; 2753 u_int nbits, normbits, mbits, shiftbits, ebits; 2754 2755 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2756 if (me == NULL) { 2757 err = ENOMEM; 2758 goto errout; 2759 } 2760 memset(me, 0, sizeof *me); 2761 me->me_krp = krp; 2762 me->me_q.q_type = UBS_CTXOP_MODEXP; 2763 2764 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2765 if (nbits <= 512) 2766 normbits = 512; 2767 else if (nbits <= 768) 2768 normbits = 768; 2769 else if (nbits <= 1024) 2770 normbits = 1024; 2771 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2772 normbits = 1536; 2773 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2774 normbits = 2048; 2775 else { 2776 err = E2BIG; 2777 goto errout; 2778 } 2779 2780 shiftbits = normbits - nbits; 2781 2782 /* XXX ??? */ 2783 me->me_modbits = nbits; 2784 me->me_shiftbits = shiftbits; 2785 me->me_normbits = normbits; 2786 2787 /* Sanity check: result bits must be >= true modulus bits. */ 2788 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2789 err = ERANGE; 2790 goto errout; 2791 } 2792 2793 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2794 &me->me_q.q_mcr, 0)) { 2795 err = ENOMEM; 2796 goto errout; 2797 } 2798 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2799 2800 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2801 &me->me_q.q_ctx, 0)) { 2802 err = ENOMEM; 2803 goto errout; 2804 } 2805 2806 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2807 if (mbits > nbits) { 2808 err = E2BIG; 2809 goto errout; 2810 } 2811 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2812 err = ENOMEM; 2813 goto errout; 2814 } 2815 memset(me->me_M.dma_vaddr, 0, normbits / 8); 2816 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2817 me->me_M.dma_vaddr, (mbits + 7) / 8); 2818 2819 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2820 err = ENOMEM; 2821 goto errout; 2822 } 2823 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2824 2825 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2826 if (ebits > nbits) { 2827 err = E2BIG; 2828 goto errout; 2829 } 2830 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2831 err = ENOMEM; 2832 goto errout; 2833 } 2834 memset(me->me_E.dma_vaddr, 0, normbits / 8); 2835 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2836 me->me_E.dma_vaddr, (ebits + 7) / 8); 2837 2838 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2839 &me->me_epb, 0)) { 2840 err = ENOMEM; 2841 goto errout; 2842 } 2843 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2844 epb->pb_addr = htole32(me->me_E.dma_paddr); 2845 epb->pb_next = 0; 2846 epb->pb_len = htole32((ebits + 7) / 8); 2847 2848 #ifdef UBSEC_DEBUG 2849 if (ubsec_debug) { 2850 printf("Epb "); 2851 ubsec_dump_pb(epb); 2852 } 2853 #endif 2854 2855 mcr->mcr_pkts = htole16(1); 2856 mcr->mcr_flags = 0; 2857 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2858 mcr->mcr_reserved = 0; 2859 mcr->mcr_pktlen = 0; 2860 2861 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2862 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2863 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2864 2865 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2866 mcr->mcr_opktbuf.pb_next = 0; 2867 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2868 2869 #ifdef DIAGNOSTIC 2870 /* Misaligned output buffer will hang the chip. */ 2871 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2872 panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev), 2873 letoh32(mcr->mcr_opktbuf.pb_addr)); 2874 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2875 panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev), 2876 letoh32(mcr->mcr_opktbuf.pb_len)); 2877 #endif 2878 2879 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2880 memset(ctx, 0, sizeof(*ctx)); 2881 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p, 2882 (nbits + 7) / 8); 2883 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2884 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2885 ctx->me_E_len = htole16(ebits); 2886 ctx->me_N_len = htole16(nbits); 2887 2888 #ifdef UBSEC_DEBUG 2889 if (ubsec_debug) { 2890 ubsec_dump_mcr(mcr); 2891 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2892 } 2893 #endif 2894 2895 /* 2896 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2897 * everything else. 2898 */ 2899 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2900 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2901 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2902 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2903 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2904 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2905 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2906 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2907 2908 /* Enqueue and we're done... */ 2909 mutex_spin_enter(&sc->sc_mtx); 2910 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2911 ubsec_feed2(sc); 2912 mutex_spin_exit(&sc->sc_mtx); 2913 2914 return (0); 2915 2916 errout: 2917 if (me != NULL) { 2918 if (me->me_q.q_mcr.dma_map != NULL) 2919 ubsec_dma_free(sc, &me->me_q.q_mcr); 2920 if (me->me_q.q_ctx.dma_map != NULL) { 2921 memset(me->me_q.q_ctx.dma_vaddr, 0, 2922 me->me_q.q_ctx.dma_size); 2923 ubsec_dma_free(sc, &me->me_q.q_ctx); 2924 } 2925 if (me->me_M.dma_map != NULL) { 2926 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2927 ubsec_dma_free(sc, &me->me_M); 2928 } 2929 if (me->me_E.dma_map != NULL) { 2930 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2931 ubsec_dma_free(sc, &me->me_E); 2932 } 2933 if (me->me_C.dma_map != NULL) { 2934 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2935 ubsec_dma_free(sc, &me->me_C); 2936 } 2937 if (me->me_epb.dma_map != NULL) 2938 ubsec_dma_free(sc, &me->me_epb); 2939 free(me, M_DEVBUF); 2940 } 2941 krp->krp_status = err; 2942 crypto_kdone(krp); 2943 return (0); 2944 } 2945 2946 static int 2947 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, 2948 int hint) 2949 { 2950 struct ubsec_q2_rsapriv *rp = NULL; 2951 struct ubsec_mcr *mcr; 2952 struct ubsec_ctx_rsapriv *ctx; 2953 int err = 0; 2954 u_int padlen, msglen; 2955 2956 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2957 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2958 if (msglen > padlen) 2959 padlen = msglen; 2960 2961 if (padlen <= 256) 2962 padlen = 256; 2963 else if (padlen <= 384) 2964 padlen = 384; 2965 else if (padlen <= 512) 2966 padlen = 512; 2967 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2968 padlen = 768; 2969 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2970 padlen = 1024; 2971 else { 2972 err = E2BIG; 2973 goto errout; 2974 } 2975 2976 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2977 err = E2BIG; 2978 goto errout; 2979 } 2980 2981 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2982 err = E2BIG; 2983 goto errout; 2984 } 2985 2986 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2987 err = E2BIG; 2988 goto errout; 2989 } 2990 2991 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO); 2992 if (rp == NULL) 2993 return (ENOMEM); 2994 rp->rpr_krp = krp; 2995 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2996 2997 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2998 &rp->rpr_q.q_mcr, 0)) { 2999 err = ENOMEM; 3000 goto errout; 3001 } 3002 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 3003 3004 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 3005 &rp->rpr_q.q_ctx, 0)) { 3006 err = ENOMEM; 3007 goto errout; 3008 } 3009 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 3010 memset(ctx, 0, sizeof *ctx); 3011 3012 /* Copy in p */ 3013 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 3014 &ctx->rpr_buf[0 * (padlen / 8)], 3015 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 3016 3017 /* Copy in q */ 3018 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 3019 &ctx->rpr_buf[1 * (padlen / 8)], 3020 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 3021 3022 /* Copy in dp */ 3023 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 3024 &ctx->rpr_buf[2 * (padlen / 8)], 3025 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 3026 3027 /* Copy in dq */ 3028 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 3029 &ctx->rpr_buf[3 * (padlen / 8)], 3030 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 3031 3032 /* Copy in pinv */ 3033 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 3034 &ctx->rpr_buf[4 * (padlen / 8)], 3035 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 3036 3037 msglen = padlen * 2; 3038 3039 /* Copy in input message (aligned buffer/length). */ 3040 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 3041 /* Is this likely? */ 3042 err = E2BIG; 3043 goto errout; 3044 } 3045 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 3046 err = ENOMEM; 3047 goto errout; 3048 } 3049 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8); 3050 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 3051 rp->rpr_msgin.dma_vaddr, 3052 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 3053 3054 /* Prepare space for output message (aligned buffer/length). */ 3055 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 3056 /* Is this likely? */ 3057 err = E2BIG; 3058 goto errout; 3059 } 3060 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 3061 err = ENOMEM; 3062 goto errout; 3063 } 3064 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8); 3065 3066 mcr->mcr_pkts = htole16(1); 3067 mcr->mcr_flags = 0; 3068 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 3069 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 3070 mcr->mcr_ipktbuf.pb_next = 0; 3071 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 3072 mcr->mcr_reserved = 0; 3073 mcr->mcr_pktlen = htole16(msglen); 3074 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 3075 mcr->mcr_opktbuf.pb_next = 0; 3076 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 3077 3078 #ifdef DIAGNOSTIC 3079 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 3080 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)", 3081 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr, 3082 (u_long) rp->rpr_msgin.dma_size); 3083 } 3084 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 3085 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)", 3086 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr, 3087 (u_long) rp->rpr_msgout.dma_size); 3088 } 3089 #endif 3090 3091 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 3092 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 3093 ctx->rpr_q_len = htole16(padlen); 3094 ctx->rpr_p_len = htole16(padlen); 3095 3096 /* 3097 * ubsec_feed2 will sync mcr and ctx, we just need to sync 3098 * everything else. 3099 */ 3100 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 3101 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 3102 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 3103 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 3104 3105 /* Enqueue and we're done... */ 3106 mutex_spin_enter(&sc->sc_mtx); 3107 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 3108 ubsec_feed2(sc); 3109 ubsecstats.hst_modexpcrt++; 3110 mutex_spin_exit(&sc->sc_mtx); 3111 return (0); 3112 3113 errout: 3114 if (rp != NULL) { 3115 if (rp->rpr_q.q_mcr.dma_map != NULL) 3116 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 3117 if (rp->rpr_msgin.dma_map != NULL) { 3118 memset(rp->rpr_msgin.dma_vaddr, 0, 3119 rp->rpr_msgin.dma_size); 3120 ubsec_dma_free(sc, &rp->rpr_msgin); 3121 } 3122 if (rp->rpr_msgout.dma_map != NULL) { 3123 memset(rp->rpr_msgout.dma_vaddr, 0, 3124 rp->rpr_msgout.dma_size); 3125 ubsec_dma_free(sc, &rp->rpr_msgout); 3126 } 3127 free(rp, M_DEVBUF); 3128 } 3129 krp->krp_status = err; 3130 crypto_kdone(krp); 3131 return (0); 3132 } 3133 3134 #ifdef UBSEC_DEBUG 3135 static void 3136 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 3137 { 3138 printf("addr 0x%x (0x%x) next 0x%x\n", 3139 pb->pb_addr, pb->pb_len, pb->pb_next); 3140 } 3141 3142 static void 3143 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c) 3144 { 3145 printf("CTX (0x%x):\n", c->ctx_len); 3146 switch (letoh16(c->ctx_op)) { 3147 case UBS_CTXOP_RNGBYPASS: 3148 case UBS_CTXOP_RNGSHA1: 3149 break; 3150 case UBS_CTXOP_MODEXP: 3151 { 3152 struct ubsec_ctx_modexp *cx = (void *)c; 3153 int i, len; 3154 3155 printf(" Elen %u, Nlen %u\n", 3156 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 3157 len = (cx->me_N_len + 7)/8; 3158 for (i = 0; i < len; i++) 3159 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 3160 printf("\n"); 3161 break; 3162 } 3163 default: 3164 printf("unknown context: %x\n", c->ctx_op); 3165 } 3166 printf("END CTX\n"); 3167 } 3168 3169 static void 3170 ubsec_dump_mcr(struct ubsec_mcr *mcr) 3171 { 3172 volatile struct ubsec_mcr_add *ma; 3173 int i; 3174 3175 printf("MCR:\n"); 3176 printf(" pkts: %u, flags 0x%x\n", 3177 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 3178 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 3179 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 3180 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 3181 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 3182 letoh16(ma->mcr_reserved)); 3183 printf(" %d: ipkt ", i); 3184 ubsec_dump_pb(&ma->mcr_ipktbuf); 3185 printf(" %d: opkt ", i); 3186 ubsec_dump_pb(&ma->mcr_opktbuf); 3187 ma++; 3188 } 3189 printf("END MCR\n"); 3190 } 3191 #endif /* UBSEC_DEBUG */ 3192 3193 /* 3194 * Return the number of significant bits of a big number. 3195 */ 3196 static int 3197 ubsec_ksigbits(struct crparam *cr) 3198 { 3199 u_int plen = (cr->crp_nbits + 7) / 8; 3200 int i, sig = plen * 8; 3201 u_int8_t c, *p = cr->crp_p; 3202 3203 for (i = plen - 1; i >= 0; i--) { 3204 c = p[i]; 3205 if (c != 0) { 3206 while ((c & 0x80) == 0) { 3207 sig--; 3208 c <<= 1; 3209 } 3210 break; 3211 } 3212 sig -= 8; 3213 } 3214 return (sig); 3215 } 3216 3217 static void 3218 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits, 3219 u_int8_t *dst, u_int dstbits) 3220 { 3221 u_int slen, dlen; 3222 int i, si, di, n; 3223 3224 slen = (srcbits + 7) / 8; 3225 dlen = (dstbits + 7) / 8; 3226 3227 for (i = 0; i < slen; i++) 3228 dst[i] = src[i]; 3229 for (i = 0; i < dlen - slen; i++) 3230 dst[slen + i] = 0; 3231 3232 n = shiftbits / 8; 3233 if (n != 0) { 3234 si = dlen - n - 1; 3235 di = dlen - 1; 3236 while (si >= 0) 3237 dst[di--] = dst[si--]; 3238 while (di >= 0) 3239 dst[di--] = 0; 3240 } 3241 3242 n = shiftbits % 8; 3243 if (n != 0) { 3244 for (i = dlen - 1; i > 0; i--) 3245 dst[i] = (dst[i] << n) | 3246 (dst[i - 1] >> (8 - n)); 3247 dst[0] = dst[0] << n; 3248 } 3249 } 3250 3251 static void 3252 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits, 3253 u_int8_t *dst, u_int dstbits) 3254 { 3255 int slen, dlen, i, n; 3256 3257 slen = (srcbits + 7) / 8; 3258 dlen = (dstbits + 7) / 8; 3259 3260 n = shiftbits / 8; 3261 for (i = 0; i < slen; i++) 3262 dst[i] = src[i + n]; 3263 for (i = 0; i < dlen - slen; i++) 3264 dst[slen + i] = 0; 3265 3266 n = shiftbits % 8; 3267 if (n != 0) { 3268 for (i = 0; i < (dlen - 1); i++) 3269 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 3270 dst[dlen - 1] = dst[dlen - 1] >> n; 3271 } 3272 } 3273
Indexes created Wed Oct 01 19:09:53 GMT 2025