ubsec.c revision 1.45
1 /* $NetBSD: ubsec.c,v 1.45 2018/12/09 11:14:02 jdolecek Exp $ */ 2 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.6 2003/01/23 21:06:43 sam Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.143 2009/03/27 13:31:30 reyk Exp$ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason (at) thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt (at) openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik (at) ipunplugged.com) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 23 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 27 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 28 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Effort sponsored in part by the Defense Advanced Research Projects 32 * Agency (DARPA) and Air Force Research Laboratory, Air Force 33 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 34 * 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ubsec.c,v 1.45 2018/12/09 11:14:02 jdolecek Exp $"); 39 40 #undef UBSEC_DEBUG 41 42 /* 43 * uBsec 5[56]01, 58xx hardware crypto accelerator 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/endian.h> 50 #ifdef __NetBSD__ 51 #define UBSEC_NO_RNG /* hangs on attach */ 52 #define letoh16 htole16 53 #define letoh32 htole32 54 #endif 55 #include <sys/errno.h> 56 #include <sys/malloc.h> 57 #include <sys/kernel.h> 58 #include <sys/mbuf.h> 59 #include <sys/device.h> 60 #include <sys/module.h> 61 #include <sys/queue.h> 62 #include <sys/sysctl.h> 63 64 #include <opencrypto/cryptodev.h> 65 #include <opencrypto/xform.h> 66 #ifdef __OpenBSD__ 67 #include <dev/rndvar.h> 68 #include <sys/md5k.h> 69 #else 70 #include <sys/cprng.h> 71 #include <sys/md5.h> 72 #include <sys/rndpool.h> 73 #include <sys/rndsource.h> 74 #endif 75 #include <sys/sha1.h> 76 77 #include <dev/pci/pcireg.h> 78 #include <dev/pci/pcivar.h> 79 #include <dev/pci/pcidevs.h> 80 81 #include <dev/pci/ubsecreg.h> 82 #include <dev/pci/ubsecvar.h> 83 84 /* 85 * Prototypes and count for the pci_device structure 86 */ 87 static int ubsec_probe(device_t, cfdata_t, void *); 88 static void ubsec_attach(device_t, device_t, void *); 89 static int ubsec_detach(device_t, int); 90 static int ubsec_sysctl_init(void); 91 static void ubsec_reset_board(struct ubsec_softc *); 92 static void ubsec_init_board(struct ubsec_softc *); 93 static void ubsec_init_pciregs(struct pci_attach_args *pa); 94 static void ubsec_cleanchip(struct ubsec_softc *); 95 static void ubsec_totalreset(struct ubsec_softc *); 96 static int ubsec_free_q(struct ubsec_softc*, struct ubsec_q *); 97 98 #ifdef __OpenBSD__ 99 struct cfattach ubsec_ca = { 100 sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 101 }; 102 103 struct cfdriver ubsec_cd = { 104 0, "ubsec", DV_DULL 105 }; 106 #else 107 CFATTACH_DECL_NEW(ubsec, sizeof(struct ubsec_softc), ubsec_probe, ubsec_attach, 108 ubsec_detach, NULL); 109 extern struct cfdriver ubsec_cd; 110 #endif 111 112 /* patchable */ 113 #ifdef UBSEC_DEBUG 114 extern int ubsec_debug; 115 int ubsec_debug=1; 116 #endif 117 118 static int ubsec_intr(void *); 119 static int ubsec_newsession(void*, u_int32_t *, struct cryptoini *); 120 static int ubsec_freesession(void*, u_int64_t); 121 static int ubsec_process(void*, struct cryptop *, int hint); 122 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 123 static void ubsec_feed(struct ubsec_softc *); 124 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 125 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 126 static void ubsec_feed2(struct ubsec_softc *); 127 static void ubsec_feed4(struct ubsec_softc *); 128 #ifndef UBSEC_NO_RNG 129 static void ubsec_rng(void *); 130 static void ubsec_rng_locked(void *); 131 static void ubsec_rng_get(size_t, void *); 132 #endif /* UBSEC_NO_RNG */ 133 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 134 struct ubsec_dma_alloc *, int); 135 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 136 static int ubsec_dmamap_aligned(bus_dmamap_t); 137 138 static int ubsec_kprocess(void*, struct cryptkop *, int); 139 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, 140 struct cryptkop *, int); 141 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, 142 struct cryptkop *, int); 143 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, 144 struct cryptkop *, int); 145 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 146 static int ubsec_ksigbits(struct crparam *); 147 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 148 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 149 150 #ifdef UBSEC_DEBUG 151 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 152 static void ubsec_dump_mcr(struct ubsec_mcr *); 153 static void ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *); 154 #endif 155 156 #define READ_REG(sc,r) \ 157 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 158 159 #define WRITE_REG(sc,reg,val) \ 160 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 161 162 #define SWAP32(x) (x) = htole32(ntohl((x))) 163 #ifndef HTOLE32 164 #define HTOLE32(x) (x) = htole32(x) 165 #endif 166 167 struct ubsec_stats ubsecstats; 168 169 static struct sysctllog *ubsec_sysctllog; 170 171 /* 172 * ubsec_maxbatch controls the number of crypto ops to voluntarily 173 * collect into one submission to the hardware. This batching happens 174 * when ops are dispatched from the crypto subsystem with a hint that 175 * more are to follow immediately. These ops must also not be marked 176 * with a ``no delay'' flag. 177 */ 178 static int ubsec_maxbatch = 1; 179 180 /* 181 * ubsec_maxaggr controls the number of crypto ops to submit to the 182 * hardware as a unit. This aggregation reduces the number of interrupts 183 * to the host at the expense of increased latency (for all but the last 184 * operation). For network traffic setting this to one yields the highest 185 * performance but at the expense of more interrupt processing. 186 */ 187 static int ubsec_maxaggr = 1; 188 189 static const struct ubsec_product { 190 pci_vendor_id_t ubsec_vendor; 191 pci_product_id_t ubsec_product; 192 int ubsec_flags; 193 int ubsec_statmask; 194 int ubsec_maxaggr; 195 const char *ubsec_name; 196 } ubsec_products[] = { 197 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5501, 198 0, 199 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 200 UBS_MIN_AGGR, 201 "Bluesteel 5501" 202 }, 203 { PCI_VENDOR_BLUESTEEL, PCI_PRODUCT_BLUESTEEL_5601, 204 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 205 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 206 UBS_MIN_AGGR, 207 "Bluesteel 5601" 208 }, 209 210 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5801, 211 0, 212 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 213 UBS_MIN_AGGR, 214 "Broadcom BCM5801" 215 }, 216 217 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5802, 218 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 219 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 220 UBS_MIN_AGGR, 221 "Broadcom BCM5802" 222 }, 223 224 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5805, 225 UBS_FLAGS_KEY | UBS_FLAGS_RNG, 226 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 227 UBS_MIN_AGGR, 228 "Broadcom BCM5805" 229 }, 230 231 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5820, 232 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 233 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 234 BS_STAT_MCR1_DONE | BS_STAT_DMAERR, 235 UBS_MIN_AGGR, 236 "Broadcom BCM5820" 237 }, 238 239 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5821, 240 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 241 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 242 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 243 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 244 UBS_MIN_AGGR, 245 "Broadcom BCM5821" 246 }, 247 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_SCA1K, 248 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 249 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 250 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 251 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 252 UBS_MIN_AGGR, 253 "Sun Crypto Accelerator 1000" 254 }, 255 { PCI_VENDOR_SUN, PCI_PRODUCT_SUN_5821, 256 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 257 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 258 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 259 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 260 UBS_MIN_AGGR, 261 "Broadcom BCM5821 (Sun)" 262 }, 263 264 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5822, 265 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 266 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY, 267 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 268 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 269 UBS_MIN_AGGR, 270 "Broadcom BCM5822" 271 }, 272 273 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5823, 274 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 275 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 276 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 277 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 278 UBS_MIN_AGGR, 279 "Broadcom BCM5823" 280 }, 281 282 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5825, 283 UBS_FLAGS_KEY | UBS_FLAGS_RNG | UBS_FLAGS_LONGCTX | 284 UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 285 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 286 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY, 287 UBS_MIN_AGGR, 288 "Broadcom BCM5825" 289 }, 290 291 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5860, 292 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 293 UBS_FLAGS_LONGCTX | 294 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 295 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 296 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 297 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 298 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 299 UBS_MAX_AGGR, 300 "Broadcom BCM5860" 301 }, 302 303 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5861, 304 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 305 UBS_FLAGS_LONGCTX | 306 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 307 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 308 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 309 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 310 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 311 UBS_MAX_AGGR, 312 "Broadcom BCM5861" 313 }, 314 315 { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_5862, 316 UBS_FLAGS_MULTIMCR | UBS_FLAGS_HWNORM | 317 UBS_FLAGS_LONGCTX | 318 UBS_FLAGS_RNG | UBS_FLAGS_RNG4 | 319 UBS_FLAGS_KEY | UBS_FLAGS_BIGKEY | UBS_FLAGS_AES, 320 BS_STAT_MCR1_DONE | BS_STAT_DMAERR | 321 BS_STAT_MCR1_ALLEMPTY | BS_STAT_MCR2_ALLEMPTY | 322 BS_STAT_MCR3_ALLEMPTY | BS_STAT_MCR4_ALLEMPTY, 323 UBS_MAX_AGGR, 324 "Broadcom BCM5862" 325 }, 326 327 { 0, 0, 328 0, 329 0, 330 0, 331 NULL 332 } 333 }; 334 335 static const struct ubsec_product * 336 ubsec_lookup(const struct pci_attach_args *pa) 337 { 338 const struct ubsec_product *up; 339 340 for (up = ubsec_products; up->ubsec_name != NULL; up++) { 341 if (PCI_VENDOR(pa->pa_id) == up->ubsec_vendor && 342 PCI_PRODUCT(pa->pa_id) == up->ubsec_product) 343 return (up); 344 } 345 return (NULL); 346 } 347 348 static int 349 ubsec_probe(device_t parent, cfdata_t match, void *aux) 350 { 351 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 352 353 if (ubsec_lookup(pa) != NULL) 354 return (1); 355 356 return (0); 357 } 358 359 static void 360 ubsec_attach(device_t parent, device_t self, void *aux) 361 { 362 struct ubsec_softc *sc = device_private(self); 363 struct pci_attach_args *pa = aux; 364 const struct ubsec_product *up; 365 pci_chipset_tag_t pc = pa->pa_pc; 366 pci_intr_handle_t ih; 367 const char *intrstr = NULL; 368 pcireg_t memtype; 369 struct ubsec_dma *dmap; 370 u_int32_t cmd, i; 371 char intrbuf[PCI_INTRSTR_LEN]; 372 373 sc->sc_dev = self; 374 sc->sc_pct = pc; 375 376 up = ubsec_lookup(pa); 377 if (up == NULL) { 378 printf("\n"); 379 panic("ubsec_attach: impossible"); 380 } 381 382 pci_aprint_devinfo_fancy(pa, "Crypto processor", up->ubsec_name, 1); 383 384 SIMPLEQ_INIT(&sc->sc_queue); 385 SIMPLEQ_INIT(&sc->sc_qchip); 386 SIMPLEQ_INIT(&sc->sc_queue2); 387 SIMPLEQ_INIT(&sc->sc_qchip2); 388 SIMPLEQ_INIT(&sc->sc_queue4); 389 SIMPLEQ_INIT(&sc->sc_qchip4); 390 SIMPLEQ_INIT(&sc->sc_q2free); 391 392 sc->sc_flags = up->ubsec_flags; 393 sc->sc_statmask = up->ubsec_statmask; 394 sc->sc_maxaggr = up->ubsec_maxaggr; 395 396 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 397 cmd |= PCI_COMMAND_MASTER_ENABLE; 398 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 399 400 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, BS_BAR); 401 if (pci_mapreg_map(pa, BS_BAR, memtype, 0, 402 &sc->sc_st, &sc->sc_sh, NULL, &sc->sc_memsize)) { 403 aprint_error_dev(self, "can't find mem space"); 404 return; 405 } 406 407 sc->sc_dmat = pa->pa_dmat; 408 409 if (pci_intr_map(pa, &ih)) { 410 aprint_error_dev(self, "couldn't map interrupt\n"); 411 return; 412 } 413 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 414 sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, ubsec_intr, sc, 415 device_xname(self)); 416 if (sc->sc_ih == NULL) { 417 aprint_error_dev(self, "couldn't establish interrupt"); 418 if (intrstr != NULL) 419 aprint_error(" at %s", intrstr); 420 aprint_error("\n"); 421 return; 422 } 423 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 424 425 sc->sc_cid = crypto_get_driverid(0); 426 if (sc->sc_cid < 0) { 427 aprint_error_dev(self, "couldn't get crypto driver id\n"); 428 pci_intr_disestablish(pc, sc->sc_ih); 429 return; 430 } 431 432 sc->sc_rng_need = RND_POOLBITS / NBBY; 433 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 434 435 SIMPLEQ_INIT(&sc->sc_freequeue); 436 dmap = sc->sc_dmaa; 437 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 438 struct ubsec_q *q; 439 440 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 441 M_DEVBUF, M_ZERO|M_NOWAIT); 442 if (q == NULL) { 443 aprint_error_dev(self, 444 "can't allocate queue buffers\n"); 445 break; 446 } 447 448 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 449 &dmap->d_alloc, 0)) { 450 aprint_error_dev(self, "can't allocate dma buffers\n"); 451 free(q, M_DEVBUF); 452 break; 453 } 454 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 455 456 q->q_dma = dmap; 457 sc->sc_queuea[i] = q; 458 459 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 460 } 461 462 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 463 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 464 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 465 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 466 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 467 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 468 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 469 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 470 if (sc->sc_flags & UBS_FLAGS_AES) { 471 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 472 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 473 } 474 475 /* 476 * Reset Broadcom chip 477 */ 478 ubsec_reset_board(sc); 479 480 /* 481 * Init Broadcom specific PCI settings 482 */ 483 ubsec_init_pciregs(pa); 484 485 /* 486 * Init Broadcom chip 487 */ 488 ubsec_init_board(sc); 489 490 #ifndef UBSEC_NO_RNG 491 if (sc->sc_flags & UBS_FLAGS_RNG) { 492 if (sc->sc_flags & UBS_FLAGS_RNG4) 493 sc->sc_statmask |= BS_STAT_MCR4_DONE; 494 else 495 sc->sc_statmask |= BS_STAT_MCR2_DONE; 496 497 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 498 &sc->sc_rng.rng_q.q_mcr, 0)) 499 goto skip_rng; 500 501 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 502 &sc->sc_rng.rng_q.q_ctx, 0)) { 503 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 504 goto skip_rng; 505 } 506 507 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 508 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 509 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 510 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 511 goto skip_rng; 512 } 513 514 rndsource_setcb(&sc->sc_rnd_source, ubsec_rng_get, sc); 515 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dev), 516 RND_TYPE_RNG, 517 RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB); 518 if (hz >= 100) 519 sc->sc_rnghz = hz / 100; 520 else 521 sc->sc_rnghz = 1; 522 #ifdef __OpenBSD__ 523 timeout_set(&sc->sc_rngto, ubsec_rng, sc); 524 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 525 #else 526 callout_init(&sc->sc_rngto, 0); 527 callout_setfunc(&sc->sc_rngto, ubsec_rng, sc); 528 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 529 #endif 530 skip_rng: 531 if (sc->sc_rnghz) 532 aprint_normal_dev(self, 533 "random number generator enabled\n"); 534 else 535 aprint_error_dev(self, 536 "WARNING: random number generator disabled\n"); 537 } 538 #endif /* UBSEC_NO_RNG */ 539 540 if (sc->sc_flags & UBS_FLAGS_KEY) { 541 sc->sc_statmask |= BS_STAT_MCR2_DONE; 542 543 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 544 ubsec_kprocess, sc); 545 #if 0 546 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 547 ubsec_kprocess, sc); 548 #endif 549 } 550 } 551 552 static int 553 ubsec_detach(device_t self, int flags) 554 { 555 struct ubsec_softc *sc = device_private(self); 556 struct ubsec_q *q, *qtmp; 557 volatile u_int32_t ctrl; 558 559 /* disable interrupts */ 560 /* XXX wait/abort current ops? where is DMAERR enabled? */ 561 ctrl = READ_REG(sc, BS_CTRL); 562 563 ctrl &= ~(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR); 564 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 565 ctrl &= ~BS_CTRL_MCR4INT; 566 567 WRITE_REG(sc, BS_CTRL, ctrl); 568 569 #ifndef UBSEC_NO_RNG 570 if (sc->sc_flags & UBS_FLAGS_RNG) { 571 callout_halt(&sc->sc_rngto, NULL); 572 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 573 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 574 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 575 rnd_detach_source(&sc->sc_rnd_source); 576 } 577 #endif /* UBSEC_NO_RNG */ 578 579 crypto_unregister_all(sc->sc_cid); 580 581 mutex_spin_enter(&sc->sc_mtx); 582 583 ubsec_totalreset(sc); /* XXX leaves the chip running */ 584 585 SIMPLEQ_FOREACH_SAFE(q, &sc->sc_freequeue, q_next, qtmp) { 586 ubsec_dma_free(sc, &q->q_dma->d_alloc); 587 if (q->q_src_map != NULL) 588 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 589 if (q->q_cached_dst_map != NULL) 590 bus_dmamap_destroy(sc->sc_dmat, q->q_cached_dst_map); 591 free(q, M_DEVBUF); 592 } 593 594 mutex_spin_exit(&sc->sc_mtx); 595 596 if (sc->sc_ih != NULL) { 597 pci_intr_disestablish(sc->sc_pct, sc->sc_ih); 598 sc->sc_ih = NULL; 599 } 600 601 if (sc->sc_memsize != 0) { 602 bus_space_unmap(sc->sc_st, sc->sc_sh, sc->sc_memsize); 603 sc->sc_memsize = 0; 604 } 605 606 return 0; 607 } 608 609 MODULE(MODULE_CLASS_DRIVER, ubsec, "pci,opencrypto"); 610 611 #ifdef _MODULE 612 #include "ioconf.c" 613 #endif 614 615 static int 616 ubsec_modcmd(modcmd_t cmd, void *data) 617 { 618 int error = 0; 619 620 switch (cmd) { 621 case MODULE_CMD_INIT: 622 #ifdef _MODULE 623 error = config_init_component(cfdriver_ioconf_ubsec, 624 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 625 #endif 626 if (error == 0) 627 error = ubsec_sysctl_init(); 628 return error; 629 case MODULE_CMD_FINI: 630 if (ubsec_sysctllog != NULL) 631 sysctl_teardown(&ubsec_sysctllog); 632 #ifdef _MODULE 633 error = config_fini_component(cfdriver_ioconf_ubsec, 634 cfattach_ioconf_ubsec, cfdata_ioconf_ubsec); 635 #endif 636 return error; 637 default: 638 return ENOTTY; 639 } 640 } 641 642 static int 643 ubsec_sysctl_init(void) 644 { 645 const struct sysctlnode *node = NULL; 646 647 ubsec_sysctllog = NULL; 648 649 sysctl_createv(&ubsec_sysctllog, 0, NULL, &node, 650 CTLFLAG_PERMANENT, 651 CTLTYPE_NODE, "ubsec", 652 SYSCTL_DESCR("ubsec opetions"), 653 NULL, 0, NULL, 0, 654 CTL_HW, CTL_CREATE, CTL_EOL); 655 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 656 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 657 CTLTYPE_INT, "maxbatch", 658 SYSCTL_DESCR("max ops to batch w/o interrupt"), 659 NULL, 0, &ubsec_maxbatch, 0, 660 CTL_CREATE, CTL_EOL); 661 sysctl_createv(&ubsec_sysctllog, 0, &node, NULL, 662 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 663 CTLTYPE_INT, "maxaggr", 664 SYSCTL_DESCR("max ops to aggregate under one interrupt"), 665 NULL, 0, &ubsec_maxaggr, 0, 666 CTL_CREATE, CTL_EOL); 667 668 return 0; 669 } 670 671 /* 672 * UBSEC Interrupt routine 673 */ 674 static int 675 ubsec_intr(void *arg) 676 { 677 struct ubsec_softc *sc = arg; 678 volatile u_int32_t stat; 679 struct ubsec_q *q; 680 struct ubsec_dma *dmap; 681 int flags; 682 int npkts = 0, i; 683 684 mutex_spin_enter(&sc->sc_mtx); 685 stat = READ_REG(sc, BS_STAT); 686 stat &= sc->sc_statmask; 687 if (stat == 0) { 688 mutex_spin_exit(&sc->sc_mtx); 689 return (0); 690 } 691 692 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 693 694 /* 695 * Check to see if we have any packets waiting for us 696 */ 697 if ((stat & BS_STAT_MCR1_DONE)) { 698 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 699 q = SIMPLEQ_FIRST(&sc->sc_qchip); 700 dmap = q->q_dma; 701 702 if ((dmap->d_dma->d_mcr.mcr_flags 703 & htole16(UBS_MCR_DONE)) == 0) 704 break; 705 706 q = SIMPLEQ_FIRST(&sc->sc_qchip); 707 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 708 709 npkts = q->q_nstacked_mcrs; 710 sc->sc_nqchip -= 1+npkts; 711 /* 712 * search for further sc_qchip ubsec_q's that share 713 * the same MCR, and complete them too, they must be 714 * at the top. 715 */ 716 for (i = 0; i < npkts; i++) { 717 if(q->q_stacked_mcr[i]) 718 ubsec_callback(sc, q->q_stacked_mcr[i]); 719 else 720 break; 721 } 722 ubsec_callback(sc, q); 723 } 724 725 /* 726 * Don't send any more packet to chip if there has been 727 * a DMAERR. 728 */ 729 if (!(stat & BS_STAT_DMAERR)) 730 ubsec_feed(sc); 731 } 732 733 /* 734 * Check to see if we have any key setups/rng's waiting for us 735 */ 736 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 737 (stat & BS_STAT_MCR2_DONE)) { 738 struct ubsec_q2 *q2; 739 struct ubsec_mcr *mcr; 740 741 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 742 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 743 744 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 745 0, q2->q_mcr.dma_map->dm_mapsize, 746 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 747 748 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 749 750 /* A bug in new devices requires to swap this field */ 751 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) 752 flags = htole16(mcr->mcr_flags); 753 else 754 flags = mcr->mcr_flags; 755 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 756 bus_dmamap_sync(sc->sc_dmat, 757 q2->q_mcr.dma_map, 0, 758 q2->q_mcr.dma_map->dm_mapsize, 759 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 760 break; 761 } 762 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 763 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, /*q2,*/ q_next); 764 ubsec_callback2(sc, q2); 765 /* 766 * Don't send any more packet to chip if there has been 767 * a DMAERR. 768 */ 769 if (!(stat & BS_STAT_DMAERR)) 770 ubsec_feed2(sc); 771 } 772 } 773 if ((sc->sc_flags & UBS_FLAGS_RNG4) && (stat & BS_STAT_MCR4_DONE)) { 774 struct ubsec_q2 *q2; 775 struct ubsec_mcr *mcr; 776 777 while (!SIMPLEQ_EMPTY(&sc->sc_qchip4)) { 778 q2 = SIMPLEQ_FIRST(&sc->sc_qchip4); 779 780 bus_dmamap_sync(sc->sc_dmat, q2->q_mcr.dma_map, 781 0, q2->q_mcr.dma_map->dm_mapsize, 782 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 783 784 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 785 786 /* A bug in new devices requires to swap this field */ 787 flags = htole16(mcr->mcr_flags); 788 789 if ((flags & htole16(UBS_MCR_DONE)) == 0) { 790 bus_dmamap_sync(sc->sc_dmat, 791 q2->q_mcr.dma_map, 0, 792 q2->q_mcr.dma_map->dm_mapsize, 793 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 794 break; 795 } 796 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip4, q_next); 797 ubsec_callback2(sc, q2); 798 /* 799 * Don't send any more packet to chip if there has been 800 * a DMAERR. 801 */ 802 if (!(stat & BS_STAT_DMAERR)) 803 ubsec_feed4(sc); 804 } 805 } 806 807 /* 808 * Check to see if we got any DMA Error 809 */ 810 if (stat & BS_STAT_DMAERR) { 811 #ifdef UBSEC_DEBUG 812 if (ubsec_debug) { 813 volatile u_int32_t a = READ_REG(sc, BS_ERR); 814 815 printf("%s: dmaerr %s@%08x\n", device_xname(sc->sc_dev), 816 (a & BS_ERR_READ) ? "read" : "write", 817 a & BS_ERR_ADDR); 818 } 819 #endif /* UBSEC_DEBUG */ 820 ubsecstats.hst_dmaerr++; 821 ubsec_totalreset(sc); 822 ubsec_feed(sc); 823 } 824 825 if (sc->sc_needwakeup) { /* XXX check high watermark */ 826 int wkeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 827 #ifdef UBSEC_DEBUG 828 if (ubsec_debug) 829 printf("%s: wakeup crypto (%x)\n", 830 device_xname(sc->sc_dev), sc->sc_needwakeup); 831 #endif /* UBSEC_DEBUG */ 832 sc->sc_needwakeup &= ~wkeup; 833 crypto_unblock(sc->sc_cid, wkeup); 834 } 835 mutex_spin_exit(&sc->sc_mtx); 836 return (1); 837 } 838 839 /* 840 * ubsec_feed() - aggregate and post requests to chip 841 * OpenBSD comments: 842 * It is assumed that the caller set splnet() 843 */ 844 static void 845 ubsec_feed(struct ubsec_softc *sc) 846 { 847 struct ubsec_q *q, *q2; 848 int npkts, i; 849 void *v; 850 u_int32_t stat; 851 #ifdef UBSEC_DEBUG 852 static int max; 853 #endif /* UBSEC_DEBUG */ 854 855 npkts = sc->sc_nqueue; 856 if (npkts > ubsecstats.hst_maxqueue) 857 ubsecstats.hst_maxqueue = npkts; 858 if (npkts < 2) 859 goto feed1; 860 861 /* 862 * Decide how many ops to combine in a single MCR. We cannot 863 * aggregate more than UBS_MAX_AGGR because this is the number 864 * of slots defined in the data structure. Otherwise we clamp 865 * based on the tunable parameter ubsec_maxaggr. Note that 866 * aggregation can happen in two ways: either by batching ops 867 * from above or because the h/w backs up and throttles us. 868 * Aggregating ops reduces the number of interrupts to the host 869 * but also (potentially) increases the latency for processing 870 * completed ops as we only get an interrupt when all aggregated 871 * ops have completed. 872 */ 873 if (npkts > sc->sc_maxaggr) 874 npkts = sc->sc_maxaggr; 875 if (npkts > ubsec_maxaggr) 876 npkts = ubsec_maxaggr; 877 if (npkts > ubsecstats.hst_maxbatch) 878 ubsecstats.hst_maxbatch = npkts; 879 if (npkts < 2) 880 goto feed1; 881 ubsecstats.hst_totbatch += npkts-1; 882 883 if ((stat = READ_REG(sc, BS_STAT)) 884 & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 885 if (stat & BS_STAT_DMAERR) { 886 ubsec_totalreset(sc); 887 ubsecstats.hst_dmaerr++; 888 } else { 889 ubsecstats.hst_mcr1full++; 890 } 891 return; 892 } 893 894 #ifdef UBSEC_DEBUG 895 if (ubsec_debug) 896 printf("merging %d records\n", npkts); 897 /* XXX temporary aggregation statistics reporting code */ 898 if (max < npkts) { 899 max = npkts; 900 printf("%s: new max aggregate %d\n", device_xname(sc->sc_dev), 901 max); 902 } 903 #endif /* UBSEC_DEBUG */ 904 905 q = SIMPLEQ_FIRST(&sc->sc_queue); 906 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 907 --sc->sc_nqueue; 908 909 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 910 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 911 if (q->q_dst_map != NULL) 912 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 913 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 914 915 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 916 917 for (i = 0; i < q->q_nstacked_mcrs; i++) { 918 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 919 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 920 0, q2->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 921 if (q2->q_dst_map != NULL) 922 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 923 0, q2->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 924 q2= SIMPLEQ_FIRST(&sc->sc_queue); 925 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q2,*/ q_next); 926 --sc->sc_nqueue; 927 928 v = ((void *)&q2->q_dma->d_dma->d_mcr); 929 v = (char*)v + (sizeof(struct ubsec_mcr) - 930 sizeof(struct ubsec_mcr_add)); 931 memcpy(&q->q_dma->d_dma->d_mcradd[i], v, 932 sizeof(struct ubsec_mcr_add)); 933 q->q_stacked_mcr[i] = q2; 934 } 935 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 936 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 937 sc->sc_nqchip += npkts; 938 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 939 ubsecstats.hst_maxqchip = sc->sc_nqchip; 940 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 941 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 942 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 943 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 944 offsetof(struct ubsec_dmachunk, d_mcr)); 945 return; 946 947 feed1: 948 while (!SIMPLEQ_EMPTY(&sc->sc_queue)) { 949 if ((stat = READ_REG(sc, BS_STAT)) 950 & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 951 if (stat & BS_STAT_DMAERR) { 952 ubsec_totalreset(sc); 953 ubsecstats.hst_dmaerr++; 954 } else { 955 ubsecstats.hst_mcr1full++; 956 } 957 break; 958 } 959 960 q = SIMPLEQ_FIRST(&sc->sc_queue); 961 962 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 963 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 964 if (q->q_dst_map != NULL) 965 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 966 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 967 bus_dmamap_sync(sc->sc_dmat, q->q_dma->d_alloc.dma_map, 968 0, q->q_dma->d_alloc.dma_map->dm_mapsize, 969 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 970 971 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 972 offsetof(struct ubsec_dmachunk, d_mcr)); 973 #ifdef UBSEC_DEBUG 974 if (ubsec_debug) 975 printf("feed: q->chip %p %08x stat %08x\n", 976 q, (u_int32_t)q->q_dma->d_alloc.dma_paddr, 977 stat); 978 #endif /* UBSEC_DEBUG */ 979 q = SIMPLEQ_FIRST(&sc->sc_queue); 980 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, /*q,*/ q_next); 981 --sc->sc_nqueue; 982 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 983 sc->sc_nqchip++; 984 } 985 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 986 ubsecstats.hst_maxqchip = sc->sc_nqchip; 987 } 988 989 /* 990 * Allocate a new 'session' and return an encoded session id. 'sidp' 991 * contains our registration id, and should contain an encoded session 992 * id on successful allocation. 993 */ 994 static int 995 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 996 { 997 struct cryptoini *c, *encini = NULL, *macini = NULL; 998 struct ubsec_softc *sc; 999 struct ubsec_session *ses = NULL; 1000 MD5_CTX md5ctx; 1001 SHA1_CTX sha1ctx; 1002 int i, sesn; 1003 1004 sc = arg; 1005 KASSERT(sc != NULL /*, ("ubsec_newsession: null softc")*/); 1006 1007 if (sidp == NULL || cri == NULL || sc == NULL) 1008 return (EINVAL); 1009 1010 for (c = cri; c != NULL; c = c->cri_next) { 1011 if (c->cri_alg == CRYPTO_MD5_HMAC_96 || 1012 c->cri_alg == CRYPTO_SHA1_HMAC_96) { 1013 if (macini) 1014 return (EINVAL); 1015 macini = c; 1016 } else if (c->cri_alg == CRYPTO_DES_CBC || 1017 c->cri_alg == CRYPTO_3DES_CBC || 1018 c->cri_alg == CRYPTO_AES_CBC) { 1019 if (encini) 1020 return (EINVAL); 1021 encini = c; 1022 } else 1023 return (EINVAL); 1024 } 1025 if (encini == NULL && macini == NULL) 1026 return (EINVAL); 1027 1028 if (encini && encini->cri_alg == CRYPTO_AES_CBC) { 1029 switch (encini->cri_klen) { 1030 case 128: 1031 case 192: 1032 case 256: 1033 break; 1034 default: 1035 return (EINVAL); 1036 } 1037 } 1038 1039 if (sc->sc_sessions == NULL) { 1040 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 1041 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1042 if (ses == NULL) 1043 return (ENOMEM); 1044 sesn = 0; 1045 sc->sc_nsessions = 1; 1046 } else { 1047 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 1048 if (sc->sc_sessions[sesn].ses_used == 0) { 1049 ses = &sc->sc_sessions[sesn]; 1050 break; 1051 } 1052 } 1053 1054 if (ses == NULL) { 1055 sesn = sc->sc_nsessions; 1056 ses = (struct ubsec_session *)malloc((sesn + 1) * 1057 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 1058 if (ses == NULL) 1059 return (ENOMEM); 1060 memcpy(ses, sc->sc_sessions, sesn * 1061 sizeof(struct ubsec_session)); 1062 memset(sc->sc_sessions, 0, sesn * 1063 sizeof(struct ubsec_session)); 1064 free(sc->sc_sessions, M_DEVBUF); 1065 sc->sc_sessions = ses; 1066 ses = &sc->sc_sessions[sesn]; 1067 sc->sc_nsessions++; 1068 } 1069 } 1070 1071 memset(ses, 0, sizeof(struct ubsec_session)); 1072 ses->ses_used = 1; 1073 if (encini) { 1074 /* get an IV, network byte order */ 1075 #ifdef __NetBSD__ 1076 cprng_fast(ses->ses_iv, sizeof(ses->ses_iv)); 1077 #else 1078 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv)); 1079 #endif 1080 1081 /* Go ahead and compute key in ubsec's byte order */ 1082 if (encini->cri_alg == CRYPTO_AES_CBC) { 1083 memcpy(ses->ses_key, encini->cri_key, 1084 encini->cri_klen / 8); 1085 } 1086 if (encini->cri_alg == CRYPTO_DES_CBC) { 1087 memcpy(&ses->ses_key[0], encini->cri_key, 8); 1088 memcpy(&ses->ses_key[2], encini->cri_key, 8); 1089 memcpy(&ses->ses_key[4], encini->cri_key, 8); 1090 } else 1091 memcpy(ses->ses_key, encini->cri_key, 24); 1092 1093 SWAP32(ses->ses_key[0]); 1094 SWAP32(ses->ses_key[1]); 1095 SWAP32(ses->ses_key[2]); 1096 SWAP32(ses->ses_key[3]); 1097 SWAP32(ses->ses_key[4]); 1098 SWAP32(ses->ses_key[5]); 1099 } 1100 1101 if (macini) { 1102 for (i = 0; i < macini->cri_klen / 8; i++) 1103 macini->cri_key[i] ^= HMAC_IPAD_VAL; 1104 1105 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1106 MD5Init(&md5ctx); 1107 MD5Update(&md5ctx, macini->cri_key, 1108 macini->cri_klen / 8); 1109 MD5Update(&md5ctx, hmac_ipad_buffer, 1110 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1111 memcpy(ses->ses_hminner, md5ctx.state, 1112 sizeof(md5ctx.state)); 1113 } else { 1114 SHA1Init(&sha1ctx); 1115 SHA1Update(&sha1ctx, macini->cri_key, 1116 macini->cri_klen / 8); 1117 SHA1Update(&sha1ctx, hmac_ipad_buffer, 1118 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1119 memcpy(ses->ses_hminner, sha1ctx.state, 1120 sizeof(sha1ctx.state)); 1121 } 1122 1123 for (i = 0; i < macini->cri_klen / 8; i++) 1124 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 1125 1126 if (macini->cri_alg == CRYPTO_MD5_HMAC_96) { 1127 MD5Init(&md5ctx); 1128 MD5Update(&md5ctx, macini->cri_key, 1129 macini->cri_klen / 8); 1130 MD5Update(&md5ctx, hmac_opad_buffer, 1131 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1132 memcpy(ses->ses_hmouter, md5ctx.state, 1133 sizeof(md5ctx.state)); 1134 } else { 1135 SHA1Init(&sha1ctx); 1136 SHA1Update(&sha1ctx, macini->cri_key, 1137 macini->cri_klen / 8); 1138 SHA1Update(&sha1ctx, hmac_opad_buffer, 1139 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 1140 memcpy(ses->ses_hmouter, sha1ctx.state, 1141 sizeof(sha1ctx.state)); 1142 } 1143 1144 for (i = 0; i < macini->cri_klen / 8; i++) 1145 macini->cri_key[i] ^= HMAC_OPAD_VAL; 1146 } 1147 1148 *sidp = UBSEC_SID(device_unit(sc->sc_dev), sesn); 1149 return (0); 1150 } 1151 1152 /* 1153 * Deallocate a session. 1154 */ 1155 static int 1156 ubsec_freesession(void *arg, u_int64_t tid) 1157 { 1158 struct ubsec_softc *sc; 1159 int session; 1160 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 1161 1162 sc = arg; 1163 KASSERT(sc != NULL /*, ("ubsec_freesession: null softc")*/); 1164 1165 session = UBSEC_SESSION(sid); 1166 if (session >= sc->sc_nsessions) 1167 return (EINVAL); 1168 1169 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 1170 return (0); 1171 } 1172 1173 #ifdef __FreeBSD__ /* Ugly gratuitous changes to bus_dma */ 1174 static void 1175 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, 1176 int error) 1177 { 1178 struct ubsec_operand *op = arg; 1179 1180 KASSERT(nsegs <= UBS_MAX_SCATTER 1181 /*, ("Too many DMA segments returned when mapping operand")*/); 1182 #ifdef UBSEC_DEBUG 1183 if (ubsec_debug) 1184 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 1185 (u_int) mapsize, nsegs); 1186 #endif 1187 op->mapsize = mapsize; 1188 op->nsegs = nsegs; 1189 memcpy(op->segs, seg, nsegs * sizeof (seg[0])); 1190 } 1191 #endif 1192 1193 static int 1194 ubsec_process(void *arg, struct cryptop *crp, int hint) 1195 { 1196 struct ubsec_q *q = NULL; 1197 #ifdef __OpenBSD__ 1198 int card; 1199 #endif 1200 int err = 0, i, j, nicealign; 1201 struct ubsec_softc *sc; 1202 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1203 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1204 int sskip, dskip, stheend, dtheend; 1205 int16_t coffset; 1206 struct ubsec_session *ses, key; 1207 struct ubsec_dma *dmap = NULL; 1208 u_int16_t flags = 0; 1209 int ivlen = 0, keylen = 0; 1210 1211 sc = arg; 1212 KASSERT(sc != NULL /*, ("ubsec_process: null softc")*/); 1213 1214 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1215 ubsecstats.hst_invalid++; 1216 return (EINVAL); 1217 } 1218 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1219 ubsecstats.hst_badsession++; 1220 return (EINVAL); 1221 } 1222 1223 mutex_spin_enter(&sc->sc_mtx); 1224 1225 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1226 ubsecstats.hst_queuefull++; 1227 sc->sc_needwakeup |= CRYPTO_SYMQ; 1228 mutex_spin_exit(&sc->sc_mtx); 1229 return(ERESTART); 1230 } 1231 1232 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1233 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, /*q,*/ q_next); 1234 mutex_spin_exit(&sc->sc_mtx); 1235 1236 dmap = q->q_dma; /* Save dma pointer */ 1237 /* don't lose the cached dmamaps q_src_map and q_cached_dst_map */ 1238 memset(q, 0, offsetof(struct ubsec_q, q_src_map)); 1239 memset(&key, 0, sizeof(key)); 1240 1241 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1242 q->q_dma = dmap; 1243 ses = &sc->sc_sessions[q->q_sesn]; 1244 1245 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1246 q->q_src_m = (struct mbuf *)crp->crp_buf; 1247 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1248 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1249 q->q_src_io = (struct uio *)crp->crp_buf; 1250 q->q_dst_io = (struct uio *)crp->crp_buf; 1251 } else { 1252 ubsecstats.hst_badflags++; 1253 err = EINVAL; 1254 goto errout; /* XXX we don't handle contiguous blocks! */ 1255 } 1256 1257 memset(&dmap->d_dma->d_mcr, 0, sizeof(struct ubsec_mcr)); 1258 1259 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1260 dmap->d_dma->d_mcr.mcr_flags = 0; 1261 q->q_crp = crp; 1262 1263 crd1 = crp->crp_desc; 1264 if (crd1 == NULL) { 1265 ubsecstats.hst_nodesc++; 1266 err = EINVAL; 1267 goto errout; 1268 } 1269 crd2 = crd1->crd_next; 1270 1271 if (crd2 == NULL) { 1272 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1273 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) { 1274 maccrd = crd1; 1275 enccrd = NULL; 1276 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1277 crd1->crd_alg == CRYPTO_3DES_CBC || 1278 crd1->crd_alg == CRYPTO_AES_CBC) { 1279 maccrd = NULL; 1280 enccrd = crd1; 1281 } else { 1282 ubsecstats.hst_badalg++; 1283 err = EINVAL; 1284 goto errout; 1285 } 1286 } else { 1287 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 1288 crd1->crd_alg == CRYPTO_SHA1_HMAC_96) && 1289 (crd2->crd_alg == CRYPTO_DES_CBC || 1290 crd2->crd_alg == CRYPTO_3DES_CBC || 1291 crd2->crd_alg == CRYPTO_AES_CBC) && 1292 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1293 maccrd = crd1; 1294 enccrd = crd2; 1295 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1296 crd1->crd_alg == CRYPTO_3DES_CBC || 1297 crd1->crd_alg == CRYPTO_AES_CBC) && 1298 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 1299 crd2->crd_alg == CRYPTO_SHA1_HMAC_96) && 1300 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1301 enccrd = crd1; 1302 maccrd = crd2; 1303 } else { 1304 /* 1305 * We cannot order the ubsec as requested 1306 */ 1307 ubsecstats.hst_badalg++; 1308 err = EINVAL; 1309 goto errout; 1310 } 1311 } 1312 1313 if (enccrd) { 1314 if (enccrd->crd_alg == CRYPTO_AES_CBC) { 1315 if ((sc->sc_flags & UBS_FLAGS_AES) == 0) { 1316 /* 1317 * We cannot order the ubsec as requested 1318 */ 1319 ubsecstats.hst_badalg++; 1320 err = EINVAL; 1321 goto errout; 1322 } 1323 flags |= htole16(UBS_PKTCTX_ENC_AES); 1324 switch (enccrd->crd_klen) { 1325 case 128: 1326 case 192: 1327 case 256: 1328 keylen = enccrd->crd_klen / 8; 1329 break; 1330 default: 1331 err = EINVAL; 1332 goto errout; 1333 } 1334 ivlen = 16; 1335 } else { 1336 flags |= htole16(UBS_PKTCTX_ENC_3DES); 1337 ivlen = 8; 1338 keylen = 24; 1339 } 1340 1341 encoffset = enccrd->crd_skip; 1342 1343 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1344 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1345 1346 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1347 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1348 else { 1349 for (i = 0; i < (ivlen / 4); i++) 1350 key.ses_iv[i] = ses->ses_iv[i]; 1351 } 1352 1353 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1354 if (crp->crp_flags & CRYPTO_F_IMBUF) 1355 m_copyback(q->q_src_m, 1356 enccrd->crd_inject, 1357 ivlen, (void *)key.ses_iv); 1358 else if (crp->crp_flags & CRYPTO_F_IOV) 1359 cuio_copyback(q->q_src_io, 1360 enccrd->crd_inject, 1361 ivlen, (void *)key.ses_iv); 1362 } 1363 } else { 1364 flags |= htole16(UBS_PKTCTX_INBOUND); 1365 1366 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1367 memcpy(key.ses_iv, enccrd->crd_iv, ivlen); 1368 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1369 m_copydata(q->q_src_m, enccrd->crd_inject, 1370 ivlen, (void *)key.ses_iv); 1371 else if (crp->crp_flags & CRYPTO_F_IOV) 1372 cuio_copydata(q->q_src_io, 1373 enccrd->crd_inject, 8, 1374 (void *)key.ses_iv); 1375 } 1376 1377 for (i = 0; i < (keylen / 4); i++) 1378 key.ses_key[i] = ses->ses_key[i]; 1379 for (i = 0; i < (ivlen / 4); i++) 1380 SWAP32(key.ses_iv[i]); 1381 } 1382 1383 if (maccrd) { 1384 macoffset = maccrd->crd_skip; 1385 1386 if (maccrd->crd_alg == CRYPTO_MD5_HMAC_96) 1387 flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1388 else 1389 flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1390 1391 for (i = 0; i < 5; i++) { 1392 key.ses_hminner[i] = ses->ses_hminner[i]; 1393 key.ses_hmouter[i] = ses->ses_hmouter[i]; 1394 1395 HTOLE32(key.ses_hminner[i]); 1396 HTOLE32(key.ses_hmouter[i]); 1397 } 1398 } 1399 1400 if (enccrd && maccrd) { 1401 /* 1402 * ubsec cannot handle packets where the end of encryption 1403 * and authentication are not the same, or where the 1404 * encrypted part begins before the authenticated part. 1405 */ 1406 if ((encoffset + enccrd->crd_len) != 1407 (macoffset + maccrd->crd_len)) { 1408 ubsecstats.hst_lenmismatch++; 1409 err = EINVAL; 1410 goto errout; 1411 } 1412 if (enccrd->crd_skip < maccrd->crd_skip) { 1413 ubsecstats.hst_skipmismatch++; 1414 err = EINVAL; 1415 goto errout; 1416 } 1417 sskip = maccrd->crd_skip; 1418 cpskip = dskip = enccrd->crd_skip; 1419 stheend = maccrd->crd_len; 1420 dtheend = enccrd->crd_len; 1421 coffset = enccrd->crd_skip - maccrd->crd_skip; 1422 cpoffset = cpskip + dtheend; 1423 #ifdef UBSEC_DEBUG 1424 if (ubsec_debug) { 1425 printf("mac: skip %d, len %d, inject %d\n", 1426 maccrd->crd_skip, maccrd->crd_len, 1427 maccrd->crd_inject); 1428 printf("enc: skip %d, len %d, inject %d\n", 1429 enccrd->crd_skip, enccrd->crd_len, 1430 enccrd->crd_inject); 1431 printf("src: skip %d, len %d\n", sskip, stheend); 1432 printf("dst: skip %d, len %d\n", dskip, dtheend); 1433 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1434 coffset, stheend, cpskip, cpoffset); 1435 } 1436 #endif 1437 } else { 1438 cpskip = dskip = sskip = macoffset + encoffset; 1439 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1440 cpoffset = cpskip + dtheend; 1441 coffset = 0; 1442 } 1443 1444 if (q->q_src_map == NULL) { 1445 /* XXX FIXME: jonathan asks, what the heck's that 0xfff0? */ 1446 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, UBS_MAX_SCATTER, 1447 0xfff0, 0, BUS_DMA_NOWAIT, &q->q_src_map) != 0) { 1448 err = ENOMEM; 1449 goto errout; 1450 } 1451 } 1452 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1453 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1454 q->q_src_m, BUS_DMA_NOWAIT) != 0) { 1455 ubsecstats.hst_noload++; 1456 err = ENOMEM; 1457 goto errout; 1458 } 1459 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1460 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1461 q->q_src_io, BUS_DMA_NOWAIT) != 0) { 1462 ubsecstats.hst_noload++; 1463 err = ENOMEM; 1464 goto errout; 1465 } 1466 } 1467 nicealign = ubsec_dmamap_aligned(q->q_src_map); 1468 1469 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1470 1471 #ifdef UBSEC_DEBUG 1472 if (ubsec_debug) 1473 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1474 #endif 1475 for (i = j = 0; i < q->q_src_map->dm_nsegs; i++) { 1476 struct ubsec_pktbuf *pb; 1477 bus_size_t packl = q->q_src_map->dm_segs[i].ds_len; 1478 bus_addr_t packp = q->q_src_map->dm_segs[i].ds_addr; 1479 1480 if (sskip >= packl) { 1481 sskip -= packl; 1482 continue; 1483 } 1484 1485 packl -= sskip; 1486 packp += sskip; 1487 sskip = 0; 1488 1489 if (packl > 0xfffc) { 1490 err = EIO; 1491 goto errout; 1492 } 1493 1494 if (j == 0) 1495 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1496 else 1497 pb = &dmap->d_dma->d_sbuf[j - 1]; 1498 1499 pb->pb_addr = htole32(packp); 1500 1501 if (stheend) { 1502 if (packl > stheend) { 1503 pb->pb_len = htole32(stheend); 1504 stheend = 0; 1505 } else { 1506 pb->pb_len = htole32(packl); 1507 stheend -= packl; 1508 } 1509 } else 1510 pb->pb_len = htole32(packl); 1511 1512 if ((i + 1) == q->q_src_map->dm_nsegs) 1513 pb->pb_next = 0; 1514 else 1515 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1516 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1517 j++; 1518 } 1519 1520 if (enccrd == NULL && maccrd != NULL) { 1521 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1522 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1523 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1524 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1525 #ifdef UBSEC_DEBUG 1526 if (ubsec_debug) 1527 printf("opkt: %x %x %x\n", 1528 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1529 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1530 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1531 1532 #endif 1533 } else { 1534 if (crp->crp_flags & CRYPTO_F_IOV) { 1535 if (!nicealign) { 1536 ubsecstats.hst_iovmisaligned++; 1537 err = EINVAL; 1538 goto errout; 1539 } 1540 if (q->q_dst_map == NULL) { 1541 if (q->q_cached_dst_map == NULL) { 1542 /* 1543 * XXX: ``what the heck's that'' 1544 * 0xfff0? 1545 */ 1546 if (bus_dmamap_create(sc->sc_dmat, 1547 0xfff0, UBS_MAX_SCATTER, 0xfff0, 0, 1548 BUS_DMA_NOWAIT, 1549 &q->q_cached_dst_map) != 0) { 1550 ubsecstats.hst_nomap++; 1551 err = ENOMEM; 1552 goto errout; 1553 } 1554 } 1555 q->q_dst_map = q->q_cached_dst_map; 1556 } 1557 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1558 q->q_dst_io, BUS_DMA_NOWAIT) != 0) { 1559 ubsecstats.hst_noload++; 1560 err = ENOMEM; 1561 goto errout; 1562 } 1563 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1564 if (nicealign) { 1565 q->q_dst_m = q->q_src_m; 1566 q->q_dst_map = q->q_src_map; 1567 } else { 1568 int totlen, len; 1569 struct mbuf *m, *top, **mp; 1570 1571 ubsecstats.hst_unaligned++; 1572 totlen = q->q_src_map->dm_mapsize; 1573 if (q->q_src_m->m_flags & M_PKTHDR) { 1574 len = MHLEN; 1575 MGETHDR(m, M_DONTWAIT, MT_DATA); 1576 /*XXX FIXME: m_dup_pkthdr */ 1577 if (m && 1 /*!m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)*/) { 1578 m_free(m); 1579 m = NULL; 1580 } 1581 } else { 1582 len = MLEN; 1583 MGET(m, M_DONTWAIT, MT_DATA); 1584 } 1585 if (m == NULL) { 1586 ubsecstats.hst_nombuf++; 1587 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1588 goto errout; 1589 } 1590 if (len == MHLEN) 1591 /*XXX was M_DUP_PKTHDR*/ 1592 M_COPY_PKTHDR(m, q->q_src_m); 1593 if (totlen >= MINCLSIZE) { 1594 MCLGET(m, M_DONTWAIT); 1595 if ((m->m_flags & M_EXT) == 0) { 1596 m_free(m); 1597 ubsecstats.hst_nomcl++; 1598 err = sc->sc_nqueue 1599 ? ERESTART : ENOMEM; 1600 goto errout; 1601 } 1602 len = MCLBYTES; 1603 } 1604 m->m_len = len; 1605 top = NULL; 1606 mp = ⊤ 1607 1608 while (totlen > 0) { 1609 if (top) { 1610 MGET(m, M_DONTWAIT, MT_DATA); 1611 if (m == NULL) { 1612 m_freem(top); 1613 ubsecstats.hst_nombuf++; 1614 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1615 goto errout; 1616 } 1617 len = MLEN; 1618 } 1619 if (top && totlen >= MINCLSIZE) { 1620 MCLGET(m, M_DONTWAIT); 1621 if ((m->m_flags & M_EXT) == 0) { 1622 *mp = m; 1623 m_freem(top); 1624 ubsecstats.hst_nomcl++; 1625 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1626 goto errout; 1627 } 1628 len = MCLBYTES; 1629 } 1630 m->m_len = len = uimin(totlen, len); 1631 totlen -= len; 1632 *mp = m; 1633 mp = &m->m_next; 1634 } 1635 q->q_dst_m = top; 1636 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1637 cpskip, cpoffset); 1638 if (q->q_dst_map == NULL) { 1639 if (q->q_cached_dst_map == NULL) { 1640 /* XXX again, what the heck is that 0xfff0? */ 1641 if (bus_dmamap_create(sc->sc_dmat, 0xfff0, 1642 UBS_MAX_SCATTER, 0xfff0, 0, BUS_DMA_NOWAIT, 1643 &q->q_cached_dst_map) != 0) { 1644 ubsecstats.hst_nomap++; 1645 err = ENOMEM; 1646 goto errout; 1647 } 1648 } 1649 q->q_dst_map = q->q_cached_dst_map; 1650 } 1651 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1652 q->q_dst_map, q->q_dst_m, 1653 BUS_DMA_NOWAIT) != 0) { 1654 ubsecstats.hst_noload++; 1655 err = ENOMEM; 1656 goto errout; 1657 } 1658 } 1659 } else { 1660 ubsecstats.hst_badflags++; 1661 err = EINVAL; 1662 goto errout; 1663 } 1664 1665 #ifdef UBSEC_DEBUG 1666 if (ubsec_debug) 1667 printf("dst skip: %d\n", dskip); 1668 #endif 1669 for (i = j = 0; i < q->q_dst_map->dm_nsegs; i++) { 1670 struct ubsec_pktbuf *pb; 1671 bus_size_t packl = q->q_dst_map->dm_segs[i].ds_len; 1672 bus_addr_t packp = q->q_dst_map->dm_segs[i].ds_addr; 1673 1674 if (dskip >= packl) { 1675 dskip -= packl; 1676 continue; 1677 } 1678 1679 packl -= dskip; 1680 packp += dskip; 1681 dskip = 0; 1682 1683 if (packl > 0xfffc) { 1684 err = EIO; 1685 goto errout; 1686 } 1687 1688 if (j == 0) 1689 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1690 else 1691 pb = &dmap->d_dma->d_dbuf[j - 1]; 1692 1693 pb->pb_addr = htole32(packp); 1694 1695 if (dtheend) { 1696 if (packl > dtheend) { 1697 pb->pb_len = htole32(dtheend); 1698 dtheend = 0; 1699 } else { 1700 pb->pb_len = htole32(packl); 1701 dtheend -= packl; 1702 } 1703 } else 1704 pb->pb_len = htole32(packl); 1705 1706 if ((i + 1) == q->q_dst_map->dm_nsegs) { 1707 if (maccrd) 1708 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1709 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1710 else 1711 pb->pb_next = 0; 1712 } else 1713 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1714 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1715 j++; 1716 } 1717 } 1718 1719 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1720 offsetof(struct ubsec_dmachunk, d_ctx)); 1721 1722 if (enccrd && enccrd->crd_alg == CRYPTO_AES_CBC) { 1723 struct ubsec_pktctx_aes128 *aes128; 1724 struct ubsec_pktctx_aes192 *aes192; 1725 struct ubsec_pktctx_aes256 *aes256; 1726 struct ubsec_pktctx_hdr *ph; 1727 u_int8_t *ctx; 1728 1729 ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1730 offsetof(struct ubsec_dmachunk, d_ctx); 1731 1732 ph = (struct ubsec_pktctx_hdr *)ctx; 1733 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_AES); 1734 ph->ph_flags = flags; 1735 ph->ph_offset = htole16(coffset >> 2); 1736 1737 switch (enccrd->crd_klen) { 1738 case 128: 1739 aes128 = (struct ubsec_pktctx_aes128 *)ctx; 1740 ph->ph_len = htole16(sizeof(*aes128)); 1741 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_128); 1742 for (i = 0; i < 4; i++) 1743 aes128->pc_aeskey[i] = key.ses_key[i]; 1744 for (i = 0; i < 5; i++) 1745 aes128->pc_hminner[i] = key.ses_hminner[i]; 1746 for (i = 0; i < 5; i++) 1747 aes128->pc_hmouter[i] = key.ses_hmouter[i]; 1748 for (i = 0; i < 4; i++) 1749 aes128->pc_iv[i] = key.ses_iv[i]; 1750 break; 1751 case 192: 1752 aes192 = (struct ubsec_pktctx_aes192 *)ctx; 1753 ph->ph_len = htole16(sizeof(*aes192)); 1754 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_192); 1755 for (i = 0; i < 6; i++) 1756 aes192->pc_aeskey[i] = key.ses_key[i]; 1757 for (i = 0; i < 5; i++) 1758 aes192->pc_hminner[i] = key.ses_hminner[i]; 1759 for (i = 0; i < 5; i++) 1760 aes192->pc_hmouter[i] = key.ses_hmouter[i]; 1761 for (i = 0; i < 4; i++) 1762 aes192->pc_iv[i] = key.ses_iv[i]; 1763 break; 1764 case 256: 1765 aes256 = (struct ubsec_pktctx_aes256 *)ctx; 1766 ph->ph_len = htole16(sizeof(*aes256)); 1767 ph->ph_flags |= htole16(UBS_PKTCTX_KEYSIZE_256); 1768 for (i = 0; i < 8; i++) 1769 aes256->pc_aeskey[i] = key.ses_key[i]; 1770 for (i = 0; i < 5; i++) 1771 aes256->pc_hminner[i] = key.ses_hminner[i]; 1772 for (i = 0; i < 5; i++) 1773 aes256->pc_hmouter[i] = key.ses_hmouter[i]; 1774 for (i = 0; i < 4; i++) 1775 aes256->pc_iv[i] = key.ses_iv[i]; 1776 break; 1777 } 1778 } else if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1779 struct ubsec_pktctx_3des *ctx; 1780 struct ubsec_pktctx_hdr *ph; 1781 1782 ctx = (struct ubsec_pktctx_3des *) 1783 ((u_int8_t *)(dmap->d_alloc.dma_vaddr) + 1784 offsetof(struct ubsec_dmachunk, d_ctx)); 1785 1786 ph = (struct ubsec_pktctx_hdr *)ctx; 1787 ph->ph_len = htole16(sizeof(*ctx)); 1788 ph->ph_type = htole16(UBS_PKTCTX_TYPE_IPSEC_3DES); 1789 ph->ph_flags = flags; 1790 ph->ph_offset = htole16(coffset >> 2); 1791 1792 for (i = 0; i < 6; i++) 1793 ctx->pc_deskey[i] = key.ses_key[i]; 1794 for (i = 0; i < 5; i++) 1795 ctx->pc_hminner[i] = key.ses_hminner[i]; 1796 for (i = 0; i < 5; i++) 1797 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1798 for (i = 0; i < 2; i++) 1799 ctx->pc_iv[i] = key.ses_iv[i]; 1800 } else { 1801 struct ubsec_pktctx *ctx = (struct ubsec_pktctx *) 1802 ((u_int8_t *)dmap->d_alloc.dma_vaddr + 1803 offsetof(struct ubsec_dmachunk, d_ctx)); 1804 1805 ctx->pc_flags = flags; 1806 ctx->pc_offset = htole16(coffset >> 2); 1807 for (i = 0; i < 6; i++) 1808 ctx->pc_deskey[i] = key.ses_key[i]; 1809 for (i = 0; i < 5; i++) 1810 ctx->pc_hminner[i] = key.ses_hminner[i]; 1811 for (i = 0; i < 5; i++) 1812 ctx->pc_hmouter[i] = key.ses_hmouter[i]; 1813 for (i = 0; i < 2; i++) 1814 ctx->pc_iv[i] = key.ses_iv[i]; 1815 } 1816 1817 mutex_spin_enter(&sc->sc_mtx); 1818 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1819 sc->sc_nqueue++; 1820 ubsecstats.hst_ipackets++; 1821 ubsecstats.hst_ibytes += dmap->d_alloc.dma_map->dm_mapsize; 1822 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= ubsec_maxbatch) 1823 ubsec_feed(sc); 1824 mutex_spin_exit(&sc->sc_mtx); 1825 return (0); 1826 1827 errout: 1828 if (q != NULL) { 1829 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1830 m_freem(q->q_dst_m); 1831 1832 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1833 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1834 } 1835 if (q->q_src_map != NULL) { 1836 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1837 } 1838 1839 mutex_spin_enter(&sc->sc_mtx); 1840 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1841 mutex_spin_exit(&sc->sc_mtx); 1842 } 1843 #if 0 /* jonathan says: this openbsd code seems to be subsumed elsewhere */ 1844 if (err == EINVAL) 1845 ubsecstats.hst_invalid++; 1846 else 1847 ubsecstats.hst_nomem++; 1848 #endif 1849 if (err != ERESTART) { 1850 crp->crp_etype = err; 1851 crypto_done(crp); 1852 } else { 1853 sc->sc_needwakeup |= CRYPTO_SYMQ; 1854 } 1855 return (err); 1856 } 1857 1858 static void 1859 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1860 { 1861 struct cryptop *crp = (struct cryptop *)q->q_crp; 1862 struct cryptodesc *crd; 1863 struct ubsec_dma *dmap = q->q_dma; 1864 1865 ubsecstats.hst_opackets++; 1866 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1867 1868 bus_dmamap_sync(sc->sc_dmat, dmap->d_alloc.dma_map, 0, 1869 dmap->d_alloc.dma_map->dm_mapsize, 1870 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1871 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1872 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1873 0, q->q_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 1874 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1875 } 1876 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, 1877 0, q->q_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 1878 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1879 1880 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1881 m_freem(q->q_src_m); 1882 crp->crp_buf = (void *)q->q_dst_m; 1883 } 1884 1885 /* copy out IV for future use */ 1886 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1887 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1888 if (crd->crd_alg != CRYPTO_DES_CBC && 1889 crd->crd_alg != CRYPTO_3DES_CBC && 1890 crd->crd_alg != CRYPTO_AES_CBC) 1891 continue; 1892 if (crp->crp_flags & CRYPTO_F_IMBUF) 1893 m_copydata((struct mbuf *)crp->crp_buf, 1894 crd->crd_skip + crd->crd_len - 8, 8, 1895 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1896 else if (crp->crp_flags & CRYPTO_F_IOV) { 1897 cuio_copydata((struct uio *)crp->crp_buf, 1898 crd->crd_skip + crd->crd_len - 8, 8, 1899 (void *)sc->sc_sessions[q->q_sesn].ses_iv); 1900 } 1901 break; 1902 } 1903 } 1904 1905 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1906 if (crd->crd_alg != CRYPTO_MD5_HMAC_96 && 1907 crd->crd_alg != CRYPTO_SHA1_HMAC_96) 1908 continue; 1909 if (crp->crp_flags & CRYPTO_F_IMBUF) 1910 m_copyback((struct mbuf *)crp->crp_buf, 1911 crd->crd_inject, 12, 1912 (void *)dmap->d_dma->d_macbuf); 1913 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1914 bcopy((void *)dmap->d_dma->d_macbuf, 1915 crp->crp_mac, 12); 1916 break; 1917 } 1918 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1919 crypto_done(crp); 1920 } 1921 1922 static void 1923 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1924 { 1925 int i, j, dlen, slen; 1926 char *dptr, *sptr; 1927 1928 j = 0; 1929 sptr = srcm->m_data; 1930 slen = srcm->m_len; 1931 dptr = dstm->m_data; 1932 dlen = dstm->m_len; 1933 1934 while (1) { 1935 for (i = 0; i < uimin(slen, dlen); i++) { 1936 if (j < hoffset || j >= toffset) 1937 *dptr++ = *sptr++; 1938 slen--; 1939 dlen--; 1940 j++; 1941 } 1942 if (slen == 0) { 1943 srcm = srcm->m_next; 1944 if (srcm == NULL) 1945 return; 1946 sptr = srcm->m_data; 1947 slen = srcm->m_len; 1948 } 1949 if (dlen == 0) { 1950 dstm = dstm->m_next; 1951 if (dstm == NULL) 1952 return; 1953 dptr = dstm->m_data; 1954 dlen = dstm->m_len; 1955 } 1956 } 1957 } 1958 1959 /* 1960 * feed the key generator, must be called at splnet() or higher. 1961 */ 1962 static void 1963 ubsec_feed2(struct ubsec_softc *sc) 1964 { 1965 struct ubsec_q2 *q; 1966 1967 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1968 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1969 break; 1970 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1971 1972 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 1973 q->q_mcr.dma_map->dm_mapsize, 1974 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1975 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 1976 q->q_ctx.dma_map->dm_mapsize, 1977 BUS_DMASYNC_PREWRITE); 1978 1979 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1980 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1981 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, /*q,*/ q_next); 1982 --sc->sc_nqueue2; 1983 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1984 } 1985 } 1986 1987 /* 1988 * feed the RNG (used instead of ubsec_feed2() on 5827+ devices) 1989 */ 1990 void 1991 ubsec_feed4(struct ubsec_softc *sc) 1992 { 1993 struct ubsec_q2 *q; 1994 1995 while (!SIMPLEQ_EMPTY(&sc->sc_queue4)) { 1996 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR4_FULL) 1997 break; 1998 q = SIMPLEQ_FIRST(&sc->sc_queue4); 1999 2000 bus_dmamap_sync(sc->sc_dmat, q->q_mcr.dma_map, 0, 2001 q->q_mcr.dma_map->dm_mapsize, 2002 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2003 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 2004 q->q_ctx.dma_map->dm_mapsize, 2005 BUS_DMASYNC_PREWRITE); 2006 2007 WRITE_REG(sc, BS_MCR4, q->q_mcr.dma_paddr); 2008 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue4, q_next); 2009 --sc->sc_nqueue4; 2010 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip4, q, q_next); 2011 } 2012 } 2013 2014 /* 2015 * Callback for handling random numbers 2016 */ 2017 static void 2018 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 2019 { 2020 struct cryptkop *krp; 2021 struct ubsec_ctx_keyop *ctx; 2022 2023 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 2024 bus_dmamap_sync(sc->sc_dmat, q->q_ctx.dma_map, 0, 2025 q->q_ctx.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2026 2027 switch (q->q_type) { 2028 #ifndef UBSEC_NO_RNG 2029 case UBS_CTXOP_RNGSHA1: 2030 case UBS_CTXOP_RNGBYPASS: { 2031 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 2032 u_int32_t *p; 2033 int i; 2034 2035 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2036 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2037 p = (u_int32_t *)rng->rng_buf.dma_vaddr; 2038 #ifndef __NetBSD__ 2039 for (i = 0; i < UBSEC_RNG_BUFSIZ; p++, i++) 2040 add_true_randomness(letoh32(*p)); 2041 #else 2042 i = UBSEC_RNG_BUFSIZ * sizeof(u_int32_t); 2043 rnd_add_data(&sc->sc_rnd_source, (char *)p, i, i * NBBY); 2044 sc->sc_rng_need -= i; 2045 #endif 2046 rng->rng_used = 0; 2047 #ifdef __OpenBSD__ 2048 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2049 #else 2050 if (sc->sc_rng_need > 0) { 2051 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2052 } 2053 #endif 2054 break; 2055 } 2056 #endif 2057 case UBS_CTXOP_MODEXP: { 2058 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2059 u_int rlen, clen; 2060 2061 krp = me->me_krp; 2062 rlen = (me->me_modbits + 7) / 8; 2063 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 2064 2065 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2066 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2067 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2068 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2069 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2070 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2071 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2072 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2073 2074 if (clen < rlen) 2075 krp->krp_status = E2BIG; 2076 else { 2077 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 2078 memset(krp->krp_param[krp->krp_iparams].crp_p, 0, 2079 (krp->krp_param[krp->krp_iparams].crp_nbits 2080 + 7) / 8); 2081 bcopy(me->me_C.dma_vaddr, 2082 krp->krp_param[krp->krp_iparams].crp_p, 2083 (me->me_modbits + 7) / 8); 2084 } else 2085 ubsec_kshift_l(me->me_shiftbits, 2086 me->me_C.dma_vaddr, me->me_normbits, 2087 krp->krp_param[krp->krp_iparams].crp_p, 2088 krp->krp_param[krp->krp_iparams].crp_nbits); 2089 } 2090 2091 crypto_kdone(krp); 2092 2093 /* bzero all potentially sensitive data */ 2094 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2095 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2096 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2097 memset(me->me_q.q_ctx.dma_vaddr, 0, me->me_q.q_ctx.dma_size); 2098 2099 /* Can't free here, so put us on the free list. */ 2100 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 2101 break; 2102 } 2103 case UBS_CTXOP_RSAPRIV: { 2104 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2105 u_int len; 2106 2107 krp = rp->rpr_krp; 2108 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 0, 2109 rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2110 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 0, 2111 rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2112 2113 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) 2114 / 8; 2115 bcopy(rp->rpr_msgout.dma_vaddr, 2116 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 2117 2118 crypto_kdone(krp); 2119 2120 memset(rp->rpr_msgin.dma_vaddr, 0, rp->rpr_msgin.dma_size); 2121 memset(rp->rpr_msgout.dma_vaddr, 0, rp->rpr_msgout.dma_size); 2122 memset(rp->rpr_q.q_ctx.dma_vaddr, 0, rp->rpr_q.q_ctx.dma_size); 2123 2124 /* Can't free here, so put us on the free list. */ 2125 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 2126 break; 2127 } 2128 default: 2129 printf("%s: unknown ctx op: %x\n", device_xname(sc->sc_dev), 2130 letoh16(ctx->ctx_op)); 2131 break; 2132 } 2133 } 2134 2135 #ifndef UBSEC_NO_RNG 2136 2137 static void 2138 ubsec_rng_get(size_t bytes, void *vsc) 2139 { 2140 struct ubsec_softc *sc = vsc; 2141 2142 mutex_spin_enter(&sc->sc_mtx); 2143 sc->sc_rng_need = bytes; 2144 ubsec_rng_locked(sc); 2145 mutex_spin_exit(&sc->sc_mtx); 2146 2147 } 2148 2149 static void 2150 ubsec_rng(void *vsc) 2151 { 2152 struct ubsec_softc *sc = vsc; 2153 mutex_spin_enter(&sc->sc_mtx); 2154 ubsec_rng_locked(sc); 2155 mutex_spin_exit(&sc->sc_mtx); 2156 } 2157 2158 static void 2159 ubsec_rng_locked(void *vsc) 2160 { 2161 struct ubsec_softc *sc = vsc; 2162 struct ubsec_q2_rng *rng = &sc->sc_rng; 2163 struct ubsec_mcr *mcr; 2164 struct ubsec_ctx_rngbypass *ctx; 2165 int *nqueue; 2166 2167 /* Caller is responsible to lock and release sc_mtx. */ 2168 KASSERT(mutex_owned(&sc->sc_mtx)); 2169 2170 if (rng->rng_used) { 2171 return; 2172 } 2173 2174 if (sc->sc_rng_need < 1) { 2175 callout_stop(&sc->sc_rngto); 2176 return; 2177 } 2178 2179 if (sc->sc_flags & UBS_FLAGS_RNG4) 2180 nqueue = &sc->sc_nqueue4; 2181 else 2182 nqueue = &sc->sc_nqueue2; 2183 2184 (*nqueue)++; 2185 if (*nqueue >= UBS_MAX_NQUEUE) 2186 goto out; 2187 2188 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 2189 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 2190 2191 mcr->mcr_pkts = htole16(1); 2192 mcr->mcr_flags = 0; 2193 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 2194 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 2195 mcr->mcr_ipktbuf.pb_len = 0; 2196 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 2197 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 2198 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 2199 UBS_PKTBUF_LEN); 2200 mcr->mcr_opktbuf.pb_next = 0; 2201 2202 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 2203 ctx->rbp_op = htole16(UBS_CTXOP_RNGSHA1); 2204 rng->rng_q.q_type = UBS_CTXOP_RNGSHA1; 2205 2206 bus_dmamap_sync(sc->sc_dmat, rng->rng_buf.dma_map, 0, 2207 rng->rng_buf.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2208 2209 if (sc->sc_flags & UBS_FLAGS_RNG4) { 2210 SIMPLEQ_INSERT_TAIL(&sc->sc_queue4, &rng->rng_q, q_next); 2211 ubsec_feed4(sc); 2212 } else { 2213 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 2214 ubsec_feed2(sc); 2215 } 2216 rng->rng_used = 1; 2217 ubsecstats.hst_rng++; 2218 2219 return; 2220 2221 out: 2222 /* 2223 * Something weird happened, generate our own call back. 2224 */ 2225 (*nqueue)--; 2226 #ifdef __OpenBSD__ 2227 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 2228 #else 2229 callout_schedule(&sc->sc_rngto, sc->sc_rnghz); 2230 #endif 2231 } 2232 #endif /* UBSEC_NO_RNG */ 2233 2234 static int 2235 ubsec_dma_malloc(struct ubsec_softc *sc, bus_size_t size, 2236 struct ubsec_dma_alloc *dma,int mapflags) 2237 { 2238 int r; 2239 2240 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, 2241 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0) 2242 goto fail_0; 2243 2244 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, 2245 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0) 2246 goto fail_1; 2247 2248 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 2249 BUS_DMA_NOWAIT, &dma->dma_map)) != 0) 2250 goto fail_2; 2251 2252 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, 2253 size, NULL, BUS_DMA_NOWAIT)) != 0) 2254 goto fail_3; 2255 2256 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2257 dma->dma_size = size; 2258 return (0); 2259 2260 fail_3: 2261 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2262 fail_2: 2263 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size); 2264 fail_1: 2265 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2266 fail_0: 2267 dma->dma_map = NULL; 2268 return (r); 2269 } 2270 2271 static void 2272 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 2273 { 2274 bus_dmamap_unload(sc->sc_dmat, dma->dma_map); 2275 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size); 2276 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg); 2277 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map); 2278 } 2279 2280 /* 2281 * Resets the board. Values in the regesters are left as is 2282 * from the reset (i.e. initial values are assigned elsewhere). 2283 */ 2284 static void 2285 ubsec_reset_board(struct ubsec_softc *sc) 2286 { 2287 volatile u_int32_t ctrl; 2288 2289 ctrl = READ_REG(sc, BS_CTRL); 2290 ctrl |= BS_CTRL_RESET; 2291 WRITE_REG(sc, BS_CTRL, ctrl); 2292 2293 /* 2294 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 2295 */ 2296 DELAY(10); 2297 2298 /* Enable RNG and interrupts on newer devices */ 2299 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2300 #ifndef UBSEC_NO_RNG 2301 WRITE_REG(sc, BS_CFG, BS_CFG_RNG); 2302 #endif 2303 WRITE_REG(sc, BS_INT, BS_INT_DMAINT); 2304 } 2305 } 2306 2307 /* 2308 * Init Broadcom registers 2309 */ 2310 static void 2311 ubsec_init_board(struct ubsec_softc *sc) 2312 { 2313 u_int32_t ctrl; 2314 2315 ctrl = READ_REG(sc, BS_CTRL); 2316 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 2317 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 2318 2319 /* 2320 * XXX: Sam Leffler's code has (UBS_FLAGS_KEY|UBS_FLAGS_RNG)). 2321 * anyone got hw docs? 2322 */ 2323 if (sc->sc_flags & UBS_FLAGS_KEY) 2324 ctrl |= BS_CTRL_MCR2INT; 2325 else 2326 ctrl &= ~BS_CTRL_MCR2INT; 2327 2328 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2329 ctrl &= ~BS_CTRL_SWNORM; 2330 2331 if (sc->sc_flags & UBS_FLAGS_MULTIMCR) { 2332 ctrl |= BS_CTRL_BSIZE240; 2333 ctrl &= ~BS_CTRL_MCR3INT; /* MCR3 is reserved for SSL */ 2334 2335 if (sc->sc_flags & UBS_FLAGS_RNG4) 2336 ctrl |= BS_CTRL_MCR4INT; 2337 else 2338 ctrl &= ~BS_CTRL_MCR4INT; 2339 } 2340 2341 WRITE_REG(sc, BS_CTRL, ctrl); 2342 } 2343 2344 /* 2345 * Init Broadcom PCI registers 2346 */ 2347 static void 2348 ubsec_init_pciregs(struct pci_attach_args *pa) 2349 { 2350 pci_chipset_tag_t pc = pa->pa_pc; 2351 u_int32_t misc; 2352 2353 /* 2354 * This will set the cache line size to 1, this will 2355 * force the BCM58xx chip just to do burst read/writes. 2356 * Cache line read/writes are to slow 2357 */ 2358 misc = pci_conf_read(pc, pa->pa_tag, PCI_BHLC_REG); 2359 misc = (misc & ~(PCI_CACHELINE_MASK << PCI_CACHELINE_SHIFT)) 2360 | ((UBS_DEF_CACHELINE & 0xff) << PCI_CACHELINE_SHIFT); 2361 pci_conf_write(pc, pa->pa_tag, PCI_BHLC_REG, misc); 2362 } 2363 2364 /* 2365 * Clean up after a chip crash. 2366 * It is assumed that the caller in splnet() 2367 */ 2368 static void 2369 ubsec_cleanchip(struct ubsec_softc *sc) 2370 { 2371 struct ubsec_q *q; 2372 2373 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 2374 q = SIMPLEQ_FIRST(&sc->sc_qchip); 2375 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, /*q,*/ q_next); 2376 ubsec_free_q(sc, q); 2377 } 2378 sc->sc_nqchip = 0; 2379 } 2380 2381 /* 2382 * free a ubsec_q 2383 * It is assumed that the caller is within splnet() 2384 */ 2385 static int 2386 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 2387 { 2388 struct ubsec_q *q2; 2389 struct cryptop *crp; 2390 int npkts; 2391 int i; 2392 2393 npkts = q->q_nstacked_mcrs; 2394 2395 for (i = 0; i < npkts; i++) { 2396 if(q->q_stacked_mcr[i]) { 2397 q2 = q->q_stacked_mcr[i]; 2398 2399 if ((q2->q_dst_m != NULL) 2400 && (q2->q_src_m != q2->q_dst_m)) 2401 m_freem(q2->q_dst_m); 2402 2403 crp = (struct cryptop *)q2->q_crp; 2404 2405 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2406 2407 crp->crp_etype = EFAULT; 2408 crypto_done(crp); 2409 } else { 2410 break; 2411 } 2412 } 2413 2414 /* 2415 * Free header MCR 2416 */ 2417 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2418 m_freem(q->q_dst_m); 2419 2420 crp = (struct cryptop *)q->q_crp; 2421 2422 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2423 2424 crp->crp_etype = EFAULT; 2425 crypto_done(crp); 2426 return(0); 2427 } 2428 2429 /* 2430 * Routine to reset the chip and clean up. 2431 * It is assumed that the caller is in splnet() 2432 */ 2433 static void 2434 ubsec_totalreset(struct ubsec_softc *sc) 2435 { 2436 ubsec_reset_board(sc); 2437 ubsec_init_board(sc); 2438 ubsec_cleanchip(sc); 2439 } 2440 2441 static int 2442 ubsec_dmamap_aligned(bus_dmamap_t map) 2443 { 2444 int i; 2445 2446 for (i = 0; i < map->dm_nsegs; i++) { 2447 if (map->dm_segs[i].ds_addr & 3) 2448 return (0); 2449 if ((i != (map->dm_nsegs - 1)) && 2450 (map->dm_segs[i].ds_len & 3)) 2451 return (0); 2452 } 2453 return (1); 2454 } 2455 2456 #ifdef __OpenBSD__ 2457 struct ubsec_softc * 2458 ubsec_kfind(struct cryptkop *krp) 2459 { 2460 struct ubsec_softc *sc; 2461 int i; 2462 2463 for (i = 0; i < ubsec_cd.cd_ndevs; i++) { 2464 sc = ubsec_cd.cd_devs[i]; 2465 if (sc == NULL) 2466 continue; 2467 if (sc->sc_cid == krp->krp_hid) 2468 return (sc); 2469 } 2470 return (NULL); 2471 } 2472 #endif 2473 2474 static void 2475 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2476 { 2477 switch (q->q_type) { 2478 case UBS_CTXOP_MODEXP: { 2479 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2480 2481 ubsec_dma_free(sc, &me->me_q.q_mcr); 2482 ubsec_dma_free(sc, &me->me_q.q_ctx); 2483 ubsec_dma_free(sc, &me->me_M); 2484 ubsec_dma_free(sc, &me->me_E); 2485 ubsec_dma_free(sc, &me->me_C); 2486 ubsec_dma_free(sc, &me->me_epb); 2487 free(me, M_DEVBUF); 2488 break; 2489 } 2490 case UBS_CTXOP_RSAPRIV: { 2491 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2492 2493 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2494 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2495 ubsec_dma_free(sc, &rp->rpr_msgin); 2496 ubsec_dma_free(sc, &rp->rpr_msgout); 2497 free(rp, M_DEVBUF); 2498 break; 2499 } 2500 default: 2501 printf("%s: invalid kfree 0x%x\n", device_xname(sc->sc_dev), 2502 q->q_type); 2503 break; 2504 } 2505 } 2506 2507 static int 2508 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2509 { 2510 struct ubsec_softc *sc; 2511 int r; 2512 2513 if (krp == NULL || krp->krp_callback == NULL) 2514 return (EINVAL); 2515 #ifdef __OpenBSD__ 2516 if ((sc = ubsec_kfind(krp)) == NULL) 2517 return (EINVAL); 2518 #else 2519 sc = arg; 2520 KASSERT(sc != NULL /*, ("ubsec_kprocess: null softc")*/); 2521 #endif 2522 2523 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2524 struct ubsec_q2 *q; 2525 2526 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2527 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, /*q,*/ q_next); 2528 ubsec_kfree(sc, q); 2529 } 2530 2531 switch (krp->krp_op) { 2532 case CRK_MOD_EXP: 2533 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2534 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2535 else 2536 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2537 break; 2538 case CRK_MOD_EXP_CRT: 2539 r = ubsec_kprocess_rsapriv(sc, krp, hint); 2540 break; 2541 default: 2542 printf("%s: kprocess: invalid op 0x%x\n", 2543 device_xname(sc->sc_dev), krp->krp_op); 2544 krp->krp_status = EOPNOTSUPP; 2545 crypto_kdone(krp); 2546 r = 0; 2547 } 2548 return (r); 2549 } 2550 2551 /* 2552 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2553 */ 2554 static int 2555 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, 2556 int hint) 2557 { 2558 struct ubsec_q2_modexp *me; 2559 struct ubsec_mcr *mcr; 2560 struct ubsec_ctx_modexp *ctx; 2561 struct ubsec_pktbuf *epb; 2562 int err = 0; 2563 u_int nbits, normbits, mbits, shiftbits, ebits; 2564 2565 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2566 if (me == NULL) { 2567 err = ENOMEM; 2568 goto errout; 2569 } 2570 memset(me, 0, sizeof *me); 2571 me->me_krp = krp; 2572 me->me_q.q_type = UBS_CTXOP_MODEXP; 2573 2574 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2575 if (nbits <= 512) 2576 normbits = 512; 2577 else if (nbits <= 768) 2578 normbits = 768; 2579 else if (nbits <= 1024) 2580 normbits = 1024; 2581 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2582 normbits = 1536; 2583 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2584 normbits = 2048; 2585 else { 2586 err = E2BIG; 2587 goto errout; 2588 } 2589 2590 shiftbits = normbits - nbits; 2591 2592 me->me_modbits = nbits; 2593 me->me_shiftbits = shiftbits; 2594 me->me_normbits = normbits; 2595 2596 /* Sanity check: result bits must be >= true modulus bits. */ 2597 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2598 err = ERANGE; 2599 goto errout; 2600 } 2601 2602 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2603 &me->me_q.q_mcr, 0)) { 2604 err = ENOMEM; 2605 goto errout; 2606 } 2607 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2608 2609 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2610 &me->me_q.q_ctx, 0)) { 2611 err = ENOMEM; 2612 goto errout; 2613 } 2614 2615 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2616 if (mbits > nbits) { 2617 err = E2BIG; 2618 goto errout; 2619 } 2620 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2621 err = ENOMEM; 2622 goto errout; 2623 } 2624 ubsec_kshift_r(shiftbits, 2625 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2626 me->me_M.dma_vaddr, normbits); 2627 2628 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2629 err = ENOMEM; 2630 goto errout; 2631 } 2632 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2633 2634 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2635 if (ebits > nbits) { 2636 err = E2BIG; 2637 goto errout; 2638 } 2639 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2640 err = ENOMEM; 2641 goto errout; 2642 } 2643 ubsec_kshift_r(shiftbits, 2644 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2645 me->me_E.dma_vaddr, normbits); 2646 2647 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2648 &me->me_epb, 0)) { 2649 err = ENOMEM; 2650 goto errout; 2651 } 2652 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2653 epb->pb_addr = htole32(me->me_E.dma_paddr); 2654 epb->pb_next = 0; 2655 epb->pb_len = htole32(normbits / 8); 2656 2657 #ifdef UBSEC_DEBUG 2658 if (ubsec_debug) { 2659 printf("Epb "); 2660 ubsec_dump_pb(epb); 2661 } 2662 #endif 2663 2664 mcr->mcr_pkts = htole16(1); 2665 mcr->mcr_flags = 0; 2666 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2667 mcr->mcr_reserved = 0; 2668 mcr->mcr_pktlen = 0; 2669 2670 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2671 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2672 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2673 2674 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2675 mcr->mcr_opktbuf.pb_next = 0; 2676 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2677 2678 #ifdef DIAGNOSTIC 2679 /* Misaligned output buffer will hang the chip. */ 2680 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2681 panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev), 2682 letoh32(mcr->mcr_opktbuf.pb_addr)); 2683 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2684 panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev), 2685 letoh32(mcr->mcr_opktbuf.pb_len)); 2686 #endif 2687 2688 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2689 memset(ctx, 0, sizeof(*ctx)); 2690 ubsec_kshift_r(shiftbits, 2691 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2692 ctx->me_N, normbits); 2693 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2694 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2695 ctx->me_E_len = htole16(nbits); 2696 ctx->me_N_len = htole16(nbits); 2697 2698 #ifdef UBSEC_DEBUG 2699 if (ubsec_debug) { 2700 ubsec_dump_mcr(mcr); 2701 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2702 } 2703 #endif 2704 2705 /* 2706 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2707 * everything else. 2708 */ 2709 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2710 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2711 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2712 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2713 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2714 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2715 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2716 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2717 2718 /* Enqueue and we're done... */ 2719 mutex_spin_enter(&sc->sc_mtx); 2720 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2721 ubsec_feed2(sc); 2722 ubsecstats.hst_modexp++; 2723 mutex_spin_exit(&sc->sc_mtx); 2724 2725 return (0); 2726 2727 errout: 2728 if (me != NULL) { 2729 if (me->me_q.q_mcr.dma_map != NULL) 2730 ubsec_dma_free(sc, &me->me_q.q_mcr); 2731 if (me->me_q.q_ctx.dma_map != NULL) { 2732 memset(me->me_q.q_ctx.dma_vaddr, 0, 2733 me->me_q.q_ctx.dma_size); 2734 ubsec_dma_free(sc, &me->me_q.q_ctx); 2735 } 2736 if (me->me_M.dma_map != NULL) { 2737 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2738 ubsec_dma_free(sc, &me->me_M); 2739 } 2740 if (me->me_E.dma_map != NULL) { 2741 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2742 ubsec_dma_free(sc, &me->me_E); 2743 } 2744 if (me->me_C.dma_map != NULL) { 2745 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2746 ubsec_dma_free(sc, &me->me_C); 2747 } 2748 if (me->me_epb.dma_map != NULL) 2749 ubsec_dma_free(sc, &me->me_epb); 2750 free(me, M_DEVBUF); 2751 } 2752 krp->krp_status = err; 2753 crypto_kdone(krp); 2754 return (0); 2755 } 2756 2757 /* 2758 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2759 */ 2760 static int 2761 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, 2762 int hint) 2763 { 2764 struct ubsec_q2_modexp *me; 2765 struct ubsec_mcr *mcr; 2766 struct ubsec_ctx_modexp *ctx; 2767 struct ubsec_pktbuf *epb; 2768 int err = 0; 2769 u_int nbits, normbits, mbits, shiftbits, ebits; 2770 2771 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2772 if (me == NULL) { 2773 err = ENOMEM; 2774 goto errout; 2775 } 2776 memset(me, 0, sizeof *me); 2777 me->me_krp = krp; 2778 me->me_q.q_type = UBS_CTXOP_MODEXP; 2779 2780 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2781 if (nbits <= 512) 2782 normbits = 512; 2783 else if (nbits <= 768) 2784 normbits = 768; 2785 else if (nbits <= 1024) 2786 normbits = 1024; 2787 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2788 normbits = 1536; 2789 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2790 normbits = 2048; 2791 else { 2792 err = E2BIG; 2793 goto errout; 2794 } 2795 2796 shiftbits = normbits - nbits; 2797 2798 /* XXX ??? */ 2799 me->me_modbits = nbits; 2800 me->me_shiftbits = shiftbits; 2801 me->me_normbits = normbits; 2802 2803 /* Sanity check: result bits must be >= true modulus bits. */ 2804 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2805 err = ERANGE; 2806 goto errout; 2807 } 2808 2809 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2810 &me->me_q.q_mcr, 0)) { 2811 err = ENOMEM; 2812 goto errout; 2813 } 2814 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2815 2816 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2817 &me->me_q.q_ctx, 0)) { 2818 err = ENOMEM; 2819 goto errout; 2820 } 2821 2822 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2823 if (mbits > nbits) { 2824 err = E2BIG; 2825 goto errout; 2826 } 2827 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2828 err = ENOMEM; 2829 goto errout; 2830 } 2831 memset(me->me_M.dma_vaddr, 0, normbits / 8); 2832 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2833 me->me_M.dma_vaddr, (mbits + 7) / 8); 2834 2835 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2836 err = ENOMEM; 2837 goto errout; 2838 } 2839 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2840 2841 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2842 if (ebits > nbits) { 2843 err = E2BIG; 2844 goto errout; 2845 } 2846 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2847 err = ENOMEM; 2848 goto errout; 2849 } 2850 memset(me->me_E.dma_vaddr, 0, normbits / 8); 2851 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2852 me->me_E.dma_vaddr, (ebits + 7) / 8); 2853 2854 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2855 &me->me_epb, 0)) { 2856 err = ENOMEM; 2857 goto errout; 2858 } 2859 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2860 epb->pb_addr = htole32(me->me_E.dma_paddr); 2861 epb->pb_next = 0; 2862 epb->pb_len = htole32((ebits + 7) / 8); 2863 2864 #ifdef UBSEC_DEBUG 2865 if (ubsec_debug) { 2866 printf("Epb "); 2867 ubsec_dump_pb(epb); 2868 } 2869 #endif 2870 2871 mcr->mcr_pkts = htole16(1); 2872 mcr->mcr_flags = 0; 2873 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2874 mcr->mcr_reserved = 0; 2875 mcr->mcr_pktlen = 0; 2876 2877 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2878 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2879 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2880 2881 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2882 mcr->mcr_opktbuf.pb_next = 0; 2883 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2884 2885 #ifdef DIAGNOSTIC 2886 /* Misaligned output buffer will hang the chip. */ 2887 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2888 panic("%s: modexp invalid addr 0x%x", device_xname(sc->sc_dev), 2889 letoh32(mcr->mcr_opktbuf.pb_addr)); 2890 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2891 panic("%s: modexp invalid len 0x%x", device_xname(sc->sc_dev), 2892 letoh32(mcr->mcr_opktbuf.pb_len)); 2893 #endif 2894 2895 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2896 memset(ctx, 0, sizeof(*ctx)); 2897 memcpy(ctx->me_N, krp->krp_param[UBS_MODEXP_PAR_N].crp_p, 2898 (nbits + 7) / 8); 2899 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2900 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2901 ctx->me_E_len = htole16(ebits); 2902 ctx->me_N_len = htole16(nbits); 2903 2904 #ifdef UBSEC_DEBUG 2905 if (ubsec_debug) { 2906 ubsec_dump_mcr(mcr); 2907 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2908 } 2909 #endif 2910 2911 /* 2912 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2913 * everything else. 2914 */ 2915 bus_dmamap_sync(sc->sc_dmat, me->me_M.dma_map, 2916 0, me->me_M.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2917 bus_dmamap_sync(sc->sc_dmat, me->me_E.dma_map, 2918 0, me->me_E.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2919 bus_dmamap_sync(sc->sc_dmat, me->me_C.dma_map, 2920 0, me->me_C.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2921 bus_dmamap_sync(sc->sc_dmat, me->me_epb.dma_map, 2922 0, me->me_epb.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2923 2924 /* Enqueue and we're done... */ 2925 mutex_spin_enter(&sc->sc_mtx); 2926 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2927 ubsec_feed2(sc); 2928 mutex_spin_exit(&sc->sc_mtx); 2929 2930 return (0); 2931 2932 errout: 2933 if (me != NULL) { 2934 if (me->me_q.q_mcr.dma_map != NULL) 2935 ubsec_dma_free(sc, &me->me_q.q_mcr); 2936 if (me->me_q.q_ctx.dma_map != NULL) { 2937 memset(me->me_q.q_ctx.dma_vaddr, 0, 2938 me->me_q.q_ctx.dma_size); 2939 ubsec_dma_free(sc, &me->me_q.q_ctx); 2940 } 2941 if (me->me_M.dma_map != NULL) { 2942 memset(me->me_M.dma_vaddr, 0, me->me_M.dma_size); 2943 ubsec_dma_free(sc, &me->me_M); 2944 } 2945 if (me->me_E.dma_map != NULL) { 2946 memset(me->me_E.dma_vaddr, 0, me->me_E.dma_size); 2947 ubsec_dma_free(sc, &me->me_E); 2948 } 2949 if (me->me_C.dma_map != NULL) { 2950 memset(me->me_C.dma_vaddr, 0, me->me_C.dma_size); 2951 ubsec_dma_free(sc, &me->me_C); 2952 } 2953 if (me->me_epb.dma_map != NULL) 2954 ubsec_dma_free(sc, &me->me_epb); 2955 free(me, M_DEVBUF); 2956 } 2957 krp->krp_status = err; 2958 crypto_kdone(krp); 2959 return (0); 2960 } 2961 2962 static int 2963 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, 2964 int hint) 2965 { 2966 struct ubsec_q2_rsapriv *rp = NULL; 2967 struct ubsec_mcr *mcr; 2968 struct ubsec_ctx_rsapriv *ctx; 2969 int err = 0; 2970 u_int padlen, msglen; 2971 2972 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2973 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2974 if (msglen > padlen) 2975 padlen = msglen; 2976 2977 if (padlen <= 256) 2978 padlen = 256; 2979 else if (padlen <= 384) 2980 padlen = 384; 2981 else if (padlen <= 512) 2982 padlen = 512; 2983 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2984 padlen = 768; 2985 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2986 padlen = 1024; 2987 else { 2988 err = E2BIG; 2989 goto errout; 2990 } 2991 2992 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2993 err = E2BIG; 2994 goto errout; 2995 } 2996 2997 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2998 err = E2BIG; 2999 goto errout; 3000 } 3001 3002 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 3003 err = E2BIG; 3004 goto errout; 3005 } 3006 3007 rp = malloc(sizeof *rp, M_DEVBUF, M_NOWAIT|M_ZERO); 3008 if (rp == NULL) 3009 return (ENOMEM); 3010 rp->rpr_krp = krp; 3011 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 3012 3013 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 3014 &rp->rpr_q.q_mcr, 0)) { 3015 err = ENOMEM; 3016 goto errout; 3017 } 3018 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 3019 3020 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 3021 &rp->rpr_q.q_ctx, 0)) { 3022 err = ENOMEM; 3023 goto errout; 3024 } 3025 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 3026 memset(ctx, 0, sizeof *ctx); 3027 3028 /* Copy in p */ 3029 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 3030 &ctx->rpr_buf[0 * (padlen / 8)], 3031 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 3032 3033 /* Copy in q */ 3034 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 3035 &ctx->rpr_buf[1 * (padlen / 8)], 3036 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 3037 3038 /* Copy in dp */ 3039 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 3040 &ctx->rpr_buf[2 * (padlen / 8)], 3041 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 3042 3043 /* Copy in dq */ 3044 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 3045 &ctx->rpr_buf[3 * (padlen / 8)], 3046 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 3047 3048 /* Copy in pinv */ 3049 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 3050 &ctx->rpr_buf[4 * (padlen / 8)], 3051 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 3052 3053 msglen = padlen * 2; 3054 3055 /* Copy in input message (aligned buffer/length). */ 3056 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 3057 /* Is this likely? */ 3058 err = E2BIG; 3059 goto errout; 3060 } 3061 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 3062 err = ENOMEM; 3063 goto errout; 3064 } 3065 memset(rp->rpr_msgin.dma_vaddr, 0, (msglen + 7) / 8); 3066 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 3067 rp->rpr_msgin.dma_vaddr, 3068 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 3069 3070 /* Prepare space for output message (aligned buffer/length). */ 3071 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 3072 /* Is this likely? */ 3073 err = E2BIG; 3074 goto errout; 3075 } 3076 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 3077 err = ENOMEM; 3078 goto errout; 3079 } 3080 memset(rp->rpr_msgout.dma_vaddr, 0, (msglen + 7) / 8); 3081 3082 mcr->mcr_pkts = htole16(1); 3083 mcr->mcr_flags = 0; 3084 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 3085 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 3086 mcr->mcr_ipktbuf.pb_next = 0; 3087 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 3088 mcr->mcr_reserved = 0; 3089 mcr->mcr_pktlen = htole16(msglen); 3090 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 3091 mcr->mcr_opktbuf.pb_next = 0; 3092 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 3093 3094 #ifdef DIAGNOSTIC 3095 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 3096 panic("%s: rsapriv: invalid msgin 0x%lx(0x%lx)", 3097 device_xname(sc->sc_dev), (u_long) rp->rpr_msgin.dma_paddr, 3098 (u_long) rp->rpr_msgin.dma_size); 3099 } 3100 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 3101 panic("%s: rsapriv: invalid msgout 0x%lx(0x%lx)", 3102 device_xname(sc->sc_dev), (u_long) rp->rpr_msgout.dma_paddr, 3103 (u_long) rp->rpr_msgout.dma_size); 3104 } 3105 #endif 3106 3107 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 3108 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 3109 ctx->rpr_q_len = htole16(padlen); 3110 ctx->rpr_p_len = htole16(padlen); 3111 3112 /* 3113 * ubsec_feed2 will sync mcr and ctx, we just need to sync 3114 * everything else. 3115 */ 3116 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgin.dma_map, 3117 0, rp->rpr_msgin.dma_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 3118 bus_dmamap_sync(sc->sc_dmat, rp->rpr_msgout.dma_map, 3119 0, rp->rpr_msgout.dma_map->dm_mapsize, BUS_DMASYNC_PREREAD); 3120 3121 /* Enqueue and we're done... */ 3122 mutex_spin_enter(&sc->sc_mtx); 3123 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 3124 ubsec_feed2(sc); 3125 ubsecstats.hst_modexpcrt++; 3126 mutex_spin_exit(&sc->sc_mtx); 3127 return (0); 3128 3129 errout: 3130 if (rp != NULL) { 3131 if (rp->rpr_q.q_mcr.dma_map != NULL) 3132 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 3133 if (rp->rpr_msgin.dma_map != NULL) { 3134 memset(rp->rpr_msgin.dma_vaddr, 0, 3135 rp->rpr_msgin.dma_size); 3136 ubsec_dma_free(sc, &rp->rpr_msgin); 3137 } 3138 if (rp->rpr_msgout.dma_map != NULL) { 3139 memset(rp->rpr_msgout.dma_vaddr, 0, 3140 rp->rpr_msgout.dma_size); 3141 ubsec_dma_free(sc, &rp->rpr_msgout); 3142 } 3143 free(rp, M_DEVBUF); 3144 } 3145 krp->krp_status = err; 3146 crypto_kdone(krp); 3147 return (0); 3148 } 3149 3150 #ifdef UBSEC_DEBUG 3151 static void 3152 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 3153 { 3154 printf("addr 0x%x (0x%x) next 0x%x\n", 3155 pb->pb_addr, pb->pb_len, pb->pb_next); 3156 } 3157 3158 static void 3159 ubsec_dump_ctx2(volatile struct ubsec_ctx_keyop *c) 3160 { 3161 printf("CTX (0x%x):\n", c->ctx_len); 3162 switch (letoh16(c->ctx_op)) { 3163 case UBS_CTXOP_RNGBYPASS: 3164 case UBS_CTXOP_RNGSHA1: 3165 break; 3166 case UBS_CTXOP_MODEXP: 3167 { 3168 struct ubsec_ctx_modexp *cx = (void *)c; 3169 int i, len; 3170 3171 printf(" Elen %u, Nlen %u\n", 3172 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 3173 len = (cx->me_N_len + 7)/8; 3174 for (i = 0; i < len; i++) 3175 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 3176 printf("\n"); 3177 break; 3178 } 3179 default: 3180 printf("unknown context: %x\n", c->ctx_op); 3181 } 3182 printf("END CTX\n"); 3183 } 3184 3185 static void 3186 ubsec_dump_mcr(struct ubsec_mcr *mcr) 3187 { 3188 volatile struct ubsec_mcr_add *ma; 3189 int i; 3190 3191 printf("MCR:\n"); 3192 printf(" pkts: %u, flags 0x%x\n", 3193 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 3194 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 3195 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 3196 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 3197 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 3198 letoh16(ma->mcr_reserved)); 3199 printf(" %d: ipkt ", i); 3200 ubsec_dump_pb(&ma->mcr_ipktbuf); 3201 printf(" %d: opkt ", i); 3202 ubsec_dump_pb(&ma->mcr_opktbuf); 3203 ma++; 3204 } 3205 printf("END MCR\n"); 3206 } 3207 #endif /* UBSEC_DEBUG */ 3208 3209 /* 3210 * Return the number of significant bits of a big number. 3211 */ 3212 static int 3213 ubsec_ksigbits(struct crparam *cr) 3214 { 3215 u_int plen = (cr->crp_nbits + 7) / 8; 3216 int i, sig = plen * 8; 3217 u_int8_t c, *p = cr->crp_p; 3218 3219 for (i = plen - 1; i >= 0; i--) { 3220 c = p[i]; 3221 if (c != 0) { 3222 while ((c & 0x80) == 0) { 3223 sig--; 3224 c <<= 1; 3225 } 3226 break; 3227 } 3228 sig -= 8; 3229 } 3230 return (sig); 3231 } 3232 3233 static void 3234 ubsec_kshift_r(u_int shiftbits, u_int8_t *src, u_int srcbits, 3235 u_int8_t *dst, u_int dstbits) 3236 { 3237 u_int slen, dlen; 3238 int i, si, di, n; 3239 3240 slen = (srcbits + 7) / 8; 3241 dlen = (dstbits + 7) / 8; 3242 3243 for (i = 0; i < slen; i++) 3244 dst[i] = src[i]; 3245 for (i = 0; i < dlen - slen; i++) 3246 dst[slen + i] = 0; 3247 3248 n = shiftbits / 8; 3249 if (n != 0) { 3250 si = dlen - n - 1; 3251 di = dlen - 1; 3252 while (si >= 0) 3253 dst[di--] = dst[si--]; 3254 while (di >= 0) 3255 dst[di--] = 0; 3256 } 3257 3258 n = shiftbits % 8; 3259 if (n != 0) { 3260 for (i = dlen - 1; i > 0; i--) 3261 dst[i] = (dst[i] << n) | 3262 (dst[i - 1] >> (8 - n)); 3263 dst[0] = dst[0] << n; 3264 } 3265 } 3266 3267 static void 3268 ubsec_kshift_l(u_int shiftbits, u_int8_t *src, u_int srcbits, 3269 u_int8_t *dst, u_int dstbits) 3270 { 3271 int slen, dlen, i, n; 3272 3273 slen = (srcbits + 7) / 8; 3274 dlen = (dstbits + 7) / 8; 3275 3276 n = shiftbits / 8; 3277 for (i = 0; i < slen; i++) 3278 dst[i] = src[i + n]; 3279 for (i = 0; i < dlen - slen; i++) 3280 dst[slen + i] = 0; 3281 3282 n = shiftbits % 8; 3283 if (n != 0) { 3284 for (i = 0; i < (dlen - 1); i++) 3285 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 3286 dst[dlen - 1] = dst[dlen - 1] >> n; 3287 } 3288 } 3289
Indexes created Sun Sep 21 20:09:37 GMT 2025