hifn7751.c revision 1.63
1 /* $NetBSD: hifn7751.c,v 1.63 2018/12/27 14:03:54 maxv Exp $ */ 2 /* $FreeBSD: hifn7751.c,v 1.5.2.7 2003/10/08 23:52:00 sam Exp $ */ 3 /* $OpenBSD: hifn7751.c,v 1.140 2003/08/01 17:55:54 deraadt Exp $ */ 4 5 /* 6 * Invertex AEON / Hifn 7751 driver 7 * Copyright (c) 1999 Invertex Inc. All rights reserved. 8 * Copyright (c) 1999 Theo de Raadt 9 * Copyright (c) 2000-2001 Network Security Technologies, Inc. 10 * http://www.netsec.net 11 * Copyright (c) 2003 Hifn Inc. 12 * 13 * This driver is based on a previous driver by Invertex, for which they 14 * requested: Please send any comments, feedback, bug-fixes, or feature 15 * requests to software (at) invertex.com. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * 1. Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * 2. Redistributions in binary form must reproduce the above copyright 24 * notice, this list of conditions and the following disclaimer in the 25 * documentation and/or other materials provided with the distribution. 26 * 3. The name of the author may not be used to endorse or promote products 27 * derived from this software without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 30 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 31 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 32 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 33 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 34 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 38 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Effort sponsored in part by the Defense Advanced Research Projects 41 * Agency (DARPA) and Air Force Research Laboratory, Air Force 42 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 43 * 44 */ 45 46 /* 47 * Driver for various Hifn pre-HIPP encryption processors. 48 */ 49 50 #include <sys/cdefs.h> 51 __KERNEL_RCSID(0, "$NetBSD: hifn7751.c,v 1.63 2018/12/27 14:03:54 maxv Exp $"); 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/mutex.h> 56 #include <sys/proc.h> 57 #include <sys/errno.h> 58 #include <sys/malloc.h> 59 #include <sys/kernel.h> 60 #include <sys/mbuf.h> 61 #include <sys/device.h> 62 #include <sys/module.h> 63 64 #ifdef __OpenBSD__ 65 #include <crypto/crypto.h> 66 #include <dev/rndvar.h> 67 #else 68 #include <opencrypto/cryptodev.h> 69 #include <sys/cprng.h> 70 #include <sys/rndpool.h> 71 #include <sys/rndsource.h> 72 #include <sys/sha1.h> 73 #endif 74 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 #include <dev/pci/pcidevs.h> 78 79 #include <dev/pci/hifn7751reg.h> 80 #include <dev/pci/hifn7751var.h> 81 82 #undef HIFN_DEBUG 83 84 #ifdef __NetBSD__ 85 #define M_DUP_PKTHDR m_copy_pkthdr /* XXX */ 86 #endif 87 88 #ifdef HIFN_DEBUG 89 extern int hifn_debug; /* patchable */ 90 int hifn_debug = 1; 91 #endif 92 93 #ifdef __OpenBSD__ 94 #define HAVE_CRYPTO_LZS /* OpenBSD OCF supports CRYPTO_COMP_LZS */ 95 #endif 96 97 /* 98 * Prototypes and count for the pci_device structure 99 */ 100 #ifdef __OpenBSD__ 101 static int hifn_probe((struct device *, void *, void *); 102 #else 103 static int hifn_probe(device_t, cfdata_t, void *); 104 #endif 105 static void hifn_attach(device_t, device_t, void *); 106 #ifdef __NetBSD__ 107 static int hifn_detach(device_t, int); 108 109 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc), 110 hifn_probe, hifn_attach, hifn_detach, NULL); 111 #else 112 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc), 113 hifn_probe, hifn_attach, NULL, NULL); 114 #endif 115 116 #ifdef __OpenBSD__ 117 struct cfdriver hifn_cd = { 118 0, "hifn", DV_DULL 119 }; 120 #endif 121 122 static void hifn_reset_board(struct hifn_softc *, int); 123 static void hifn_reset_puc(struct hifn_softc *); 124 static void hifn_puc_wait(struct hifn_softc *); 125 static const char *hifn_enable_crypto(struct hifn_softc *, pcireg_t); 126 static void hifn_set_retry(struct hifn_softc *); 127 static void hifn_init_dma(struct hifn_softc *); 128 static void hifn_init_pci_registers(struct hifn_softc *); 129 static int hifn_sramsize(struct hifn_softc *); 130 static int hifn_dramsize(struct hifn_softc *); 131 static int hifn_ramtype(struct hifn_softc *); 132 static void hifn_sessions(struct hifn_softc *); 133 static int hifn_intr(void *); 134 static u_int hifn_write_command(struct hifn_command *, u_int8_t *); 135 static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt); 136 static int hifn_newsession(void*, u_int32_t *, struct cryptoini *); 137 static int hifn_freesession(void*, u_int64_t); 138 static int hifn_process(void*, struct cryptop *, int); 139 static void hifn_callback(struct hifn_softc *, struct hifn_command *, 140 u_int8_t *); 141 static int hifn_crypto(struct hifn_softc *, struct hifn_command *, 142 struct cryptop*, int); 143 static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *); 144 static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *); 145 static int hifn_dmamap_aligned(bus_dmamap_t); 146 static int hifn_dmamap_load_src(struct hifn_softc *, 147 struct hifn_command *); 148 static int hifn_dmamap_load_dst(struct hifn_softc *, 149 struct hifn_command *); 150 static int hifn_init_pubrng(struct hifn_softc *); 151 static void hifn_rng(void *); 152 static void hifn_rng_locked(void *); 153 static void hifn_tick(void *); 154 static void hifn_abort(struct hifn_softc *); 155 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, 156 int *); 157 static void hifn_write_4(struct hifn_softc *, int, bus_size_t, u_int32_t); 158 static u_int32_t hifn_read_4(struct hifn_softc *, int, bus_size_t); 159 #ifdef HAVE_CRYPTO_LZS 160 static int hifn_compression(struct hifn_softc *, struct cryptop *, 161 struct hifn_command *); 162 static struct mbuf *hifn_mkmbuf_chain(int, struct mbuf *); 163 static int hifn_compress_enter(struct hifn_softc *, struct hifn_command *); 164 static void hifn_callback_comp(struct hifn_softc *, struct hifn_command *, 165 u_int8_t *); 166 #endif /* HAVE_CRYPTO_LZS */ 167 168 struct hifn_stats hifnstats; 169 170 static const struct hifn_product { 171 pci_vendor_id_t hifn_vendor; 172 pci_product_id_t hifn_product; 173 int hifn_flags; 174 const char *hifn_name; 175 } hifn_products[] = { 176 { PCI_VENDOR_INVERTEX, PCI_PRODUCT_INVERTEX_AEON, 177 0, 178 "Invertex AEON", 179 }, 180 181 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7751, 182 0, 183 "Hifn 7751", 184 }, 185 { PCI_VENDOR_NETSEC, PCI_PRODUCT_NETSEC_7751, 186 0, 187 "Hifn 7751 (NetSec)" 188 }, 189 190 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7811, 191 HIFN_IS_7811 | HIFN_HAS_RNG | HIFN_HAS_LEDS | HIFN_NO_BURSTWRITE, 192 "Hifn 7811", 193 }, 194 195 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7951, 196 HIFN_HAS_RNG | HIFN_HAS_PUBLIC, 197 "Hifn 7951", 198 }, 199 200 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7955, 201 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 202 "Hifn 7955", 203 }, 204 205 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7956, 206 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 207 "Hifn 7956", 208 }, 209 210 211 { 0, 0, 212 0, 213 NULL 214 } 215 }; 216 217 static const struct hifn_product * 218 hifn_lookup(const struct pci_attach_args *pa) 219 { 220 const struct hifn_product *hp; 221 222 for (hp = hifn_products; hp->hifn_name != NULL; hp++) { 223 if (PCI_VENDOR(pa->pa_id) == hp->hifn_vendor && 224 PCI_PRODUCT(pa->pa_id) == hp->hifn_product) 225 return (hp); 226 } 227 return (NULL); 228 } 229 230 static int 231 hifn_probe(device_t parent, cfdata_t match, void *aux) 232 { 233 struct pci_attach_args *pa = aux; 234 235 if (hifn_lookup(pa) != NULL) 236 return 1; 237 238 return 0; 239 } 240 241 static void 242 hifn_attach(device_t parent, device_t self, void *aux) 243 { 244 struct hifn_softc *sc = device_private(self); 245 struct pci_attach_args *pa = aux; 246 const struct hifn_product *hp; 247 pci_chipset_tag_t pc = pa->pa_pc; 248 pci_intr_handle_t ih; 249 const char *intrstr = NULL; 250 const char *hifncap; 251 char rbase; 252 #ifdef __NetBSD__ 253 #define iosize0 sc->sc_iosz0 254 #define iosize1 sc->sc_iosz1 255 #else 256 bus_size_t iosize0, iosize1; 257 #endif 258 u_int32_t cmd; 259 u_int16_t ena; 260 bus_dma_segment_t seg; 261 bus_dmamap_t dmamap; 262 int rseg; 263 void *kva; 264 char intrbuf[PCI_INTRSTR_LEN]; 265 266 hp = hifn_lookup(pa); 267 if (hp == NULL) { 268 printf("\n"); 269 panic("hifn_attach: impossible"); 270 } 271 272 pci_aprint_devinfo_fancy(pa, "Crypto processor", hp->hifn_name, 1); 273 274 sc->sc_dv = self; 275 sc->sc_pci_pc = pa->pa_pc; 276 sc->sc_pci_tag = pa->pa_tag; 277 278 sc->sc_flags = hp->hifn_flags; 279 280 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 281 cmd |= PCI_COMMAND_MASTER_ENABLE; 282 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 283 284 if (pci_mapreg_map(pa, HIFN_BAR0, PCI_MAPREG_TYPE_MEM, 0, 285 &sc->sc_st0, &sc->sc_sh0, NULL, &iosize0)) { 286 aprint_error_dev(sc->sc_dv, "can't map mem space %d\n", 0); 287 return; 288 } 289 290 if (pci_mapreg_map(pa, HIFN_BAR1, PCI_MAPREG_TYPE_MEM, 0, 291 &sc->sc_st1, &sc->sc_sh1, NULL, &iosize1)) { 292 aprint_error_dev(sc->sc_dv, "can't find mem space %d\n", 1); 293 goto fail_io0; 294 } 295 296 hifn_set_retry(sc); 297 298 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 299 sc->sc_waw_lastgroup = -1; 300 sc->sc_waw_lastreg = 1; 301 } 302 303 sc->sc_dmat = pa->pa_dmat; 304 if (bus_dmamem_alloc(sc->sc_dmat, sizeof(*sc->sc_dma), PAGE_SIZE, 0, 305 &seg, 1, &rseg, BUS_DMA_NOWAIT)) { 306 aprint_error_dev(sc->sc_dv, "can't alloc DMA buffer\n"); 307 goto fail_io1; 308 } 309 if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, sizeof(*sc->sc_dma), &kva, 310 BUS_DMA_NOWAIT)) { 311 aprint_error_dev(sc->sc_dv, "can't map DMA buffers (%lu bytes)\n", 312 (u_long)sizeof(*sc->sc_dma)); 313 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 314 goto fail_io1; 315 } 316 if (bus_dmamap_create(sc->sc_dmat, sizeof(*sc->sc_dma), 1, 317 sizeof(*sc->sc_dma), 0, BUS_DMA_NOWAIT, &dmamap)) { 318 aprint_error_dev(sc->sc_dv, "can't create DMA map\n"); 319 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 320 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 321 goto fail_io1; 322 } 323 if (bus_dmamap_load(sc->sc_dmat, dmamap, kva, sizeof(*sc->sc_dma), 324 NULL, BUS_DMA_NOWAIT)) { 325 aprint_error_dev(sc->sc_dv, "can't load DMA map\n"); 326 bus_dmamap_destroy(sc->sc_dmat, dmamap); 327 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 328 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 329 goto fail_io1; 330 } 331 sc->sc_dmamap = dmamap; 332 sc->sc_dma = (struct hifn_dma *)kva; 333 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 334 335 hifn_reset_board(sc, 0); 336 337 if ((hifncap = hifn_enable_crypto(sc, pa->pa_id)) == NULL) { 338 aprint_error_dev(sc->sc_dv, "crypto enabling failed\n"); 339 goto fail_mem; 340 } 341 hifn_reset_puc(sc); 342 343 hifn_init_dma(sc); 344 hifn_init_pci_registers(sc); 345 346 /* XXX can't dynamically determine ram type for 795x; force dram */ 347 if (sc->sc_flags & HIFN_IS_7956) 348 sc->sc_drammodel = 1; 349 else if (hifn_ramtype(sc)) 350 goto fail_mem; 351 352 if (sc->sc_drammodel == 0) 353 hifn_sramsize(sc); 354 else 355 hifn_dramsize(sc); 356 357 /* 358 * Workaround for NetSec 7751 rev A: half ram size because two 359 * of the address lines were left floating 360 */ 361 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETSEC && 362 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETSEC_7751 && 363 PCI_REVISION(pa->pa_class) == 0x61) 364 sc->sc_ramsize >>= 1; 365 366 if (pci_intr_map(pa, &ih)) { 367 aprint_error_dev(sc->sc_dv, "couldn't map interrupt\n"); 368 goto fail_mem; 369 } 370 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 371 #ifdef __OpenBSD__ 372 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc, 373 device_xname(self)); 374 #else 375 sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, hifn_intr, sc, 376 device_xname(self)); 377 #endif 378 if (sc->sc_ih == NULL) { 379 aprint_error_dev(sc->sc_dv, "couldn't establish interrupt\n"); 380 if (intrstr != NULL) 381 aprint_error(" at %s", intrstr); 382 aprint_error("\n"); 383 goto fail_mem; 384 } 385 386 hifn_sessions(sc); 387 388 rseg = sc->sc_ramsize / 1024; 389 rbase = 'K'; 390 if (sc->sc_ramsize >= (1024 * 1024)) { 391 rbase = 'M'; 392 rseg /= 1024; 393 } 394 aprint_normal_dev(sc->sc_dv, "%s, %d%cB %cRAM, interrupting at %s\n", 395 hifncap, rseg, rbase, 396 sc->sc_drammodel ? 'D' : 'S', intrstr); 397 398 sc->sc_cid = crypto_get_driverid(0); 399 if (sc->sc_cid < 0) { 400 aprint_error_dev(sc->sc_dv, "couldn't get crypto driver id\n"); 401 goto fail_intr; 402 } 403 404 WRITE_REG_0(sc, HIFN_0_PUCNFG, 405 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID); 406 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 407 408 switch (ena) { 409 case HIFN_PUSTAT_ENA_2: 410 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 411 hifn_newsession, hifn_freesession, hifn_process, sc); 412 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0, 413 hifn_newsession, hifn_freesession, hifn_process, sc); 414 if (sc->sc_flags & HIFN_HAS_AES) 415 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 416 hifn_newsession, hifn_freesession, 417 hifn_process, sc); 418 /*FALLTHROUGH*/ 419 case HIFN_PUSTAT_ENA_1: 420 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0, 421 hifn_newsession, hifn_freesession, hifn_process, sc); 422 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0, 423 hifn_newsession, hifn_freesession, hifn_process, sc); 424 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 425 hifn_newsession, hifn_freesession, hifn_process, sc); 426 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 427 hifn_newsession, hifn_freesession, hifn_process, sc); 428 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 429 hifn_newsession, hifn_freesession, hifn_process, sc); 430 break; 431 } 432 433 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 0, 434 sc->sc_dmamap->dm_mapsize, 435 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 436 437 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 438 439 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) { 440 hifn_init_pubrng(sc); 441 sc->sc_rng_need = RND_POOLBITS / NBBY; 442 } 443 444 #ifdef __OpenBSD__ 445 timeout_set(&sc->sc_tickto, hifn_tick, sc); 446 timeout_add(&sc->sc_tickto, hz); 447 #else 448 callout_init(&sc->sc_tickto, CALLOUT_MPSAFE); 449 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 450 #endif 451 return; 452 453 fail_intr: 454 pci_intr_disestablish(pc, sc->sc_ih); 455 fail_mem: 456 bus_dmamap_unload(sc->sc_dmat, dmamap); 457 bus_dmamap_destroy(sc->sc_dmat, dmamap); 458 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 459 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 460 461 /* Turn off DMA polling */ 462 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 463 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 464 465 fail_io1: 466 bus_space_unmap(sc->sc_st1, sc->sc_sh1, iosize1); 467 fail_io0: 468 bus_space_unmap(sc->sc_st0, sc->sc_sh0, iosize0); 469 } 470 471 #ifdef __NetBSD__ 472 static int 473 hifn_detach(device_t self, int flags) 474 { 475 struct hifn_softc *sc = device_private(self); 476 477 hifn_abort(sc); 478 479 hifn_reset_board(sc, 1); 480 481 pci_intr_disestablish(sc->sc_pci_pc, sc->sc_ih); 482 483 crypto_unregister_all(sc->sc_cid); 484 485 rnd_detach_source(&sc->sc_rnd_source); 486 487 mutex_enter(&sc->sc_mtx); 488 callout_halt(&sc->sc_tickto, NULL); 489 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) 490 callout_halt(&sc->sc_rngto, NULL); 491 mutex_exit(&sc->sc_mtx); 492 493 bus_space_unmap(sc->sc_st1, sc->sc_sh1, sc->sc_iosz1); 494 bus_space_unmap(sc->sc_st0, sc->sc_sh0, sc->sc_iosz0); 495 496 /* 497 * XXX It's not clear if any additional buffers have been 498 * XXX allocated and require free()ing 499 */ 500 501 return 0; 502 } 503 504 MODULE(MODULE_CLASS_DRIVER, hifn, "pci,opencrypto"); 505 506 #ifdef _MODULE 507 #include "ioconf.c" 508 #endif 509 510 static int 511 hifn_modcmd(modcmd_t cmd, void *data) 512 { 513 int error = 0; 514 515 switch(cmd) { 516 case MODULE_CMD_INIT: 517 #ifdef _MODULE 518 error = config_init_component(cfdriver_ioconf_hifn, 519 cfattach_ioconf_hifn, cfdata_ioconf_hifn); 520 #endif 521 return error; 522 case MODULE_CMD_FINI: 523 #ifdef _MODULE 524 error = config_fini_component(cfdriver_ioconf_hifn, 525 cfattach_ioconf_hifn, cfdata_ioconf_hifn); 526 #endif 527 return error; 528 default: 529 return ENOTTY; 530 } 531 } 532 533 #endif /* ifdef __NetBSD__ */ 534 535 static void 536 hifn_rng_get(size_t bytes, void *priv) 537 { 538 struct hifn_softc *sc = priv; 539 540 mutex_enter(&sc->sc_mtx); 541 sc->sc_rng_need = bytes; 542 callout_reset(&sc->sc_rngto, 0, hifn_rng, sc); 543 mutex_exit(&sc->sc_mtx); 544 } 545 546 static int 547 hifn_init_pubrng(struct hifn_softc *sc) 548 { 549 u_int32_t r; 550 int i; 551 552 if ((sc->sc_flags & HIFN_IS_7811) == 0) { 553 /* Reset 7951 public key/rng engine */ 554 WRITE_REG_1(sc, HIFN_1_PUB_RESET, 555 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET); 556 557 for (i = 0; i < 100; i++) { 558 DELAY(1000); 559 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) & 560 HIFN_PUBRST_RESET) == 0) 561 break; 562 } 563 564 if (i == 100) { 565 printf("%s: public key init failed\n", 566 device_xname(sc->sc_dv)); 567 return (1); 568 } 569 } 570 571 /* Enable the rng, if available */ 572 if (sc->sc_flags & HIFN_HAS_RNG) { 573 if (sc->sc_flags & HIFN_IS_7811) { 574 r = READ_REG_1(sc, HIFN_1_7811_RNGENA); 575 if (r & HIFN_7811_RNGENA_ENA) { 576 r &= ~HIFN_7811_RNGENA_ENA; 577 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 578 } 579 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG, 580 HIFN_7811_RNGCFG_DEFL); 581 r |= HIFN_7811_RNGENA_ENA; 582 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 583 } else 584 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG, 585 READ_REG_1(sc, HIFN_1_RNG_CONFIG) | 586 HIFN_RNGCFG_ENA); 587 588 /* 589 * The Hifn RNG documentation states that at their 590 * recommended "conservative" RNG config values, 591 * the RNG must warm up for 0.4s before providing 592 * data that meet their worst-case estimate of 0.06 593 * bits of random data per output register bit. 594 */ 595 DELAY(4000); 596 597 #ifdef __NetBSD__ 598 rndsource_setcb(&sc->sc_rnd_source, hifn_rng_get, sc); 599 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dv), 600 RND_TYPE_RNG, 601 RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB); 602 #endif 603 604 if (hz >= 100) 605 sc->sc_rnghz = hz / 100; 606 else 607 sc->sc_rnghz = 1; 608 #ifdef __OpenBSD__ 609 timeout_set(&sc->sc_rngto, hifn_rng, sc); 610 #else /* !__OpenBSD__ */ 611 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE); 612 #endif /* !__OpenBSD__ */ 613 } 614 615 /* Enable public key engine, if available */ 616 if (sc->sc_flags & HIFN_HAS_PUBLIC) { 617 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 618 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE; 619 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 620 } 621 622 /* Call directly into the RNG once to prime the pool. */ 623 hifn_rng(sc); /* Sets callout/timeout at end */ 624 625 return (0); 626 } 627 628 static void 629 hifn_rng_locked(void *vsc) 630 { 631 struct hifn_softc *sc = vsc; 632 #ifdef __NetBSD__ 633 uint32_t num[64]; 634 #else 635 uint32_t num[2]; 636 #endif 637 uint32_t sts; 638 int i; 639 size_t got, gotent; 640 641 if (sc->sc_rng_need < 1) { 642 callout_stop(&sc->sc_rngto); 643 return; 644 } 645 646 if (sc->sc_flags & HIFN_IS_7811) { 647 for (i = 0; i < 5; i++) { /* XXX why 5? */ 648 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS); 649 if (sts & HIFN_7811_RNGSTS_UFL) { 650 printf("%s: RNG underflow: disabling\n", 651 device_xname(sc->sc_dv)); 652 return; 653 } 654 if ((sts & HIFN_7811_RNGSTS_RDY) == 0) 655 break; 656 657 /* 658 * There are at least two words in the RNG FIFO 659 * at this point. 660 */ 661 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 662 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 663 got = 2 * sizeof(num[0]); 664 gotent = (got * NBBY) / HIFN_RNG_BITSPER; 665 666 #ifdef __NetBSD__ 667 rnd_add_data(&sc->sc_rnd_source, num, got, gotent); 668 sc->sc_rng_need -= gotent; 669 #else 670 /* 671 * XXX This is a really bad idea. 672 * XXX Hifn estimate as little as 0.06 673 * XXX actual bits of entropy per output 674 * XXX register bit. How can we tell the 675 * XXX kernel RNG subsystem we're handing 676 * XXX it 64 "true" random bits, for any 677 * XXX sane value of "true"? 678 * XXX 679 * XXX The right thing to do here, if we 680 * XXX cannot supply an estimate ourselves, 681 * XXX would be to hash the bits locally. 682 */ 683 add_true_randomness(num[0]); 684 add_true_randomness(num[1]); 685 #endif 686 687 } 688 } else { 689 int nwords = 0; 690 691 if (sc->sc_rng_need) { 692 nwords = (sc->sc_rng_need * NBBY) / HIFN_RNG_BITSPER; 693 nwords = MIN(__arraycount(num), nwords); 694 } 695 696 if (nwords < 2) { 697 nwords = 2; 698 } 699 700 /* 701 * We must be *extremely* careful here. The Hifn 702 * 795x differ from the published 6500 RNG design 703 * in more ways than the obvious lack of the output 704 * FIFO and LFSR control registers. In fact, there 705 * is only one LFSR, instead of the 6500's two, and 706 * it's 32 bits, not 31. 707 * 708 * Further, a block diagram obtained from Hifn shows 709 * a very curious latching of this register: the LFSR 710 * rotates at a frequency of RNG_Clk / 8, but the 711 * RNG_Data register is latched at a frequency of 712 * RNG_Clk, which means that it is possible for 713 * consecutive reads of the RNG_Data register to read 714 * identical state from the LFSR. The simplest 715 * workaround seems to be to read eight samples from 716 * the register for each one that we use. Since each 717 * read must require at least one PCI cycle, and 718 * RNG_Clk is at least PCI_Clk, this is safe. 719 */ 720 for(i = 0 ; i < nwords * 8; i++) 721 { 722 volatile u_int32_t regtmp; 723 regtmp = READ_REG_1(sc, HIFN_1_RNG_DATA); 724 num[i / 8] = regtmp; 725 } 726 727 got = nwords * sizeof(num[0]); 728 gotent = (got * NBBY) / HIFN_RNG_BITSPER; 729 #ifdef __NetBSD__ 730 rnd_add_data(&sc->sc_rnd_source, num, got, gotent); 731 sc->sc_rng_need -= gotent; 732 #else 733 /* XXX a bad idea; see 7811 block above */ 734 add_true_randomness(num[0]); 735 #endif 736 } 737 738 #ifdef __OpenBSD__ 739 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 740 #else 741 if (sc->sc_rng_need > 0) { 742 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc); 743 } 744 #endif 745 } 746 747 static void 748 hifn_rng(void *vsc) 749 { 750 struct hifn_softc *sc = vsc; 751 752 mutex_spin_enter(&sc->sc_mtx); 753 hifn_rng_locked(vsc); 754 mutex_spin_exit(&sc->sc_mtx); 755 } 756 757 static void 758 hifn_puc_wait(struct hifn_softc *sc) 759 { 760 int i; 761 762 for (i = 5000; i > 0; i--) { 763 DELAY(1); 764 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET)) 765 break; 766 } 767 if (!i) 768 printf("%s: proc unit did not reset\n", device_xname(sc->sc_dv)); 769 } 770 771 /* 772 * Reset the processing unit. 773 */ 774 static void 775 hifn_reset_puc(struct hifn_softc *sc) 776 { 777 /* Reset processing unit */ 778 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 779 hifn_puc_wait(sc); 780 } 781 782 static void 783 hifn_set_retry(struct hifn_softc *sc) 784 { 785 u_int32_t r; 786 787 r = pci_conf_read(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT); 788 r &= 0xffff0000; 789 pci_conf_write(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT, r); 790 } 791 792 /* 793 * Resets the board. Values in the regesters are left as is 794 * from the reset (i.e. initial values are assigned elsewhere). 795 */ 796 static void 797 hifn_reset_board(struct hifn_softc *sc, int full) 798 { 799 u_int32_t reg; 800 801 /* 802 * Set polling in the DMA configuration register to zero. 0x7 avoids 803 * resetting the board and zeros out the other fields. 804 */ 805 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 806 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 807 808 /* 809 * Now that polling has been disabled, we have to wait 1 ms 810 * before resetting the board. 811 */ 812 DELAY(1000); 813 814 /* Reset the DMA unit */ 815 if (full) { 816 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 817 DELAY(1000); 818 } else { 819 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, 820 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET); 821 hifn_reset_puc(sc); 822 } 823 824 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 825 826 /* Bring dma unit out of reset */ 827 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 828 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 829 830 hifn_puc_wait(sc); 831 832 hifn_set_retry(sc); 833 834 if (sc->sc_flags & HIFN_IS_7811) { 835 for (reg = 0; reg < 1000; reg++) { 836 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) & 837 HIFN_MIPSRST_CRAMINIT) 838 break; 839 DELAY(1000); 840 } 841 if (reg == 1000) 842 printf(": cram init timeout\n"); 843 } 844 } 845 846 static u_int32_t 847 hifn_next_signature(u_int32_t a, u_int cnt) 848 { 849 int i; 850 u_int32_t v; 851 852 for (i = 0; i < cnt; i++) { 853 854 /* get the parity */ 855 v = a & 0x80080125; 856 v ^= v >> 16; 857 v ^= v >> 8; 858 v ^= v >> 4; 859 v ^= v >> 2; 860 v ^= v >> 1; 861 862 a = (v & 1) ^ (a << 1); 863 } 864 865 return a; 866 } 867 868 static struct pci2id { 869 u_short pci_vendor; 870 u_short pci_prod; 871 char card_id[13]; 872 } const pci2id[] = { 873 { 874 PCI_VENDOR_HIFN, 875 PCI_PRODUCT_HIFN_7951, 876 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 877 0x00, 0x00, 0x00, 0x00, 0x00 } 878 }, { 879 PCI_VENDOR_HIFN, 880 PCI_PRODUCT_HIFN_7955, 881 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 882 0x00, 0x00, 0x00, 0x00, 0x00 } 883 }, { 884 PCI_VENDOR_HIFN, 885 PCI_PRODUCT_HIFN_7956, 886 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 887 0x00, 0x00, 0x00, 0x00, 0x00 } 888 }, { 889 PCI_VENDOR_NETSEC, 890 PCI_PRODUCT_NETSEC_7751, 891 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 892 0x00, 0x00, 0x00, 0x00, 0x00 } 893 }, { 894 PCI_VENDOR_INVERTEX, 895 PCI_PRODUCT_INVERTEX_AEON, 896 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 897 0x00, 0x00, 0x00, 0x00, 0x00 } 898 }, { 899 PCI_VENDOR_HIFN, 900 PCI_PRODUCT_HIFN_7811, 901 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 902 0x00, 0x00, 0x00, 0x00, 0x00 } 903 }, { 904 /* 905 * Other vendors share this PCI ID as well, such as 906 * http://www.powercrypt.com, and obviously they also 907 * use the same key. 908 */ 909 PCI_VENDOR_HIFN, 910 PCI_PRODUCT_HIFN_7751, 911 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 912 0x00, 0x00, 0x00, 0x00, 0x00 } 913 }, 914 }; 915 916 /* 917 * Checks to see if crypto is already enabled. If crypto isn't enable, 918 * "hifn_enable_crypto" is called to enable it. The check is important, 919 * as enabling crypto twice will lock the board. 920 */ 921 static const char * 922 hifn_enable_crypto(struct hifn_softc *sc, pcireg_t pciid) 923 { 924 u_int32_t dmacfg, ramcfg, encl, addr, i; 925 const char *offtbl = NULL; 926 927 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { 928 if (pci2id[i].pci_vendor == PCI_VENDOR(pciid) && 929 pci2id[i].pci_prod == PCI_PRODUCT(pciid)) { 930 offtbl = pci2id[i].card_id; 931 break; 932 } 933 } 934 935 if (offtbl == NULL) { 936 #ifdef HIFN_DEBUG 937 aprint_debug_dev(sc->sc_dv, "Unknown card!\n"); 938 #endif 939 return (NULL); 940 } 941 942 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG); 943 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG); 944 945 /* 946 * The RAM config register's encrypt level bit needs to be set before 947 * every read performed on the encryption level register. 948 */ 949 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 950 951 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 952 953 /* 954 * Make sure we don't re-unlock. Two unlocks kills chip until the 955 * next reboot. 956 */ 957 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) { 958 #ifdef HIFN_DEBUG 959 aprint_debug_dev(sc->sc_dv, "Strong Crypto already enabled!\n"); 960 #endif 961 goto report; 962 } 963 964 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) { 965 #ifdef HIFN_DEBUG 966 aprint_debug_dev(sc->sc_dv, "Unknown encryption level\n"); 967 #endif 968 return (NULL); 969 } 970 971 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK | 972 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 973 DELAY(1000); 974 addr = READ_REG_1(sc, HIFN_1_UNLOCK_SECRET1); 975 DELAY(1000); 976 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, 0); 977 DELAY(1000); 978 979 for (i = 0; i <= 12; i++) { 980 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 981 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, addr); 982 983 DELAY(1000); 984 } 985 986 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 987 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 988 989 #ifdef HIFN_DEBUG 990 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2) 991 aprint_debug("Encryption engine is permanently locked until next system reset."); 992 else 993 aprint_debug("Encryption engine enabled successfully!"); 994 #endif 995 996 report: 997 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg); 998 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg); 999 1000 switch (encl) { 1001 case HIFN_PUSTAT_ENA_0: 1002 return ("LZS-only (no encr/auth)"); 1003 1004 case HIFN_PUSTAT_ENA_1: 1005 return ("DES"); 1006 1007 case HIFN_PUSTAT_ENA_2: 1008 if (sc->sc_flags & HIFN_HAS_AES) 1009 return ("3DES/AES"); 1010 else 1011 return ("3DES"); 1012 1013 default: 1014 return ("disabled"); 1015 } 1016 /* NOTREACHED */ 1017 } 1018 1019 /* 1020 * Give initial values to the registers listed in the "Register Space" 1021 * section of the HIFN Software Development reference manual. 1022 */ 1023 static void 1024 hifn_init_pci_registers(struct hifn_softc *sc) 1025 { 1026 /* write fixed values needed by the Initialization registers */ 1027 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 1028 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 1029 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 1030 1031 /* write all 4 ring address registers */ 1032 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 1033 offsetof(struct hifn_dma, cmdr[0])); 1034 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 1035 offsetof(struct hifn_dma, srcr[0])); 1036 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 1037 offsetof(struct hifn_dma, dstr[0])); 1038 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 1039 offsetof(struct hifn_dma, resr[0])); 1040 1041 DELAY(2000); 1042 1043 /* write status register */ 1044 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1045 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 1046 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 1047 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 1048 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 1049 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 1050 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 1051 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 1052 HIFN_DMACSR_S_WAIT | 1053 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1054 HIFN_DMACSR_C_WAIT | 1055 HIFN_DMACSR_ENGINE | 1056 ((sc->sc_flags & HIFN_HAS_PUBLIC) ? 1057 HIFN_DMACSR_PUBDONE : 0) | 1058 ((sc->sc_flags & HIFN_IS_7811) ? 1059 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0)); 1060 1061 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0; 1062 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 1063 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 1064 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 1065 HIFN_DMAIER_ENGINE | 1066 ((sc->sc_flags & HIFN_IS_7811) ? 1067 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0); 1068 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 1069 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1070 CLR_LED(sc, HIFN_MIPSRST_LED0 | HIFN_MIPSRST_LED1 | HIFN_MIPSRST_LED2); 1071 1072 if (sc->sc_flags & HIFN_IS_7956) { 1073 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 1074 HIFN_PUCNFG_TCALLPHASES | 1075 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32); 1076 WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956); 1077 } else { 1078 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 1079 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 1080 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 1081 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM)); 1082 } 1083 1084 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1085 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 1086 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 1087 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 1088 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1089 } 1090 1091 /* 1092 * The maximum number of sessions supported by the card 1093 * is dependent on the amount of context ram, which 1094 * encryption algorithms are enabled, and how compression 1095 * is configured. This should be configured before this 1096 * routine is called. 1097 */ 1098 static void 1099 hifn_sessions(struct hifn_softc *sc) 1100 { 1101 u_int32_t pucnfg; 1102 int ctxsize; 1103 1104 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG); 1105 1106 if (pucnfg & HIFN_PUCNFG_COMPSING) { 1107 if (pucnfg & HIFN_PUCNFG_ENCCNFG) 1108 ctxsize = 128; 1109 else 1110 ctxsize = 512; 1111 /* 1112 * 7955/7956 has internal context memory of 32K 1113 */ 1114 if (sc->sc_flags & HIFN_IS_7956) 1115 sc->sc_maxses = 32768 / ctxsize; 1116 else 1117 sc->sc_maxses = 1 + 1118 ((sc->sc_ramsize - 32768) / ctxsize); 1119 } 1120 else 1121 sc->sc_maxses = sc->sc_ramsize / 16384; 1122 1123 if (sc->sc_maxses > 2048) 1124 sc->sc_maxses = 2048; 1125 } 1126 1127 /* 1128 * Determine ram type (sram or dram). Board should be just out of a reset 1129 * state when this is called. 1130 */ 1131 static int 1132 hifn_ramtype(struct hifn_softc *sc) 1133 { 1134 u_int8_t data[8], dataexpect[8]; 1135 int i; 1136 1137 for (i = 0; i < sizeof(data); i++) 1138 data[i] = dataexpect[i] = 0x55; 1139 if (hifn_writeramaddr(sc, 0, data)) 1140 return (-1); 1141 if (hifn_readramaddr(sc, 0, data)) 1142 return (-1); 1143 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1144 sc->sc_drammodel = 1; 1145 return (0); 1146 } 1147 1148 for (i = 0; i < sizeof(data); i++) 1149 data[i] = dataexpect[i] = 0xaa; 1150 if (hifn_writeramaddr(sc, 0, data)) 1151 return (-1); 1152 if (hifn_readramaddr(sc, 0, data)) 1153 return (-1); 1154 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1155 sc->sc_drammodel = 1; 1156 return (0); 1157 } 1158 1159 return (0); 1160 } 1161 1162 #define HIFN_SRAM_MAX (32 << 20) 1163 #define HIFN_SRAM_STEP_SIZE 16384 1164 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE) 1165 1166 static int 1167 hifn_sramsize(struct hifn_softc *sc) 1168 { 1169 u_int32_t a; 1170 u_int8_t data[8]; 1171 u_int8_t dataexpect[sizeof(data)]; 1172 int32_t i; 1173 1174 for (i = 0; i < sizeof(data); i++) 1175 data[i] = dataexpect[i] = i ^ 0x5a; 1176 1177 for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) { 1178 a = i * HIFN_SRAM_STEP_SIZE; 1179 memcpy(data, &i, sizeof(i)); 1180 hifn_writeramaddr(sc, a, data); 1181 } 1182 1183 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) { 1184 a = i * HIFN_SRAM_STEP_SIZE; 1185 memcpy(dataexpect, &i, sizeof(i)); 1186 if (hifn_readramaddr(sc, a, data) < 0) 1187 return (0); 1188 if (memcmp(data, dataexpect, sizeof(data)) != 0) 1189 return (0); 1190 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE; 1191 } 1192 1193 return (0); 1194 } 1195 1196 /* 1197 * XXX For dram boards, one should really try all of the 1198 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG 1199 * is already set up correctly. 1200 */ 1201 static int 1202 hifn_dramsize(struct hifn_softc *sc) 1203 { 1204 u_int32_t cnfg; 1205 1206 if (sc->sc_flags & HIFN_IS_7956) { 1207 /* 1208 * 7955/7956 have a fixed internal ram of only 32K. 1209 */ 1210 sc->sc_ramsize = 32768; 1211 } else { 1212 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) & 1213 HIFN_PUCNFG_DRAMMASK; 1214 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18); 1215 } 1216 return (0); 1217 } 1218 1219 static void 1220 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, 1221 int *resp) 1222 { 1223 struct hifn_dma *dma = sc->sc_dma; 1224 1225 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1226 dma->cmdi = 0; 1227 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1228 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1229 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1230 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1231 } 1232 *cmdp = dma->cmdi++; 1233 dma->cmdk = dma->cmdi; 1234 1235 if (dma->srci == HIFN_D_SRC_RSIZE) { 1236 dma->srci = 0; 1237 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID | 1238 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1239 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1240 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1241 } 1242 *srcp = dma->srci++; 1243 dma->srck = dma->srci; 1244 1245 if (dma->dsti == HIFN_D_DST_RSIZE) { 1246 dma->dsti = 0; 1247 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID | 1248 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1249 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE, 1250 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1251 } 1252 *dstp = dma->dsti++; 1253 dma->dstk = dma->dsti; 1254 1255 if (dma->resi == HIFN_D_RES_RSIZE) { 1256 dma->resi = 0; 1257 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1258 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1259 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1260 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1261 } 1262 *resp = dma->resi++; 1263 dma->resk = dma->resi; 1264 } 1265 1266 static int 1267 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1268 { 1269 struct hifn_dma *dma = sc->sc_dma; 1270 struct hifn_base_command wc; 1271 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1272 int r, cmdi, resi, srci, dsti; 1273 1274 wc.masks = htole16(3 << 13); 1275 wc.session_num = htole16(addr >> 14); 1276 wc.total_source_count = htole16(8); 1277 wc.total_dest_count = htole16(addr & 0x3fff); 1278 1279 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1280 1281 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1282 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1283 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1284 1285 /* build write command */ 1286 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1287 *(struct hifn_base_command *)dma->command_bufs[cmdi] = wc; 1288 memcpy(&dma->test_src, data, sizeof(dma->test_src)); 1289 1290 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1291 + offsetof(struct hifn_dma, test_src)); 1292 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1293 + offsetof(struct hifn_dma, test_dst)); 1294 1295 dma->cmdr[cmdi].l = htole32(16 | masks); 1296 dma->srcr[srci].l = htole32(8 | masks); 1297 dma->dstr[dsti].l = htole32(4 | masks); 1298 dma->resr[resi].l = htole32(4 | masks); 1299 1300 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1301 0, sc->sc_dmamap->dm_mapsize, 1302 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1303 1304 for (r = 10000; r >= 0; r--) { 1305 DELAY(10); 1306 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1307 0, sc->sc_dmamap->dm_mapsize, 1308 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1309 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1310 break; 1311 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1312 0, sc->sc_dmamap->dm_mapsize, 1313 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1314 } 1315 if (r == 0) { 1316 printf("%s: writeramaddr -- " 1317 "result[%d](addr %d) still valid\n", 1318 device_xname(sc->sc_dv), resi, addr); 1319 r = -1; 1320 return (-1); 1321 } else 1322 r = 0; 1323 1324 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1325 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1326 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1327 1328 return (r); 1329 } 1330 1331 static int 1332 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1333 { 1334 struct hifn_dma *dma = sc->sc_dma; 1335 struct hifn_base_command rc; 1336 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1337 int r, cmdi, srci, dsti, resi; 1338 1339 rc.masks = htole16(2 << 13); 1340 rc.session_num = htole16(addr >> 14); 1341 rc.total_source_count = htole16(addr & 0x3fff); 1342 rc.total_dest_count = htole16(8); 1343 1344 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1345 1346 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1347 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1348 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1349 1350 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1351 *(struct hifn_base_command *)dma->command_bufs[cmdi] = rc; 1352 1353 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1354 offsetof(struct hifn_dma, test_src)); 1355 dma->test_src = 0; 1356 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1357 offsetof(struct hifn_dma, test_dst)); 1358 dma->test_dst = 0; 1359 dma->cmdr[cmdi].l = htole32(8 | masks); 1360 dma->srcr[srci].l = htole32(8 | masks); 1361 dma->dstr[dsti].l = htole32(8 | masks); 1362 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks); 1363 1364 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1365 0, sc->sc_dmamap->dm_mapsize, 1366 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1367 1368 for (r = 10000; r >= 0; r--) { 1369 DELAY(10); 1370 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1371 0, sc->sc_dmamap->dm_mapsize, 1372 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1373 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1374 break; 1375 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1376 0, sc->sc_dmamap->dm_mapsize, 1377 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1378 } 1379 if (r == 0) { 1380 printf("%s: readramaddr -- " 1381 "result[%d](addr %d) still valid\n", 1382 device_xname(sc->sc_dv), resi, addr); 1383 r = -1; 1384 } else { 1385 r = 0; 1386 memcpy(data, &dma->test_dst, sizeof(dma->test_dst)); 1387 } 1388 1389 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1390 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1391 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1392 1393 return (r); 1394 } 1395 1396 /* 1397 * Initialize the descriptor rings. 1398 */ 1399 static void 1400 hifn_init_dma(struct hifn_softc *sc) 1401 { 1402 struct hifn_dma *dma = sc->sc_dma; 1403 int i; 1404 1405 hifn_set_retry(sc); 1406 1407 /* initialize static pointer values */ 1408 for (i = 0; i < HIFN_D_CMD_RSIZE; i++) 1409 dma->cmdr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1410 offsetof(struct hifn_dma, command_bufs[i][0])); 1411 for (i = 0; i < HIFN_D_RES_RSIZE; i++) 1412 dma->resr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1413 offsetof(struct hifn_dma, result_bufs[i][0])); 1414 1415 dma->cmdr[HIFN_D_CMD_RSIZE].p = 1416 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1417 offsetof(struct hifn_dma, cmdr[0])); 1418 dma->srcr[HIFN_D_SRC_RSIZE].p = 1419 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1420 offsetof(struct hifn_dma, srcr[0])); 1421 dma->dstr[HIFN_D_DST_RSIZE].p = 1422 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1423 offsetof(struct hifn_dma, dstr[0])); 1424 dma->resr[HIFN_D_RES_RSIZE].p = 1425 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1426 offsetof(struct hifn_dma, resr[0])); 1427 1428 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 1429 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 1430 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 1431 } 1432 1433 /* 1434 * Writes out the raw command buffer space. Returns the 1435 * command buffer size. 1436 */ 1437 static u_int 1438 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf) 1439 { 1440 u_int8_t *buf_pos; 1441 struct hifn_base_command *base_cmd; 1442 struct hifn_mac_command *mac_cmd; 1443 struct hifn_crypt_command *cry_cmd; 1444 struct hifn_comp_command *comp_cmd; 1445 int using_mac, using_crypt, using_comp, len, ivlen; 1446 u_int32_t dlen, slen; 1447 1448 buf_pos = buf; 1449 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC; 1450 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT; 1451 using_comp = cmd->base_masks & HIFN_BASE_CMD_COMP; 1452 1453 base_cmd = (struct hifn_base_command *)buf_pos; 1454 base_cmd->masks = htole16(cmd->base_masks); 1455 slen = cmd->src_map->dm_mapsize; 1456 if (cmd->sloplen) 1457 dlen = cmd->dst_map->dm_mapsize - cmd->sloplen + 1458 sizeof(u_int32_t); 1459 else 1460 dlen = cmd->dst_map->dm_mapsize; 1461 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO); 1462 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1463 dlen >>= 16; 1464 slen >>= 16; 1465 base_cmd->session_num = htole16(cmd->session_num | 1466 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1467 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1468 buf_pos += sizeof(struct hifn_base_command); 1469 1470 if (using_comp) { 1471 comp_cmd = (struct hifn_comp_command *)buf_pos; 1472 dlen = cmd->compcrd->crd_len; 1473 comp_cmd->source_count = htole16(dlen & 0xffff); 1474 dlen >>= 16; 1475 comp_cmd->masks = htole16(cmd->comp_masks | 1476 ((dlen << HIFN_COMP_CMD_SRCLEN_S) & HIFN_COMP_CMD_SRCLEN_M)); 1477 comp_cmd->header_skip = htole16(cmd->compcrd->crd_skip); 1478 comp_cmd->reserved = 0; 1479 buf_pos += sizeof(struct hifn_comp_command); 1480 } 1481 1482 if (using_mac) { 1483 mac_cmd = (struct hifn_mac_command *)buf_pos; 1484 dlen = cmd->maccrd->crd_len; 1485 mac_cmd->source_count = htole16(dlen & 0xffff); 1486 dlen >>= 16; 1487 mac_cmd->masks = htole16(cmd->mac_masks | 1488 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M)); 1489 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip); 1490 mac_cmd->reserved = 0; 1491 buf_pos += sizeof(struct hifn_mac_command); 1492 } 1493 1494 if (using_crypt) { 1495 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1496 dlen = cmd->enccrd->crd_len; 1497 cry_cmd->source_count = htole16(dlen & 0xffff); 1498 dlen >>= 16; 1499 cry_cmd->masks = htole16(cmd->cry_masks | 1500 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M)); 1501 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip); 1502 cry_cmd->reserved = 0; 1503 buf_pos += sizeof(struct hifn_crypt_command); 1504 } 1505 1506 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) { 1507 memcpy(buf_pos, cmd->mac, HIFN_MAC_KEY_LENGTH); 1508 buf_pos += HIFN_MAC_KEY_LENGTH; 1509 } 1510 1511 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) { 1512 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1513 case HIFN_CRYPT_CMD_ALG_3DES: 1514 memcpy(buf_pos, cmd->ck, HIFN_3DES_KEY_LENGTH); 1515 buf_pos += HIFN_3DES_KEY_LENGTH; 1516 break; 1517 case HIFN_CRYPT_CMD_ALG_DES: 1518 memcpy(buf_pos, cmd->ck, HIFN_DES_KEY_LENGTH); 1519 buf_pos += HIFN_DES_KEY_LENGTH; 1520 break; 1521 case HIFN_CRYPT_CMD_ALG_RC4: 1522 len = 256; 1523 do { 1524 int clen; 1525 1526 clen = MIN(cmd->cklen, len); 1527 memcpy(buf_pos, cmd->ck, clen); 1528 len -= clen; 1529 buf_pos += clen; 1530 } while (len > 0); 1531 memset(buf_pos, 0, 4); 1532 buf_pos += 4; 1533 break; 1534 case HIFN_CRYPT_CMD_ALG_AES: 1535 /* 1536 * AES keys are variable 128, 192 and 1537 * 256 bits (16, 24 and 32 bytes). 1538 */ 1539 memcpy(buf_pos, cmd->ck, cmd->cklen); 1540 buf_pos += cmd->cklen; 1541 break; 1542 } 1543 } 1544 1545 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) { 1546 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1547 case HIFN_CRYPT_CMD_ALG_AES: 1548 ivlen = HIFN_AES_IV_LENGTH; 1549 break; 1550 default: 1551 ivlen = HIFN_IV_LENGTH; 1552 break; 1553 } 1554 memcpy(buf_pos, cmd->iv, ivlen); 1555 buf_pos += ivlen; 1556 } 1557 1558 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC | HIFN_BASE_CMD_CRYPT | 1559 HIFN_BASE_CMD_COMP)) == 0) { 1560 memset(buf_pos, 0, 8); 1561 buf_pos += 8; 1562 } 1563 1564 return (buf_pos - buf); 1565 } 1566 1567 static int 1568 hifn_dmamap_aligned(bus_dmamap_t map) 1569 { 1570 int i; 1571 1572 for (i = 0; i < map->dm_nsegs; i++) { 1573 if (map->dm_segs[i].ds_addr & 3) 1574 return (0); 1575 if ((i != (map->dm_nsegs - 1)) && 1576 (map->dm_segs[i].ds_len & 3)) 1577 return (0); 1578 } 1579 return (1); 1580 } 1581 1582 static int 1583 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd) 1584 { 1585 struct hifn_dma *dma = sc->sc_dma; 1586 bus_dmamap_t map = cmd->dst_map; 1587 u_int32_t p, l; 1588 int idx, used = 0, i; 1589 1590 idx = dma->dsti; 1591 for (i = 0; i < map->dm_nsegs - 1; i++) { 1592 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1593 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1594 HIFN_D_MASKDONEIRQ | map->dm_segs[i].ds_len); 1595 HIFN_DSTR_SYNC(sc, idx, 1596 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1597 used++; 1598 1599 if (++idx == HIFN_D_DST_RSIZE) { 1600 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1601 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1602 HIFN_DSTR_SYNC(sc, idx, 1603 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1604 idx = 0; 1605 } 1606 } 1607 1608 if (cmd->sloplen == 0) { 1609 p = map->dm_segs[i].ds_addr; 1610 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1611 map->dm_segs[i].ds_len; 1612 } else { 1613 p = sc->sc_dmamap->dm_segs[0].ds_addr + 1614 offsetof(struct hifn_dma, slop[cmd->slopidx]); 1615 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1616 sizeof(u_int32_t); 1617 1618 if ((map->dm_segs[i].ds_len - cmd->sloplen) != 0) { 1619 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1620 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1621 HIFN_D_MASKDONEIRQ | 1622 (map->dm_segs[i].ds_len - cmd->sloplen)); 1623 HIFN_DSTR_SYNC(sc, idx, 1624 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1625 used++; 1626 1627 if (++idx == HIFN_D_DST_RSIZE) { 1628 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1629 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1630 HIFN_DSTR_SYNC(sc, idx, 1631 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1632 idx = 0; 1633 } 1634 } 1635 } 1636 dma->dstr[idx].p = htole32(p); 1637 dma->dstr[idx].l = htole32(l); 1638 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1639 used++; 1640 1641 if (++idx == HIFN_D_DST_RSIZE) { 1642 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP | 1643 HIFN_D_MASKDONEIRQ); 1644 HIFN_DSTR_SYNC(sc, idx, 1645 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1646 idx = 0; 1647 } 1648 1649 dma->dsti = idx; 1650 dma->dstu += used; 1651 return (idx); 1652 } 1653 1654 static int 1655 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd) 1656 { 1657 struct hifn_dma *dma = sc->sc_dma; 1658 bus_dmamap_t map = cmd->src_map; 1659 int idx, i; 1660 u_int32_t last = 0; 1661 1662 idx = dma->srci; 1663 for (i = 0; i < map->dm_nsegs; i++) { 1664 if (i == map->dm_nsegs - 1) 1665 last = HIFN_D_LAST; 1666 1667 dma->srcr[idx].p = htole32(map->dm_segs[i].ds_addr); 1668 dma->srcr[idx].l = htole32(map->dm_segs[i].ds_len | 1669 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last); 1670 HIFN_SRCR_SYNC(sc, idx, 1671 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1672 1673 if (++idx == HIFN_D_SRC_RSIZE) { 1674 dma->srcr[idx].l = htole32(HIFN_D_VALID | 1675 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1676 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1677 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1678 idx = 0; 1679 } 1680 } 1681 dma->srci = idx; 1682 dma->srcu += map->dm_nsegs; 1683 return (idx); 1684 } 1685 1686 static int 1687 hifn_crypto(struct hifn_softc *sc, struct hifn_command *cmd, 1688 struct cryptop *crp, int hint) 1689 { 1690 struct hifn_dma *dma = sc->sc_dma; 1691 u_int32_t cmdlen; 1692 int cmdi, resi, err = 0; 1693 1694 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 1695 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) 1696 return (ENOMEM); 1697 1698 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1699 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 1700 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 1701 err = ENOMEM; 1702 goto err_srcmap1; 1703 } 1704 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1705 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 1706 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 1707 err = ENOMEM; 1708 goto err_srcmap1; 1709 } 1710 } else { 1711 err = EINVAL; 1712 goto err_srcmap1; 1713 } 1714 1715 if (hifn_dmamap_aligned(cmd->src_map)) { 1716 cmd->sloplen = cmd->src_map->dm_mapsize & 3; 1717 if (crp->crp_flags & CRYPTO_F_IOV) 1718 cmd->dstu.dst_io = cmd->srcu.src_io; 1719 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1720 cmd->dstu.dst_m = cmd->srcu.src_m; 1721 cmd->dst_map = cmd->src_map; 1722 } else { 1723 if (crp->crp_flags & CRYPTO_F_IOV) { 1724 err = EINVAL; 1725 goto err_srcmap; 1726 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1727 int totlen, len; 1728 struct mbuf *m, *m0, *mlast; 1729 1730 totlen = cmd->src_map->dm_mapsize; 1731 if (cmd->srcu.src_m->m_flags & M_PKTHDR) { 1732 len = MHLEN; 1733 MGETHDR(m0, M_DONTWAIT, MT_DATA); 1734 } else { 1735 len = MLEN; 1736 MGET(m0, M_DONTWAIT, MT_DATA); 1737 } 1738 if (m0 == NULL) { 1739 err = ENOMEM; 1740 goto err_srcmap; 1741 } 1742 if (len == MHLEN) 1743 M_DUP_PKTHDR(m0, cmd->srcu.src_m); 1744 if (totlen >= MINCLSIZE) { 1745 MCLGET(m0, M_DONTWAIT); 1746 if (m0->m_flags & M_EXT) 1747 len = MCLBYTES; 1748 } 1749 totlen -= len; 1750 m0->m_pkthdr.len = m0->m_len = len; 1751 mlast = m0; 1752 1753 while (totlen > 0) { 1754 MGET(m, M_DONTWAIT, MT_DATA); 1755 if (m == NULL) { 1756 err = ENOMEM; 1757 m_freem(m0); 1758 goto err_srcmap; 1759 } 1760 len = MLEN; 1761 if (totlen >= MINCLSIZE) { 1762 MCLGET(m, M_DONTWAIT); 1763 if (m->m_flags & M_EXT) 1764 len = MCLBYTES; 1765 } 1766 1767 m->m_len = len; 1768 if (m0->m_flags & M_PKTHDR) 1769 m0->m_pkthdr.len += len; 1770 totlen -= len; 1771 1772 mlast->m_next = m; 1773 mlast = m; 1774 } 1775 cmd->dstu.dst_m = m0; 1776 } 1777 } 1778 1779 if (cmd->dst_map == NULL) { 1780 if (bus_dmamap_create(sc->sc_dmat, 1781 HIFN_MAX_SEGLEN * MAX_SCATTER, MAX_SCATTER, 1782 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 1783 err = ENOMEM; 1784 goto err_srcmap; 1785 } 1786 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1787 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 1788 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 1789 err = ENOMEM; 1790 goto err_dstmap1; 1791 } 1792 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1793 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 1794 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 1795 err = ENOMEM; 1796 goto err_dstmap1; 1797 } 1798 } 1799 } 1800 1801 #ifdef HIFN_DEBUG 1802 if (hifn_debug) 1803 printf("%s: Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n", 1804 device_xname(sc->sc_dv), 1805 READ_REG_1(sc, HIFN_1_DMA_CSR), 1806 READ_REG_1(sc, HIFN_1_DMA_IER), 1807 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1808 cmd->src_map->dm_nsegs, cmd->dst_map->dm_nsegs); 1809 #endif 1810 1811 if (cmd->src_map == cmd->dst_map) 1812 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1813 0, cmd->src_map->dm_mapsize, 1814 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 1815 else { 1816 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1817 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1818 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 1819 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 1820 } 1821 1822 /* 1823 * need 1 cmd, and 1 res 1824 * need N src, and N dst 1825 */ 1826 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 1827 (dma->resu + 1) > HIFN_D_RES_RSIZE) { 1828 err = ENOMEM; 1829 goto err_dstmap; 1830 } 1831 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 1832 (dma->dstu + cmd->dst_map->dm_nsegs + 1) > HIFN_D_DST_RSIZE) { 1833 err = ENOMEM; 1834 goto err_dstmap; 1835 } 1836 1837 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1838 dma->cmdi = 0; 1839 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1840 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1841 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1842 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1843 } 1844 cmdi = dma->cmdi++; 1845 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 1846 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 1847 1848 /* .p for command/result already set */ 1849 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 1850 HIFN_D_MASKDONEIRQ); 1851 HIFN_CMDR_SYNC(sc, cmdi, 1852 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1853 dma->cmdu++; 1854 if (sc->sc_c_busy == 0) { 1855 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1856 sc->sc_c_busy = 1; 1857 SET_LED(sc, HIFN_MIPSRST_LED0); 1858 } 1859 1860 /* 1861 * We don't worry about missing an interrupt (which a "command wait" 1862 * interrupt salvages us from), unless there is more than one command 1863 * in the queue. 1864 * 1865 * XXX We do seem to miss some interrupts. So we always enable 1866 * XXX command wait. From OpenBSD revision 1.149. 1867 * 1868 */ 1869 #if 0 1870 if (dma->cmdu > 1) { 1871 #endif 1872 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 1873 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1874 #if 0 1875 } 1876 #endif 1877 1878 hifnstats.hst_ipackets++; 1879 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 1880 1881 hifn_dmamap_load_src(sc, cmd); 1882 if (sc->sc_s_busy == 0) { 1883 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1884 sc->sc_s_busy = 1; 1885 SET_LED(sc, HIFN_MIPSRST_LED1); 1886 } 1887 1888 /* 1889 * Unlike other descriptors, we don't mask done interrupt from 1890 * result descriptor. 1891 */ 1892 #ifdef HIFN_DEBUG 1893 if (hifn_debug) 1894 printf("load res\n"); 1895 #endif 1896 if (dma->resi == HIFN_D_RES_RSIZE) { 1897 dma->resi = 0; 1898 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1899 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1900 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1901 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1902 } 1903 resi = dma->resi++; 1904 dma->hifn_commands[resi] = cmd; 1905 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 1906 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 1907 HIFN_D_VALID | HIFN_D_LAST); 1908 HIFN_RESR_SYNC(sc, resi, 1909 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1910 dma->resu++; 1911 if (sc->sc_r_busy == 0) { 1912 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1913 sc->sc_r_busy = 1; 1914 SET_LED(sc, HIFN_MIPSRST_LED2); 1915 } 1916 1917 if (cmd->sloplen) 1918 cmd->slopidx = resi; 1919 1920 hifn_dmamap_load_dst(sc, cmd); 1921 1922 if (sc->sc_d_busy == 0) { 1923 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1924 sc->sc_d_busy = 1; 1925 } 1926 1927 #ifdef HIFN_DEBUG 1928 if (hifn_debug) 1929 printf("%s: command: stat %8x ier %8x\n", 1930 device_xname(sc->sc_dv), 1931 READ_REG_1(sc, HIFN_1_DMA_CSR), READ_REG_1(sc, HIFN_1_DMA_IER)); 1932 #endif 1933 1934 sc->sc_active = 5; 1935 return (err); /* success */ 1936 1937 err_dstmap: 1938 if (cmd->src_map != cmd->dst_map) 1939 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 1940 err_dstmap1: 1941 if (cmd->src_map != cmd->dst_map) 1942 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 1943 err_srcmap: 1944 if (crp->crp_flags & CRYPTO_F_IMBUF && 1945 cmd->srcu.src_m != cmd->dstu.dst_m) 1946 m_freem(cmd->dstu.dst_m); 1947 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 1948 err_srcmap1: 1949 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 1950 return (err); 1951 } 1952 1953 static void 1954 hifn_tick(void *vsc) 1955 { 1956 struct hifn_softc *sc = vsc; 1957 1958 mutex_spin_enter(&sc->sc_mtx); 1959 if (sc->sc_active == 0) { 1960 struct hifn_dma *dma = sc->sc_dma; 1961 u_int32_t r = 0; 1962 1963 if (dma->cmdu == 0 && sc->sc_c_busy) { 1964 sc->sc_c_busy = 0; 1965 r |= HIFN_DMACSR_C_CTRL_DIS; 1966 CLR_LED(sc, HIFN_MIPSRST_LED0); 1967 } 1968 if (dma->srcu == 0 && sc->sc_s_busy) { 1969 sc->sc_s_busy = 0; 1970 r |= HIFN_DMACSR_S_CTRL_DIS; 1971 CLR_LED(sc, HIFN_MIPSRST_LED1); 1972 } 1973 if (dma->dstu == 0 && sc->sc_d_busy) { 1974 sc->sc_d_busy = 0; 1975 r |= HIFN_DMACSR_D_CTRL_DIS; 1976 } 1977 if (dma->resu == 0 && sc->sc_r_busy) { 1978 sc->sc_r_busy = 0; 1979 r |= HIFN_DMACSR_R_CTRL_DIS; 1980 CLR_LED(sc, HIFN_MIPSRST_LED2); 1981 } 1982 if (r) 1983 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r); 1984 } 1985 else 1986 sc->sc_active--; 1987 #ifdef __OpenBSD__ 1988 timeout_add(&sc->sc_tickto, hz); 1989 #else 1990 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 1991 #endif 1992 mutex_spin_exit(&sc->sc_mtx); 1993 } 1994 1995 static int 1996 hifn_intr(void *arg) 1997 { 1998 struct hifn_softc *sc = arg; 1999 struct hifn_dma *dma = sc->sc_dma; 2000 u_int32_t dmacsr, restart; 2001 int i, u; 2002 2003 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR); 2004 2005 #ifdef HIFN_DEBUG 2006 if (hifn_debug) 2007 printf("%s: irq: stat %08x ien %08x u %d/%d/%d/%d\n", 2008 device_xname(sc->sc_dv), 2009 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), 2010 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 2011 #endif 2012 2013 mutex_spin_enter(&sc->sc_mtx); 2014 2015 /* Nothing in the DMA unit interrupted */ 2016 if ((dmacsr & sc->sc_dmaier) == 0) { 2017 mutex_spin_exit(&sc->sc_mtx); 2018 return (0); 2019 } 2020 2021 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier); 2022 2023 if (dmacsr & HIFN_DMACSR_ENGINE) 2024 WRITE_REG_0(sc, HIFN_0_PUISR, READ_REG_0(sc, HIFN_0_PUISR)); 2025 2026 if ((sc->sc_flags & HIFN_HAS_PUBLIC) && 2027 (dmacsr & HIFN_DMACSR_PUBDONE)) 2028 WRITE_REG_1(sc, HIFN_1_PUB_STATUS, 2029 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 2030 2031 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 2032 if (restart) 2033 printf("%s: overrun %x\n", device_xname(sc->sc_dv), dmacsr); 2034 2035 if (sc->sc_flags & HIFN_IS_7811) { 2036 if (dmacsr & HIFN_DMACSR_ILLR) 2037 printf("%s: illegal read\n", device_xname(sc->sc_dv)); 2038 if (dmacsr & HIFN_DMACSR_ILLW) 2039 printf("%s: illegal write\n", device_xname(sc->sc_dv)); 2040 } 2041 2042 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 2043 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 2044 if (restart) { 2045 printf("%s: abort, resetting.\n", device_xname(sc->sc_dv)); 2046 hifnstats.hst_abort++; 2047 hifn_abort(sc); 2048 goto out; 2049 } 2050 2051 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->resu == 0)) { 2052 /* 2053 * If no slots to process and we receive a "waiting on 2054 * command" interrupt, we disable the "waiting on command" 2055 * (by clearing it). 2056 */ 2057 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 2058 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 2059 } 2060 2061 /* clear the rings */ 2062 i = dma->resk; 2063 while (dma->resu != 0) { 2064 HIFN_RESR_SYNC(sc, i, 2065 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2066 if (dma->resr[i].l & htole32(HIFN_D_VALID)) { 2067 HIFN_RESR_SYNC(sc, i, 2068 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2069 break; 2070 } 2071 2072 if (i != HIFN_D_RES_RSIZE) { 2073 struct hifn_command *cmd; 2074 2075 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD); 2076 cmd = dma->hifn_commands[i]; 2077 KASSERT(cmd != NULL 2078 /*("hifn_intr: null command slot %u", i)*/); 2079 dma->hifn_commands[i] = NULL; 2080 2081 hifn_callback(sc, cmd, dma->result_bufs[i]); 2082 hifnstats.hst_opackets++; 2083 } 2084 2085 if (++i == (HIFN_D_RES_RSIZE + 1)) 2086 i = 0; 2087 else 2088 dma->resu--; 2089 } 2090 dma->resk = i; 2091 2092 i = dma->srck; u = dma->srcu; 2093 while (u != 0) { 2094 HIFN_SRCR_SYNC(sc, i, 2095 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2096 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) { 2097 HIFN_SRCR_SYNC(sc, i, 2098 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2099 break; 2100 } 2101 if (++i == (HIFN_D_SRC_RSIZE + 1)) 2102 i = 0; 2103 else 2104 u--; 2105 } 2106 dma->srck = i; dma->srcu = u; 2107 2108 i = dma->cmdk; u = dma->cmdu; 2109 while (u != 0) { 2110 HIFN_CMDR_SYNC(sc, i, 2111 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2112 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) { 2113 HIFN_CMDR_SYNC(sc, i, 2114 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2115 break; 2116 } 2117 if (i != HIFN_D_CMD_RSIZE) { 2118 u--; 2119 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE); 2120 } 2121 if (++i == (HIFN_D_CMD_RSIZE + 1)) 2122 i = 0; 2123 } 2124 dma->cmdk = i; dma->cmdu = u; 2125 2126 out: 2127 mutex_spin_exit(&sc->sc_mtx); 2128 return (1); 2129 } 2130 2131 /* 2132 * Allocate a new 'session' and return an encoded session id. 'sidp' 2133 * contains our registration id, and should contain an encoded session 2134 * id on successful allocation. 2135 */ 2136 static int 2137 hifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 2138 { 2139 struct cryptoini *c; 2140 struct hifn_softc *sc = arg; 2141 int i, mac = 0, cry = 0, comp = 0, retval = EINVAL; 2142 2143 KASSERT(sc != NULL /*, ("hifn_newsession: null softc")*/); 2144 if (sidp == NULL || cri == NULL || sc == NULL) 2145 return retval; 2146 2147 mutex_spin_enter(&sc->sc_mtx); 2148 2149 for (i = 0; i < sc->sc_maxses; i++) 2150 if (sc->sc_sessions[i].hs_state == HS_STATE_FREE) 2151 break; 2152 if (i == sc->sc_maxses) { 2153 retval = ENOMEM; 2154 goto out; 2155 } 2156 2157 for (c = cri; c != NULL; c = c->cri_next) { 2158 switch (c->cri_alg) { 2159 case CRYPTO_MD5: 2160 case CRYPTO_SHA1: 2161 case CRYPTO_MD5_HMAC_96: 2162 case CRYPTO_SHA1_HMAC_96: 2163 if (mac) { 2164 goto out; 2165 } 2166 mac = 1; 2167 break; 2168 case CRYPTO_DES_CBC: 2169 case CRYPTO_3DES_CBC: 2170 case CRYPTO_AES_CBC: 2171 /* Note that this is an initialization 2172 vector, not a cipher key; any function 2173 giving sufficient Hamming distance 2174 between outputs is fine. Use of RC4 2175 to generate IVs has been FIPS140-2 2176 certified by several labs. */ 2177 #ifdef __NetBSD__ 2178 cprng_fast(sc->sc_sessions[i].hs_iv, 2179 c->cri_alg == CRYPTO_AES_CBC ? 2180 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2181 #else /* FreeBSD and OpenBSD have get_random_bytes */ 2182 /* XXX this may read fewer, does it matter? */ 2183 get_random_bytes(sc->sc_sessions[i].hs_iv, 2184 c->cri_alg == CRYPTO_AES_CBC ? 2185 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2186 #endif 2187 /*FALLTHROUGH*/ 2188 case CRYPTO_ARC4: 2189 if (cry) { 2190 goto out; 2191 } 2192 cry = 1; 2193 break; 2194 #ifdef HAVE_CRYPTO_LZS 2195 case CRYPTO_LZS_COMP: 2196 if (comp) { 2197 goto out; 2198 } 2199 comp = 1; 2200 break; 2201 #endif 2202 default: 2203 goto out; 2204 } 2205 } 2206 if (mac == 0 && cry == 0 && comp == 0) { 2207 goto out; 2208 } 2209 2210 /* 2211 * XXX only want to support compression without chaining to 2212 * MAC/crypt engine right now 2213 */ 2214 if ((comp && mac) || (comp && cry)) { 2215 goto out; 2216 } 2217 2218 *sidp = HIFN_SID(device_unit(sc->sc_dv), i); 2219 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2220 2221 retval = 0; 2222 out: 2223 mutex_spin_exit(&sc->sc_mtx); 2224 return retval; 2225 } 2226 2227 /* 2228 * Deallocate a session. 2229 * XXX this routine should run a zero'd mac/encrypt key into context ram. 2230 * XXX to blow away any keys already stored there. 2231 */ 2232 static int 2233 hifn_freesession(void *arg, u_int64_t tid) 2234 { 2235 struct hifn_softc *sc = arg; 2236 int session; 2237 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 2238 2239 KASSERT(sc != NULL /*, ("hifn_freesession: null softc")*/); 2240 if (sc == NULL) 2241 return (EINVAL); 2242 2243 mutex_spin_enter(&sc->sc_mtx); 2244 session = HIFN_SESSION(sid); 2245 if (session >= sc->sc_maxses) { 2246 mutex_spin_exit(&sc->sc_mtx); 2247 return (EINVAL); 2248 } 2249 2250 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 2251 mutex_spin_exit(&sc->sc_mtx); 2252 return (0); 2253 } 2254 2255 static int 2256 hifn_process(void *arg, struct cryptop *crp, int hint) 2257 { 2258 struct hifn_softc *sc = arg; 2259 struct hifn_command *cmd = NULL; 2260 int session, err, ivlen; 2261 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 2262 2263 if (crp == NULL || crp->crp_callback == NULL) { 2264 hifnstats.hst_invalid++; 2265 return (EINVAL); 2266 } 2267 2268 mutex_spin_enter(&sc->sc_mtx); 2269 session = HIFN_SESSION(crp->crp_sid); 2270 2271 if (sc == NULL || session >= sc->sc_maxses) { 2272 err = EINVAL; 2273 goto errout; 2274 } 2275 2276 cmd = (struct hifn_command *)malloc(sizeof(struct hifn_command), 2277 M_DEVBUF, M_NOWAIT|M_ZERO); 2278 if (cmd == NULL) { 2279 hifnstats.hst_nomem++; 2280 err = ENOMEM; 2281 goto errout; 2282 } 2283 2284 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2285 cmd->srcu.src_m = (struct mbuf *)crp->crp_buf; 2286 cmd->dstu.dst_m = (struct mbuf *)crp->crp_buf; 2287 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2288 cmd->srcu.src_io = (struct uio *)crp->crp_buf; 2289 cmd->dstu.dst_io = (struct uio *)crp->crp_buf; 2290 } else { 2291 err = EINVAL; 2292 goto errout; /* XXX we don't handle contiguous buffers! */ 2293 } 2294 2295 crd1 = crp->crp_desc; 2296 if (crd1 == NULL) { 2297 err = EINVAL; 2298 goto errout; 2299 } 2300 crd2 = crd1->crd_next; 2301 2302 if (crd2 == NULL) { 2303 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2304 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2305 crd1->crd_alg == CRYPTO_SHA1 || 2306 crd1->crd_alg == CRYPTO_MD5) { 2307 maccrd = crd1; 2308 enccrd = NULL; 2309 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 2310 crd1->crd_alg == CRYPTO_3DES_CBC || 2311 crd1->crd_alg == CRYPTO_AES_CBC || 2312 crd1->crd_alg == CRYPTO_ARC4) { 2313 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0) 2314 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2315 maccrd = NULL; 2316 enccrd = crd1; 2317 #ifdef HAVE_CRYPTO_LZS 2318 } else if (crd1->crd_alg == CRYPTO_LZS_COMP) { 2319 return (hifn_compression(sc, crp, cmd)); 2320 #endif 2321 } else { 2322 err = EINVAL; 2323 goto errout; 2324 } 2325 } else { 2326 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2327 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2328 crd1->crd_alg == CRYPTO_MD5 || 2329 crd1->crd_alg == CRYPTO_SHA1) && 2330 (crd2->crd_alg == CRYPTO_DES_CBC || 2331 crd2->crd_alg == CRYPTO_3DES_CBC || 2332 crd2->crd_alg == CRYPTO_AES_CBC || 2333 crd2->crd_alg == CRYPTO_ARC4) && 2334 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 2335 cmd->base_masks = HIFN_BASE_CMD_DECODE; 2336 maccrd = crd1; 2337 enccrd = crd2; 2338 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 2339 crd1->crd_alg == CRYPTO_ARC4 || 2340 crd1->crd_alg == CRYPTO_3DES_CBC || 2341 crd1->crd_alg == CRYPTO_AES_CBC) && 2342 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 2343 crd2->crd_alg == CRYPTO_SHA1_HMAC_96 || 2344 crd2->crd_alg == CRYPTO_MD5 || 2345 crd2->crd_alg == CRYPTO_SHA1) && 2346 (crd1->crd_flags & CRD_F_ENCRYPT)) { 2347 enccrd = crd1; 2348 maccrd = crd2; 2349 } else { 2350 /* 2351 * We cannot order the 7751 as requested 2352 */ 2353 err = EINVAL; 2354 goto errout; 2355 } 2356 } 2357 2358 if (enccrd) { 2359 cmd->enccrd = enccrd; 2360 cmd->base_masks |= HIFN_BASE_CMD_CRYPT; 2361 switch (enccrd->crd_alg) { 2362 case CRYPTO_ARC4: 2363 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4; 2364 if ((enccrd->crd_flags & CRD_F_ENCRYPT) 2365 != sc->sc_sessions[session].hs_prev_op) 2366 sc->sc_sessions[session].hs_state = 2367 HS_STATE_USED; 2368 break; 2369 case CRYPTO_DES_CBC: 2370 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES | 2371 HIFN_CRYPT_CMD_MODE_CBC | 2372 HIFN_CRYPT_CMD_NEW_IV; 2373 break; 2374 case CRYPTO_3DES_CBC: 2375 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES | 2376 HIFN_CRYPT_CMD_MODE_CBC | 2377 HIFN_CRYPT_CMD_NEW_IV; 2378 break; 2379 case CRYPTO_AES_CBC: 2380 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES | 2381 HIFN_CRYPT_CMD_MODE_CBC | 2382 HIFN_CRYPT_CMD_NEW_IV; 2383 break; 2384 default: 2385 err = EINVAL; 2386 goto errout; 2387 } 2388 if (enccrd->crd_alg != CRYPTO_ARC4) { 2389 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? 2390 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2391 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 2392 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2393 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2394 else 2395 bcopy(sc->sc_sessions[session].hs_iv, 2396 cmd->iv, ivlen); 2397 2398 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) 2399 == 0) { 2400 if (crp->crp_flags & CRYPTO_F_IMBUF) 2401 m_copyback(cmd->srcu.src_m, 2402 enccrd->crd_inject, 2403 ivlen, cmd->iv); 2404 else if (crp->crp_flags & CRYPTO_F_IOV) 2405 cuio_copyback(cmd->srcu.src_io, 2406 enccrd->crd_inject, 2407 ivlen, cmd->iv); 2408 } 2409 } else { 2410 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2411 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2412 else if (crp->crp_flags & CRYPTO_F_IMBUF) 2413 m_copydata(cmd->srcu.src_m, 2414 enccrd->crd_inject, ivlen, cmd->iv); 2415 else if (crp->crp_flags & CRYPTO_F_IOV) 2416 cuio_copydata(cmd->srcu.src_io, 2417 enccrd->crd_inject, ivlen, cmd->iv); 2418 } 2419 } 2420 2421 cmd->ck = enccrd->crd_key; 2422 cmd->cklen = enccrd->crd_klen >> 3; 2423 2424 /* 2425 * Need to specify the size for the AES key in the masks. 2426 */ 2427 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) == 2428 HIFN_CRYPT_CMD_ALG_AES) { 2429 switch (cmd->cklen) { 2430 case 16: 2431 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128; 2432 break; 2433 case 24: 2434 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192; 2435 break; 2436 case 32: 2437 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256; 2438 break; 2439 default: 2440 err = EINVAL; 2441 goto errout; 2442 } 2443 } 2444 2445 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2446 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; 2447 } 2448 2449 if (maccrd) { 2450 cmd->maccrd = maccrd; 2451 cmd->base_masks |= HIFN_BASE_CMD_MAC; 2452 2453 switch (maccrd->crd_alg) { 2454 case CRYPTO_MD5: 2455 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2456 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2457 HIFN_MAC_CMD_POS_IPSEC; 2458 break; 2459 case CRYPTO_MD5_HMAC_96: 2460 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2461 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2462 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2463 break; 2464 case CRYPTO_SHA1: 2465 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2466 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2467 HIFN_MAC_CMD_POS_IPSEC; 2468 break; 2469 case CRYPTO_SHA1_HMAC_96: 2470 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2471 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2472 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2473 break; 2474 } 2475 2476 if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC_96 || 2477 maccrd->crd_alg == CRYPTO_MD5_HMAC_96) && 2478 sc->sc_sessions[session].hs_state == HS_STATE_USED) { 2479 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY; 2480 memcpy(cmd->mac, maccrd->crd_key, maccrd->crd_klen >> 3); 2481 memset(cmd->mac + (maccrd->crd_klen >> 3), 0, 2482 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3)); 2483 } 2484 } 2485 2486 cmd->crp = crp; 2487 cmd->session_num = session; 2488 cmd->softc = sc; 2489 2490 err = hifn_crypto(sc, cmd, crp, hint); 2491 if (err == 0) { 2492 if (enccrd) 2493 sc->sc_sessions[session].hs_prev_op = 2494 enccrd->crd_flags & CRD_F_ENCRYPT; 2495 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2496 sc->sc_sessions[session].hs_state = HS_STATE_KEY; 2497 mutex_spin_exit(&sc->sc_mtx); 2498 return 0; 2499 } else if (err == ERESTART) { 2500 /* 2501 * There weren't enough resources to dispatch the request 2502 * to the part. Notify the caller so they'll requeue this 2503 * request and resubmit it again soon. 2504 */ 2505 #ifdef HIFN_DEBUG 2506 if (hifn_debug) 2507 printf("%s: requeue request\n", device_xname(sc->sc_dv)); 2508 #endif 2509 free(cmd, M_DEVBUF); 2510 sc->sc_needwakeup |= CRYPTO_SYMQ; 2511 mutex_spin_exit(&sc->sc_mtx); 2512 return (err); 2513 } 2514 2515 errout: 2516 if (cmd != NULL) 2517 free(cmd, M_DEVBUF); 2518 if (err == EINVAL) 2519 hifnstats.hst_invalid++; 2520 else 2521 hifnstats.hst_nomem++; 2522 crp->crp_etype = err; 2523 mutex_spin_exit(&sc->sc_mtx); 2524 crypto_done(crp); 2525 return (0); 2526 } 2527 2528 static void 2529 hifn_abort(struct hifn_softc *sc) 2530 { 2531 struct hifn_dma *dma = sc->sc_dma; 2532 struct hifn_command *cmd; 2533 struct cryptop *crp; 2534 int i, u; 2535 2536 i = dma->resk; u = dma->resu; 2537 while (u != 0) { 2538 cmd = dma->hifn_commands[i]; 2539 KASSERT(cmd != NULL /*, ("hifn_abort: null cmd slot %u", i)*/); 2540 dma->hifn_commands[i] = NULL; 2541 crp = cmd->crp; 2542 2543 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) { 2544 /* Salvage what we can. */ 2545 hifnstats.hst_opackets++; 2546 hifn_callback(sc, cmd, dma->result_bufs[i]); 2547 } else { 2548 if (cmd->src_map == cmd->dst_map) { 2549 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2550 0, cmd->src_map->dm_mapsize, 2551 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2552 } else { 2553 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2554 0, cmd->src_map->dm_mapsize, 2555 BUS_DMASYNC_POSTWRITE); 2556 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2557 0, cmd->dst_map->dm_mapsize, 2558 BUS_DMASYNC_POSTREAD); 2559 } 2560 2561 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2562 m_freem(cmd->srcu.src_m); 2563 crp->crp_buf = (void *)cmd->dstu.dst_m; 2564 } 2565 2566 /* non-shared buffers cannot be restarted */ 2567 if (cmd->src_map != cmd->dst_map) { 2568 /* 2569 * XXX should be EAGAIN, delayed until 2570 * after the reset. 2571 */ 2572 crp->crp_etype = ENOMEM; 2573 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2574 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2575 } else 2576 crp->crp_etype = ENOMEM; 2577 2578 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2579 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2580 2581 free(cmd, M_DEVBUF); 2582 if (crp->crp_etype != EAGAIN) 2583 crypto_done(crp); 2584 } 2585 2586 if (++i == HIFN_D_RES_RSIZE) 2587 i = 0; 2588 u--; 2589 } 2590 dma->resk = i; dma->resu = u; 2591 2592 /* Force upload of key next time */ 2593 for (i = 0; i < sc->sc_maxses; i++) 2594 if (sc->sc_sessions[i].hs_state == HS_STATE_KEY) 2595 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2596 2597 hifn_reset_board(sc, 1); 2598 hifn_init_dma(sc); 2599 hifn_init_pci_registers(sc); 2600 } 2601 2602 static void 2603 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *resbuf) 2604 { 2605 struct hifn_dma *dma = sc->sc_dma; 2606 struct cryptop *crp = cmd->crp; 2607 struct cryptodesc *crd; 2608 struct mbuf *m; 2609 int totlen, i, u, ivlen; 2610 2611 if (cmd->src_map == cmd->dst_map) 2612 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2613 0, cmd->src_map->dm_mapsize, 2614 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2615 else { 2616 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2617 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2618 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2619 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2620 } 2621 2622 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2623 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2624 crp->crp_buf = (void *)cmd->dstu.dst_m; 2625 totlen = cmd->src_map->dm_mapsize; 2626 for (m = cmd->dstu.dst_m; m != NULL; m = m->m_next) { 2627 if (totlen < m->m_len) { 2628 m->m_len = totlen; 2629 totlen = 0; 2630 } else 2631 totlen -= m->m_len; 2632 } 2633 cmd->dstu.dst_m->m_pkthdr.len = 2634 cmd->srcu.src_m->m_pkthdr.len; 2635 m_freem(cmd->srcu.src_m); 2636 } 2637 } 2638 2639 if (cmd->sloplen != 0) { 2640 if (crp->crp_flags & CRYPTO_F_IMBUF) 2641 m_copyback((struct mbuf *)crp->crp_buf, 2642 cmd->src_map->dm_mapsize - cmd->sloplen, 2643 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2644 else if (crp->crp_flags & CRYPTO_F_IOV) 2645 cuio_copyback((struct uio *)crp->crp_buf, 2646 cmd->src_map->dm_mapsize - cmd->sloplen, 2647 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2648 } 2649 2650 i = dma->dstk; u = dma->dstu; 2651 while (u != 0) { 2652 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2653 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2654 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2655 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2656 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2657 offsetof(struct hifn_dma, dstr[i]), 2658 sizeof(struct hifn_desc), 2659 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2660 break; 2661 } 2662 if (++i == (HIFN_D_DST_RSIZE + 1)) 2663 i = 0; 2664 else 2665 u--; 2666 } 2667 dma->dstk = i; dma->dstu = u; 2668 2669 hifnstats.hst_obytes += cmd->dst_map->dm_mapsize; 2670 2671 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) == 2672 HIFN_BASE_CMD_CRYPT) { 2673 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2674 if (crd->crd_alg != CRYPTO_DES_CBC && 2675 crd->crd_alg != CRYPTO_3DES_CBC && 2676 crd->crd_alg != CRYPTO_AES_CBC) 2677 continue; 2678 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? 2679 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2680 if (crp->crp_flags & CRYPTO_F_IMBUF) 2681 m_copydata((struct mbuf *)crp->crp_buf, 2682 crd->crd_skip + crd->crd_len - ivlen, 2683 ivlen, 2684 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2685 else if (crp->crp_flags & CRYPTO_F_IOV) { 2686 cuio_copydata((struct uio *)crp->crp_buf, 2687 crd->crd_skip + crd->crd_len - ivlen, 2688 ivlen, 2689 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2690 } 2691 /* XXX We do not handle contig data */ 2692 break; 2693 } 2694 } 2695 2696 if (cmd->base_masks & HIFN_BASE_CMD_MAC) { 2697 u_int8_t *macbuf; 2698 2699 macbuf = resbuf + sizeof(struct hifn_base_result); 2700 if (cmd->base_masks & HIFN_BASE_CMD_COMP) 2701 macbuf += sizeof(struct hifn_comp_result); 2702 macbuf += sizeof(struct hifn_mac_result); 2703 2704 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2705 int len; 2706 2707 if (crd->crd_alg == CRYPTO_MD5) 2708 len = 16; 2709 else if (crd->crd_alg == CRYPTO_SHA1) 2710 len = 20; 2711 else if (crd->crd_alg == CRYPTO_MD5_HMAC_96 || 2712 crd->crd_alg == CRYPTO_SHA1_HMAC_96) 2713 len = 12; 2714 else 2715 continue; 2716 2717 if (crp->crp_flags & CRYPTO_F_IMBUF) 2718 m_copyback((struct mbuf *)crp->crp_buf, 2719 crd->crd_inject, len, macbuf); 2720 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac) 2721 memcpy(crp->crp_mac, (void *)macbuf, len); 2722 break; 2723 } 2724 } 2725 2726 if (cmd->src_map != cmd->dst_map) { 2727 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2728 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2729 } 2730 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2731 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2732 free(cmd, M_DEVBUF); 2733 crypto_done(crp); 2734 } 2735 2736 #ifdef HAVE_CRYPTO_LZS 2737 2738 static int 2739 hifn_compression(struct hifn_softc *sc, struct cryptop *crp, 2740 struct hifn_command *cmd) 2741 { 2742 struct cryptodesc *crd = crp->crp_desc; 2743 int s, err = 0; 2744 2745 cmd->compcrd = crd; 2746 cmd->base_masks |= HIFN_BASE_CMD_COMP; 2747 2748 if ((crp->crp_flags & CRYPTO_F_IMBUF) == 0) { 2749 /* 2750 * XXX can only handle mbufs right now since we can 2751 * XXX dynamically resize them. 2752 */ 2753 err = EINVAL; 2754 return (ENOMEM); 2755 } 2756 2757 if ((crd->crd_flags & CRD_F_COMP) == 0) 2758 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2759 if (crd->crd_alg == CRYPTO_LZS_COMP) 2760 cmd->comp_masks |= HIFN_COMP_CMD_ALG_LZS | 2761 HIFN_COMP_CMD_CLEARHIST; 2762 2763 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2764 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) { 2765 err = ENOMEM; 2766 goto fail; 2767 } 2768 2769 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2770 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 2771 err = ENOMEM; 2772 goto fail; 2773 } 2774 2775 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2776 int len; 2777 2778 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 2779 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 2780 err = ENOMEM; 2781 goto fail; 2782 } 2783 2784 len = cmd->src_map->dm_mapsize / MCLBYTES; 2785 if ((cmd->src_map->dm_mapsize % MCLBYTES) != 0) 2786 len++; 2787 len *= MCLBYTES; 2788 2789 if ((crd->crd_flags & CRD_F_COMP) == 0) 2790 len *= 4; 2791 2792 if (len > HIFN_MAX_DMALEN) 2793 len = HIFN_MAX_DMALEN; 2794 2795 cmd->dstu.dst_m = hifn_mkmbuf_chain(len, cmd->srcu.src_m); 2796 if (cmd->dstu.dst_m == NULL) { 2797 err = ENOMEM; 2798 goto fail; 2799 } 2800 2801 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2802 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2803 err = ENOMEM; 2804 goto fail; 2805 } 2806 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2807 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 2808 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 2809 err = ENOMEM; 2810 goto fail; 2811 } 2812 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 2813 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 2814 err = ENOMEM; 2815 goto fail; 2816 } 2817 } 2818 2819 if (cmd->src_map == cmd->dst_map) 2820 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2821 0, cmd->src_map->dm_mapsize, 2822 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 2823 else { 2824 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2825 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2826 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2827 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2828 } 2829 2830 cmd->crp = crp; 2831 /* 2832 * Always use session 0. The modes of compression we use are 2833 * stateless and there is always at least one compression 2834 * context, zero. 2835 */ 2836 cmd->session_num = 0; 2837 cmd->softc = sc; 2838 2839 err = hifn_compress_enter(sc, cmd); 2840 2841 if (err != 0) 2842 goto fail; 2843 return (0); 2844 2845 fail: 2846 if (cmd->dst_map != NULL) { 2847 if (cmd->dst_map->dm_nsegs > 0) 2848 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2849 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2850 } 2851 if (cmd->src_map != NULL) { 2852 if (cmd->src_map->dm_nsegs > 0) 2853 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2854 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2855 } 2856 free(cmd, M_DEVBUF); 2857 if (err == EINVAL) 2858 hifnstats.hst_invalid++; 2859 else 2860 hifnstats.hst_nomem++; 2861 crp->crp_etype = err; 2862 crypto_done(crp); 2863 return (0); 2864 } 2865 2866 static int 2867 hifn_compress_enter(struct hifn_softc *sc, struct hifn_command *cmd) 2868 { 2869 struct hifn_dma *dma = sc->sc_dma; 2870 int cmdi, resi; 2871 u_int32_t cmdlen; 2872 2873 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 2874 (dma->resu + 1) > HIFN_D_CMD_RSIZE) 2875 return (ENOMEM); 2876 2877 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 2878 (dma->dstu + cmd->dst_map->dm_nsegs) > HIFN_D_DST_RSIZE) 2879 return (ENOMEM); 2880 2881 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 2882 dma->cmdi = 0; 2883 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 2884 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2885 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 2886 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2887 } 2888 cmdi = dma->cmdi++; 2889 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 2890 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 2891 2892 /* .p for command/result already set */ 2893 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 2894 HIFN_D_MASKDONEIRQ); 2895 HIFN_CMDR_SYNC(sc, cmdi, 2896 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2897 dma->cmdu++; 2898 if (sc->sc_c_busy == 0) { 2899 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 2900 sc->sc_c_busy = 1; 2901 SET_LED(sc, HIFN_MIPSRST_LED0); 2902 } 2903 2904 /* 2905 * We don't worry about missing an interrupt (which a "command wait" 2906 * interrupt salvages us from), unless there is more than one command 2907 * in the queue. 2908 */ 2909 if (dma->cmdu > 1) { 2910 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 2911 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 2912 } 2913 2914 hifnstats.hst_ipackets++; 2915 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 2916 2917 hifn_dmamap_load_src(sc, cmd); 2918 if (sc->sc_s_busy == 0) { 2919 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 2920 sc->sc_s_busy = 1; 2921 SET_LED(sc, HIFN_MIPSRST_LED1); 2922 } 2923 2924 /* 2925 * Unlike other descriptors, we don't mask done interrupt from 2926 * result descriptor. 2927 */ 2928 if (dma->resi == HIFN_D_RES_RSIZE) { 2929 dma->resi = 0; 2930 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 2931 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2932 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 2933 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2934 } 2935 resi = dma->resi++; 2936 dma->hifn_commands[resi] = cmd; 2937 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 2938 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 2939 HIFN_D_VALID | HIFN_D_LAST); 2940 HIFN_RESR_SYNC(sc, resi, 2941 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2942 dma->resu++; 2943 if (sc->sc_r_busy == 0) { 2944 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 2945 sc->sc_r_busy = 1; 2946 SET_LED(sc, HIFN_MIPSRST_LED2); 2947 } 2948 2949 if (cmd->sloplen) 2950 cmd->slopidx = resi; 2951 2952 hifn_dmamap_load_dst(sc, cmd); 2953 2954 if (sc->sc_d_busy == 0) { 2955 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 2956 sc->sc_d_busy = 1; 2957 } 2958 sc->sc_active = 5; 2959 cmd->cmd_callback = hifn_callback_comp; 2960 return (0); 2961 } 2962 2963 static void 2964 hifn_callback_comp(struct hifn_softc *sc, struct hifn_command *cmd, 2965 u_int8_t *resbuf) 2966 { 2967 struct hifn_base_result baseres; 2968 struct cryptop *crp = cmd->crp; 2969 struct hifn_dma *dma = sc->sc_dma; 2970 struct mbuf *m; 2971 int err = 0, i, u; 2972 u_int32_t olen; 2973 bus_size_t dstsize; 2974 2975 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2976 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2977 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2978 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2979 2980 dstsize = cmd->dst_map->dm_mapsize; 2981 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2982 2983 memcpy(&baseres, resbuf, sizeof(struct hifn_base_result)); 2984 2985 i = dma->dstk; u = dma->dstu; 2986 while (u != 0) { 2987 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2988 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2989 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2990 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2991 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2992 offsetof(struct hifn_dma, dstr[i]), 2993 sizeof(struct hifn_desc), 2994 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2995 break; 2996 } 2997 if (++i == (HIFN_D_DST_RSIZE + 1)) 2998 i = 0; 2999 else 3000 u--; 3001 } 3002 dma->dstk = i; dma->dstu = u; 3003 3004 if (baseres.flags & htole16(HIFN_BASE_RES_DSTOVERRUN)) { 3005 bus_size_t xlen; 3006 3007 xlen = dstsize; 3008 3009 m_freem(cmd->dstu.dst_m); 3010 3011 if (xlen == HIFN_MAX_DMALEN) { 3012 /* We've done all we can. */ 3013 err = E2BIG; 3014 goto out; 3015 } 3016 3017 xlen += MCLBYTES; 3018 3019 if (xlen > HIFN_MAX_DMALEN) 3020 xlen = HIFN_MAX_DMALEN; 3021 3022 cmd->dstu.dst_m = hifn_mkmbuf_chain(xlen, 3023 cmd->srcu.src_m); 3024 if (cmd->dstu.dst_m == NULL) { 3025 err = ENOMEM; 3026 goto out; 3027 } 3028 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 3029 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 3030 err = ENOMEM; 3031 goto out; 3032 } 3033 3034 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 3035 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 3036 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 3037 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 3038 3039 err = hifn_compress_enter(sc, cmd); 3040 if (err != 0) 3041 goto out; 3042 return; 3043 } 3044 3045 olen = dstsize - (letoh16(baseres.dst_cnt) | 3046 (((letoh16(baseres.session) & HIFN_BASE_RES_DSTLEN_M) >> 3047 HIFN_BASE_RES_DSTLEN_S) << 16)); 3048 3049 crp->crp_olen = olen - cmd->compcrd->crd_skip; 3050 3051 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 3052 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 3053 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 3054 3055 m = cmd->dstu.dst_m; 3056 if (m->m_flags & M_PKTHDR) 3057 m->m_pkthdr.len = olen; 3058 crp->crp_buf = (void *)m; 3059 for (; m != NULL; m = m->m_next) { 3060 if (olen >= m->m_len) 3061 olen -= m->m_len; 3062 else { 3063 m->m_len = olen; 3064 olen = 0; 3065 } 3066 } 3067 3068 m_freem(cmd->srcu.src_m); 3069 free(cmd, M_DEVBUF); 3070 crp->crp_etype = 0; 3071 crypto_done(crp); 3072 return; 3073 3074 out: 3075 if (cmd->dst_map != NULL) { 3076 if (cmd->src_map->dm_nsegs != 0) 3077 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 3078 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 3079 } 3080 if (cmd->src_map != NULL) { 3081 if (cmd->src_map->dm_nsegs != 0) 3082 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 3083 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 3084 } 3085 if (cmd->dstu.dst_m != NULL) 3086 m_freem(cmd->dstu.dst_m); 3087 free(cmd, M_DEVBUF); 3088 crp->crp_etype = err; 3089 crypto_done(crp); 3090 } 3091 3092 static struct mbuf * 3093 hifn_mkmbuf_chain(int totlen, struct mbuf *mtemplate) 3094 { 3095 int len; 3096 struct mbuf *m, *m0, *mlast; 3097 3098 if (mtemplate->m_flags & M_PKTHDR) { 3099 len = MHLEN; 3100 MGETHDR(m0, M_DONTWAIT, MT_DATA); 3101 } else { 3102 len = MLEN; 3103 MGET(m0, M_DONTWAIT, MT_DATA); 3104 } 3105 if (m0 == NULL) 3106 return (NULL); 3107 if (len == MHLEN) 3108 M_DUP_PKTHDR(m0, mtemplate); 3109 MCLGET(m0, M_DONTWAIT); 3110 if (!(m0->m_flags & M_EXT)) { 3111 m_freem(m0); 3112 return (NULL); 3113 } 3114 len = MCLBYTES; 3115 3116 totlen -= len; 3117 m0->m_pkthdr.len = m0->m_len = len; 3118 mlast = m0; 3119 3120 while (totlen > 0) { 3121 MGET(m, M_DONTWAIT, MT_DATA); 3122 if (m == NULL) { 3123 m_freem(m0); 3124 return (NULL); 3125 } 3126 MCLGET(m, M_DONTWAIT); 3127 if (!(m->m_flags & M_EXT)) { 3128 m_freem(m); 3129 m_freem(m0); 3130 return (NULL); 3131 } 3132 len = MCLBYTES; 3133 m->m_len = len; 3134 if (m0->m_flags & M_PKTHDR) 3135 m0->m_pkthdr.len += len; 3136 totlen -= len; 3137 3138 mlast->m_next = m; 3139 mlast = m; 3140 } 3141 3142 return (m0); 3143 } 3144 #endif /* HAVE_CRYPTO_LZS */ 3145 3146 static void 3147 hifn_write_4(struct hifn_softc *sc, int reggrp, bus_size_t reg, u_int32_t val) 3148 { 3149 /* 3150 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0 3151 * and Group 1 registers; avoid conditions that could create 3152 * burst writes by doing a read in between the writes. 3153 */ 3154 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3155 if (sc->sc_waw_lastgroup == reggrp && 3156 sc->sc_waw_lastreg == reg - 4) { 3157 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID); 3158 } 3159 sc->sc_waw_lastgroup = reggrp; 3160 sc->sc_waw_lastreg = reg; 3161 } 3162 if (reggrp == 0) 3163 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val); 3164 else 3165 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val); 3166 3167 } 3168 3169 static u_int32_t 3170 hifn_read_4(struct hifn_softc *sc, int reggrp, bus_size_t reg) 3171 { 3172 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3173 sc->sc_waw_lastgroup = -1; 3174 sc->sc_waw_lastreg = 1; 3175 } 3176 if (reggrp == 0) 3177 return (bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg)); 3178 return (bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg)); 3179 } 3180
Indexes created Sun Sep 28 16:09:52 GMT 2025