hifn7751.c revision 1.68
1 /* $NetBSD: hifn7751.c,v 1.68 2020/05/17 00:49:28 riastradh Exp $ */ 2 /* $FreeBSD: hifn7751.c,v 1.5.2.7 2003/10/08 23:52:00 sam Exp $ */ 3 /* $OpenBSD: hifn7751.c,v 1.140 2003/08/01 17:55:54 deraadt Exp $ */ 4 5 /* 6 * Invertex AEON / Hifn 7751 driver 7 * Copyright (c) 1999 Invertex Inc. All rights reserved. 8 * Copyright (c) 1999 Theo de Raadt 9 * Copyright (c) 2000-2001 Network Security Technologies, Inc. 10 * http://www.netsec.net 11 * Copyright (c) 2003 Hifn Inc. 12 * 13 * This driver is based on a previous driver by Invertex, for which they 14 * requested: Please send any comments, feedback, bug-fixes, or feature 15 * requests to software (at) invertex.com. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * 1. Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * 2. Redistributions in binary form must reproduce the above copyright 24 * notice, this list of conditions and the following disclaimer in the 25 * documentation and/or other materials provided with the distribution. 26 * 3. The name of the author may not be used to endorse or promote products 27 * derived from this software without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 30 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 31 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 32 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 33 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 34 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 38 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Effort sponsored in part by the Defense Advanced Research Projects 41 * Agency (DARPA) and Air Force Research Laboratory, Air Force 42 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 43 * 44 */ 45 46 /* 47 * Driver for various Hifn pre-HIPP encryption processors. 48 */ 49 50 #include <sys/cdefs.h> 51 __KERNEL_RCSID(0, "$NetBSD: hifn7751.c,v 1.68 2020/05/17 00:49:28 riastradh Exp $"); 52 53 #include <sys/param.h> 54 #include <sys/cprng.h> 55 #include <sys/device.h> 56 #include <sys/endian.h> 57 #include <sys/errno.h> 58 #include <sys/kernel.h> 59 #include <sys/malloc.h> 60 #include <sys/mbuf.h> 61 #include <sys/module.h> 62 #include <sys/mutex.h> 63 #include <sys/proc.h> 64 #include <sys/rndsource.h> 65 #include <sys/sha1.h> 66 #include <sys/systm.h> 67 68 #include <opencrypto/cryptodev.h> 69 70 #include <dev/pci/pcireg.h> 71 #include <dev/pci/pcivar.h> 72 #include <dev/pci/pcidevs.h> 73 74 #include <dev/pci/hifn7751reg.h> 75 #include <dev/pci/hifn7751var.h> 76 77 #undef HIFN_DEBUG 78 79 #ifdef HIFN_DEBUG 80 extern int hifn_debug; /* patchable */ 81 int hifn_debug = 1; 82 #endif 83 84 /* 85 * Prototypes and count for the pci_device structure 86 */ 87 static int hifn_match(device_t, cfdata_t, void *); 88 static void hifn_attach(device_t, device_t, void *); 89 static int hifn_detach(device_t, int); 90 91 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc), 92 hifn_match, hifn_attach, hifn_detach, NULL); 93 94 static void hifn_reset_board(struct hifn_softc *, int); 95 static void hifn_reset_puc(struct hifn_softc *); 96 static void hifn_puc_wait(struct hifn_softc *); 97 static const char *hifn_enable_crypto(struct hifn_softc *, pcireg_t); 98 static void hifn_set_retry(struct hifn_softc *); 99 static void hifn_init_dma(struct hifn_softc *); 100 static void hifn_init_pci_registers(struct hifn_softc *); 101 static int hifn_sramsize(struct hifn_softc *); 102 static int hifn_dramsize(struct hifn_softc *); 103 static int hifn_ramtype(struct hifn_softc *); 104 static void hifn_sessions(struct hifn_softc *); 105 static int hifn_intr(void *); 106 static u_int hifn_write_command(struct hifn_command *, uint8_t *); 107 static uint32_t hifn_next_signature(uint32_t a, u_int cnt); 108 static int hifn_newsession(void*, uint32_t *, struct cryptoini *); 109 static int hifn_freesession(void*, uint64_t); 110 static int hifn_process(void*, struct cryptop *, int); 111 static void hifn_callback(struct hifn_softc *, struct hifn_command *, 112 uint8_t *); 113 static int hifn_crypto(struct hifn_softc *, struct hifn_command *, 114 struct cryptop*, int); 115 static int hifn_readramaddr(struct hifn_softc *, int, uint8_t *); 116 static int hifn_writeramaddr(struct hifn_softc *, int, uint8_t *); 117 static int hifn_dmamap_aligned(bus_dmamap_t); 118 static int hifn_dmamap_load_src(struct hifn_softc *, 119 struct hifn_command *); 120 static int hifn_dmamap_load_dst(struct hifn_softc *, 121 struct hifn_command *); 122 static int hifn_init_pubrng(struct hifn_softc *); 123 static void hifn_rng(void *); 124 static void hifn_rng_locked(void *); 125 static void hifn_tick(void *); 126 static void hifn_abort(struct hifn_softc *); 127 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, 128 int *); 129 static void hifn_write_4(struct hifn_softc *, int, bus_size_t, uint32_t); 130 static uint32_t hifn_read_4(struct hifn_softc *, int, bus_size_t); 131 #ifdef CRYPTO_LZS_COMP 132 static int hifn_compression(struct hifn_softc *, struct cryptop *, 133 struct hifn_command *); 134 static struct mbuf *hifn_mkmbuf_chain(int, struct mbuf *); 135 static int hifn_compress_enter(struct hifn_softc *, struct hifn_command *); 136 static void hifn_callback_comp(struct hifn_softc *, struct hifn_command *, 137 uint8_t *); 138 #endif /* CRYPTO_LZS_COMP */ 139 140 struct hifn_stats hifnstats; 141 142 static const struct hifn_product { 143 pci_vendor_id_t hifn_vendor; 144 pci_product_id_t hifn_product; 145 int hifn_flags; 146 const char *hifn_name; 147 } hifn_products[] = { 148 { PCI_VENDOR_INVERTEX, PCI_PRODUCT_INVERTEX_AEON, 149 0, 150 "Invertex AEON", 151 }, 152 153 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7751, 154 0, 155 "Hifn 7751", 156 }, 157 { PCI_VENDOR_NETSEC, PCI_PRODUCT_NETSEC_7751, 158 0, 159 "Hifn 7751 (NetSec)" 160 }, 161 162 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7811, 163 HIFN_IS_7811 | HIFN_HAS_RNG | HIFN_HAS_LEDS | HIFN_NO_BURSTWRITE, 164 "Hifn 7811", 165 }, 166 167 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7951, 168 HIFN_HAS_RNG | HIFN_HAS_PUBLIC, 169 "Hifn 7951", 170 }, 171 172 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7955, 173 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 174 "Hifn 7955", 175 }, 176 177 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7956, 178 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 179 "Hifn 7956", 180 }, 181 182 183 { 0, 0, 184 0, 185 NULL 186 } 187 }; 188 189 static const struct hifn_product * 190 hifn_lookup(const struct pci_attach_args *pa) 191 { 192 const struct hifn_product *hp; 193 194 for (hp = hifn_products; hp->hifn_name != NULL; hp++) { 195 if (PCI_VENDOR(pa->pa_id) == hp->hifn_vendor && 196 PCI_PRODUCT(pa->pa_id) == hp->hifn_product) 197 return (hp); 198 } 199 return (NULL); 200 } 201 202 static int 203 hifn_match(device_t parent, cfdata_t match, void *aux) 204 { 205 struct pci_attach_args *pa = aux; 206 207 if (hifn_lookup(pa) != NULL) 208 return 1; 209 210 return 0; 211 } 212 213 static void 214 hifn_attach(device_t parent, device_t self, void *aux) 215 { 216 struct hifn_softc *sc = device_private(self); 217 struct pci_attach_args *pa = aux; 218 const struct hifn_product *hp; 219 pci_chipset_tag_t pc = pa->pa_pc; 220 pci_intr_handle_t ih; 221 const char *intrstr = NULL; 222 const char *hifncap; 223 char rbase; 224 uint32_t cmd; 225 uint16_t ena; 226 bus_dma_segment_t seg; 227 bus_dmamap_t dmamap; 228 int rseg; 229 void *kva; 230 char intrbuf[PCI_INTRSTR_LEN]; 231 232 hp = hifn_lookup(pa); 233 if (hp == NULL) { 234 printf("\n"); 235 panic("hifn_attach: impossible"); 236 } 237 238 pci_aprint_devinfo_fancy(pa, "Crypto processor", hp->hifn_name, 1); 239 240 sc->sc_dv = self; 241 sc->sc_pci_pc = pa->pa_pc; 242 sc->sc_pci_tag = pa->pa_tag; 243 244 sc->sc_flags = hp->hifn_flags; 245 246 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 247 cmd |= PCI_COMMAND_MASTER_ENABLE; 248 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 249 250 if (pci_mapreg_map(pa, HIFN_BAR0, PCI_MAPREG_TYPE_MEM, 0, 251 &sc->sc_st0, &sc->sc_sh0, NULL, &sc->sc_iosz0)) { 252 aprint_error_dev(sc->sc_dv, "can't map mem space %d\n", 0); 253 return; 254 } 255 256 if (pci_mapreg_map(pa, HIFN_BAR1, PCI_MAPREG_TYPE_MEM, 0, 257 &sc->sc_st1, &sc->sc_sh1, NULL, &sc->sc_iosz1)) { 258 aprint_error_dev(sc->sc_dv, "can't find mem space %d\n", 1); 259 goto fail_io0; 260 } 261 262 hifn_set_retry(sc); 263 264 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 265 sc->sc_waw_lastgroup = -1; 266 sc->sc_waw_lastreg = 1; 267 } 268 269 sc->sc_dmat = pa->pa_dmat; 270 if (bus_dmamem_alloc(sc->sc_dmat, sizeof(*sc->sc_dma), PAGE_SIZE, 0, 271 &seg, 1, &rseg, BUS_DMA_NOWAIT)) { 272 aprint_error_dev(sc->sc_dv, "can't alloc DMA buffer\n"); 273 goto fail_io1; 274 } 275 if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, sizeof(*sc->sc_dma), &kva, 276 BUS_DMA_NOWAIT)) { 277 aprint_error_dev(sc->sc_dv, "can't map DMA buffers (%lu bytes)\n", 278 (u_long)sizeof(*sc->sc_dma)); 279 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 280 goto fail_io1; 281 } 282 if (bus_dmamap_create(sc->sc_dmat, sizeof(*sc->sc_dma), 1, 283 sizeof(*sc->sc_dma), 0, BUS_DMA_NOWAIT, &dmamap)) { 284 aprint_error_dev(sc->sc_dv, "can't create DMA map\n"); 285 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 286 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 287 goto fail_io1; 288 } 289 if (bus_dmamap_load(sc->sc_dmat, dmamap, kva, sizeof(*sc->sc_dma), 290 NULL, BUS_DMA_NOWAIT)) { 291 aprint_error_dev(sc->sc_dv, "can't load DMA map\n"); 292 bus_dmamap_destroy(sc->sc_dmat, dmamap); 293 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 294 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 295 goto fail_io1; 296 } 297 sc->sc_dmamap = dmamap; 298 sc->sc_dma = (struct hifn_dma *)kva; 299 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 300 301 hifn_reset_board(sc, 0); 302 303 if ((hifncap = hifn_enable_crypto(sc, pa->pa_id)) == NULL) { 304 aprint_error_dev(sc->sc_dv, "crypto enabling failed\n"); 305 goto fail_mem; 306 } 307 hifn_reset_puc(sc); 308 309 hifn_init_dma(sc); 310 hifn_init_pci_registers(sc); 311 312 /* XXX can't dynamically determine ram type for 795x; force dram */ 313 if (sc->sc_flags & HIFN_IS_7956) 314 sc->sc_drammodel = 1; 315 else if (hifn_ramtype(sc)) 316 goto fail_mem; 317 318 if (sc->sc_drammodel == 0) 319 hifn_sramsize(sc); 320 else 321 hifn_dramsize(sc); 322 323 /* 324 * Workaround for NetSec 7751 rev A: half ram size because two 325 * of the address lines were left floating 326 */ 327 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETSEC && 328 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETSEC_7751 && 329 PCI_REVISION(pa->pa_class) == 0x61) 330 sc->sc_ramsize >>= 1; 331 332 if (pci_intr_map(pa, &ih)) { 333 aprint_error_dev(sc->sc_dv, "couldn't map interrupt\n"); 334 goto fail_mem; 335 } 336 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 337 sc->sc_ih = pci_intr_establish_xname(pc, ih, IPL_NET, hifn_intr, sc, 338 device_xname(self)); 339 if (sc->sc_ih == NULL) { 340 aprint_error_dev(sc->sc_dv, "couldn't establish interrupt\n"); 341 if (intrstr != NULL) 342 aprint_error(" at %s", intrstr); 343 aprint_error("\n"); 344 goto fail_mem; 345 } 346 347 hifn_sessions(sc); 348 349 rseg = sc->sc_ramsize / 1024; 350 rbase = 'K'; 351 if (sc->sc_ramsize >= (1024 * 1024)) { 352 rbase = 'M'; 353 rseg /= 1024; 354 } 355 aprint_normal_dev(sc->sc_dv, "%s, %d%cB %cRAM, interrupting at %s\n", 356 hifncap, rseg, rbase, 357 sc->sc_drammodel ? 'D' : 'S', intrstr); 358 359 sc->sc_cid = crypto_get_driverid(0); 360 if (sc->sc_cid < 0) { 361 aprint_error_dev(sc->sc_dv, "couldn't get crypto driver id\n"); 362 goto fail_intr; 363 } 364 365 WRITE_REG_0(sc, HIFN_0_PUCNFG, 366 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID); 367 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 368 369 switch (ena) { 370 case HIFN_PUSTAT_ENA_2: 371 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 372 hifn_newsession, hifn_freesession, hifn_process, sc); 373 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0, 374 hifn_newsession, hifn_freesession, hifn_process, sc); 375 if (sc->sc_flags & HIFN_HAS_AES) 376 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 377 hifn_newsession, hifn_freesession, 378 hifn_process, sc); 379 /*FALLTHROUGH*/ 380 case HIFN_PUSTAT_ENA_1: 381 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0, 382 hifn_newsession, hifn_freesession, hifn_process, sc); 383 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0, 384 hifn_newsession, hifn_freesession, hifn_process, sc); 385 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 386 hifn_newsession, hifn_freesession, hifn_process, sc); 387 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 388 hifn_newsession, hifn_freesession, hifn_process, sc); 389 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 390 hifn_newsession, hifn_freesession, hifn_process, sc); 391 break; 392 } 393 394 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 0, 395 sc->sc_dmamap->dm_mapsize, 396 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 397 398 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_VM); 399 400 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) { 401 hifn_init_pubrng(sc); 402 } 403 404 callout_init(&sc->sc_tickto, CALLOUT_MPSAFE); 405 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 406 return; 407 408 fail_intr: 409 pci_intr_disestablish(pc, sc->sc_ih); 410 fail_mem: 411 bus_dmamap_unload(sc->sc_dmat, dmamap); 412 bus_dmamap_destroy(sc->sc_dmat, dmamap); 413 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 414 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 415 416 /* Turn off DMA polling */ 417 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 418 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 419 420 fail_io1: 421 bus_space_unmap(sc->sc_st1, sc->sc_sh1, sc->sc_iosz1); 422 fail_io0: 423 bus_space_unmap(sc->sc_st0, sc->sc_sh0, sc->sc_iosz0); 424 } 425 426 static int 427 hifn_detach(device_t self, int flags) 428 { 429 struct hifn_softc *sc = device_private(self); 430 431 hifn_abort(sc); 432 433 hifn_reset_board(sc, 1); 434 435 pci_intr_disestablish(sc->sc_pci_pc, sc->sc_ih); 436 437 crypto_unregister_all(sc->sc_cid); 438 439 rnd_detach_source(&sc->sc_rnd_source); 440 441 mutex_enter(&sc->sc_mtx); 442 callout_halt(&sc->sc_tickto, NULL); 443 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) 444 callout_halt(&sc->sc_rngto, NULL); 445 mutex_exit(&sc->sc_mtx); 446 447 bus_space_unmap(sc->sc_st1, sc->sc_sh1, sc->sc_iosz1); 448 bus_space_unmap(sc->sc_st0, sc->sc_sh0, sc->sc_iosz0); 449 450 /* 451 * XXX It's not clear if any additional buffers have been 452 * XXX allocated and require free()ing 453 */ 454 455 return 0; 456 } 457 458 MODULE(MODULE_CLASS_DRIVER, hifn, "pci,opencrypto"); 459 460 #ifdef _MODULE 461 #include "ioconf.c" 462 #endif 463 464 static int 465 hifn_modcmd(modcmd_t cmd, void *data) 466 { 467 int error = 0; 468 469 switch (cmd) { 470 case MODULE_CMD_INIT: 471 #ifdef _MODULE 472 error = config_init_component(cfdriver_ioconf_hifn, 473 cfattach_ioconf_hifn, cfdata_ioconf_hifn); 474 #endif 475 return error; 476 case MODULE_CMD_FINI: 477 #ifdef _MODULE 478 error = config_fini_component(cfdriver_ioconf_hifn, 479 cfattach_ioconf_hifn, cfdata_ioconf_hifn); 480 #endif 481 return error; 482 default: 483 return ENOTTY; 484 } 485 } 486 487 static void 488 hifn_rng_get(size_t bytes, void *priv) 489 { 490 struct hifn_softc *sc = priv; 491 492 mutex_enter(&sc->sc_mtx); 493 sc->sc_rng_need = bytes; 494 callout_reset(&sc->sc_rngto, 0, hifn_rng, sc); 495 mutex_exit(&sc->sc_mtx); 496 } 497 498 static int 499 hifn_init_pubrng(struct hifn_softc *sc) 500 { 501 uint32_t r; 502 int i; 503 504 if ((sc->sc_flags & HIFN_IS_7811) == 0) { 505 /* Reset 7951 public key/rng engine */ 506 WRITE_REG_1(sc, HIFN_1_PUB_RESET, 507 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET); 508 509 for (i = 0; i < 100; i++) { 510 DELAY(1000); 511 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) & 512 HIFN_PUBRST_RESET) == 0) 513 break; 514 } 515 516 if (i == 100) { 517 printf("%s: public key init failed\n", 518 device_xname(sc->sc_dv)); 519 return (1); 520 } 521 } 522 523 /* Enable the rng, if available */ 524 if (sc->sc_flags & HIFN_HAS_RNG) { 525 if (sc->sc_flags & HIFN_IS_7811) { 526 r = READ_REG_1(sc, HIFN_1_7811_RNGENA); 527 if (r & HIFN_7811_RNGENA_ENA) { 528 r &= ~HIFN_7811_RNGENA_ENA; 529 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 530 } 531 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG, 532 HIFN_7811_RNGCFG_DEFL); 533 r |= HIFN_7811_RNGENA_ENA; 534 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 535 } else 536 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG, 537 READ_REG_1(sc, HIFN_1_RNG_CONFIG) | 538 HIFN_RNGCFG_ENA); 539 540 /* 541 * The Hifn RNG documentation states that at their 542 * recommended "conservative" RNG config values, 543 * the RNG must warm up for 0.4s before providing 544 * data that meet their worst-case estimate of 0.06 545 * bits of random data per output register bit. 546 */ 547 DELAY(4000); 548 549 if (hz >= 100) 550 sc->sc_rnghz = hz / 100; 551 else 552 sc->sc_rnghz = 1; 553 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE); 554 rndsource_setcb(&sc->sc_rnd_source, hifn_rng_get, sc); 555 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dv), 556 RND_TYPE_RNG, RND_FLAG_COLLECT_VALUE|RND_FLAG_HASCB); 557 } 558 559 /* Enable public key engine, if available */ 560 if (sc->sc_flags & HIFN_HAS_PUBLIC) { 561 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 562 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE; 563 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 564 } 565 566 return (0); 567 } 568 569 static void 570 hifn_rng_locked(void *vsc) 571 { 572 struct hifn_softc *sc = vsc; 573 uint32_t num[64]; 574 uint32_t sts; 575 int i; 576 size_t got, gotent; 577 578 if (sc->sc_rng_need < 1) { 579 callout_stop(&sc->sc_rngto); 580 return; 581 } 582 583 if (sc->sc_flags & HIFN_IS_7811) { 584 for (i = 0; i < 5; i++) { /* XXX why 5? */ 585 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS); 586 if (sts & HIFN_7811_RNGSTS_UFL) { 587 printf("%s: RNG underflow: disabling\n", 588 device_xname(sc->sc_dv)); 589 return; 590 } 591 if ((sts & HIFN_7811_RNGSTS_RDY) == 0) 592 break; 593 594 /* 595 * There are at least two words in the RNG FIFO 596 * at this point. 597 */ 598 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 599 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 600 got = 2 * sizeof(num[0]); 601 gotent = (got * NBBY) / HIFN_RNG_BITSPER; 602 rnd_add_data(&sc->sc_rnd_source, num, got, gotent); 603 sc->sc_rng_need -= gotent; 604 } 605 } else { 606 int nwords = 0; 607 608 if (sc->sc_rng_need) { 609 nwords = (sc->sc_rng_need * NBBY) / HIFN_RNG_BITSPER; 610 nwords = MIN((int)__arraycount(num), nwords); 611 } 612 613 if (nwords < 2) { 614 nwords = 2; 615 } 616 617 /* 618 * We must be *extremely* careful here. The Hifn 619 * 795x differ from the published 6500 RNG design 620 * in more ways than the obvious lack of the output 621 * FIFO and LFSR control registers. In fact, there 622 * is only one LFSR, instead of the 6500's two, and 623 * it's 32 bits, not 31. 624 * 625 * Further, a block diagram obtained from Hifn shows 626 * a very curious latching of this register: the LFSR 627 * rotates at a frequency of RNG_Clk / 8, but the 628 * RNG_Data register is latched at a frequency of 629 * RNG_Clk, which means that it is possible for 630 * consecutive reads of the RNG_Data register to read 631 * identical state from the LFSR. The simplest 632 * workaround seems to be to read eight samples from 633 * the register for each one that we use. Since each 634 * read must require at least one PCI cycle, and 635 * RNG_Clk is at least PCI_Clk, this is safe. 636 */ 637 for (i = 0 ; i < nwords * 8; i++) { 638 volatile uint32_t regtmp; 639 regtmp = READ_REG_1(sc, HIFN_1_RNG_DATA); 640 num[i / 8] = regtmp; 641 } 642 643 got = nwords * sizeof(num[0]); 644 gotent = (got * NBBY) / HIFN_RNG_BITSPER; 645 rnd_add_data(&sc->sc_rnd_source, num, got, gotent); 646 sc->sc_rng_need -= gotent; 647 } 648 649 if (sc->sc_rng_need > 0) { 650 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc); 651 } 652 } 653 654 static void 655 hifn_rng(void *vsc) 656 { 657 struct hifn_softc *sc = vsc; 658 659 mutex_spin_enter(&sc->sc_mtx); 660 hifn_rng_locked(vsc); 661 mutex_spin_exit(&sc->sc_mtx); 662 } 663 664 static void 665 hifn_puc_wait(struct hifn_softc *sc) 666 { 667 int i; 668 669 for (i = 5000; i > 0; i--) { 670 DELAY(1); 671 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET)) 672 break; 673 } 674 if (!i) 675 printf("%s: proc unit did not reset\n", device_xname(sc->sc_dv)); 676 } 677 678 /* 679 * Reset the processing unit. 680 */ 681 static void 682 hifn_reset_puc(struct hifn_softc *sc) 683 { 684 /* Reset processing unit */ 685 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 686 hifn_puc_wait(sc); 687 } 688 689 static void 690 hifn_set_retry(struct hifn_softc *sc) 691 { 692 uint32_t r; 693 694 r = pci_conf_read(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT); 695 r &= 0xffff0000; 696 pci_conf_write(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT, r); 697 } 698 699 /* 700 * Resets the board. Values in the regesters are left as is 701 * from the reset (i.e. initial values are assigned elsewhere). 702 */ 703 static void 704 hifn_reset_board(struct hifn_softc *sc, int full) 705 { 706 uint32_t reg; 707 708 /* 709 * Set polling in the DMA configuration register to zero. 0x7 avoids 710 * resetting the board and zeros out the other fields. 711 */ 712 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 713 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 714 715 /* 716 * Now that polling has been disabled, we have to wait 1 ms 717 * before resetting the board. 718 */ 719 DELAY(1000); 720 721 /* Reset the DMA unit */ 722 if (full) { 723 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 724 DELAY(1000); 725 } else { 726 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, 727 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET); 728 hifn_reset_puc(sc); 729 } 730 731 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 732 733 /* Bring dma unit out of reset */ 734 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 735 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 736 737 hifn_puc_wait(sc); 738 739 hifn_set_retry(sc); 740 741 if (sc->sc_flags & HIFN_IS_7811) { 742 for (reg = 0; reg < 1000; reg++) { 743 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) & 744 HIFN_MIPSRST_CRAMINIT) 745 break; 746 DELAY(1000); 747 } 748 if (reg == 1000) 749 printf(": cram init timeout\n"); 750 } 751 } 752 753 static uint32_t 754 hifn_next_signature(uint32_t a, u_int cnt) 755 { 756 u_int i; 757 uint32_t v; 758 759 for (i = 0; i < cnt; i++) { 760 761 /* get the parity */ 762 v = a & 0x80080125; 763 v ^= v >> 16; 764 v ^= v >> 8; 765 v ^= v >> 4; 766 v ^= v >> 2; 767 v ^= v >> 1; 768 769 a = (v & 1) ^ (a << 1); 770 } 771 772 return a; 773 } 774 775 static struct pci2id { 776 u_short pci_vendor; 777 u_short pci_prod; 778 char card_id[13]; 779 } const pci2id[] = { 780 { 781 PCI_VENDOR_HIFN, 782 PCI_PRODUCT_HIFN_7951, 783 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 784 0x00, 0x00, 0x00, 0x00, 0x00 } 785 }, { 786 PCI_VENDOR_HIFN, 787 PCI_PRODUCT_HIFN_7955, 788 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 789 0x00, 0x00, 0x00, 0x00, 0x00 } 790 }, { 791 PCI_VENDOR_HIFN, 792 PCI_PRODUCT_HIFN_7956, 793 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 794 0x00, 0x00, 0x00, 0x00, 0x00 } 795 }, { 796 PCI_VENDOR_NETSEC, 797 PCI_PRODUCT_NETSEC_7751, 798 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 799 0x00, 0x00, 0x00, 0x00, 0x00 } 800 }, { 801 PCI_VENDOR_INVERTEX, 802 PCI_PRODUCT_INVERTEX_AEON, 803 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 804 0x00, 0x00, 0x00, 0x00, 0x00 } 805 }, { 806 PCI_VENDOR_HIFN, 807 PCI_PRODUCT_HIFN_7811, 808 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 809 0x00, 0x00, 0x00, 0x00, 0x00 } 810 }, { 811 /* 812 * Other vendors share this PCI ID as well, such as 813 * http://www.powercrypt.com, and obviously they also 814 * use the same key. 815 */ 816 PCI_VENDOR_HIFN, 817 PCI_PRODUCT_HIFN_7751, 818 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 819 0x00, 0x00, 0x00, 0x00, 0x00 } 820 }, 821 }; 822 823 /* 824 * Checks to see if crypto is already enabled. If crypto isn't enable, 825 * "hifn_enable_crypto" is called to enable it. The check is important, 826 * as enabling crypto twice will lock the board. 827 */ 828 static const char * 829 hifn_enable_crypto(struct hifn_softc *sc, pcireg_t pciid) 830 { 831 uint32_t dmacfg, ramcfg, encl, addr, i; 832 const char *offtbl = NULL; 833 834 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { 835 if (pci2id[i].pci_vendor == PCI_VENDOR(pciid) && 836 pci2id[i].pci_prod == PCI_PRODUCT(pciid)) { 837 offtbl = pci2id[i].card_id; 838 break; 839 } 840 } 841 842 if (offtbl == NULL) { 843 #ifdef HIFN_DEBUG 844 aprint_debug_dev(sc->sc_dv, "Unknown card!\n"); 845 #endif 846 return (NULL); 847 } 848 849 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG); 850 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG); 851 852 /* 853 * The RAM config register's encrypt level bit needs to be set before 854 * every read performed on the encryption level register. 855 */ 856 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 857 858 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 859 860 /* 861 * Make sure we don't re-unlock. Two unlocks kills chip until the 862 * next reboot. 863 */ 864 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) { 865 #ifdef HIFN_DEBUG 866 aprint_debug_dev(sc->sc_dv, "Strong Crypto already enabled!\n"); 867 #endif 868 goto report; 869 } 870 871 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) { 872 #ifdef HIFN_DEBUG 873 aprint_debug_dev(sc->sc_dv, "Unknown encryption level\n"); 874 #endif 875 return (NULL); 876 } 877 878 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK | 879 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 880 DELAY(1000); 881 addr = READ_REG_1(sc, HIFN_1_UNLOCK_SECRET1); 882 DELAY(1000); 883 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, 0); 884 DELAY(1000); 885 886 for (i = 0; i <= 12; i++) { 887 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 888 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, addr); 889 890 DELAY(1000); 891 } 892 893 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 894 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 895 896 #ifdef HIFN_DEBUG 897 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2) 898 aprint_debug("Encryption engine is permanently locked until next system reset."); 899 else 900 aprint_debug("Encryption engine enabled successfully!"); 901 #endif 902 903 report: 904 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg); 905 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg); 906 907 switch (encl) { 908 case HIFN_PUSTAT_ENA_0: 909 return ("LZS-only (no encr/auth)"); 910 911 case HIFN_PUSTAT_ENA_1: 912 return ("DES"); 913 914 case HIFN_PUSTAT_ENA_2: 915 if (sc->sc_flags & HIFN_HAS_AES) 916 return ("3DES/AES"); 917 else 918 return ("3DES"); 919 920 default: 921 return ("disabled"); 922 } 923 /* NOTREACHED */ 924 } 925 926 /* 927 * Give initial values to the registers listed in the "Register Space" 928 * section of the HIFN Software Development reference manual. 929 */ 930 static void 931 hifn_init_pci_registers(struct hifn_softc *sc) 932 { 933 /* write fixed values needed by the Initialization registers */ 934 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 935 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 936 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 937 938 /* write all 4 ring address registers */ 939 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 940 offsetof(struct hifn_dma, cmdr[0])); 941 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 942 offsetof(struct hifn_dma, srcr[0])); 943 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 944 offsetof(struct hifn_dma, dstr[0])); 945 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 946 offsetof(struct hifn_dma, resr[0])); 947 948 DELAY(2000); 949 950 /* write status register */ 951 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 952 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 953 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 954 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 955 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 956 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 957 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 958 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 959 HIFN_DMACSR_S_WAIT | 960 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 961 HIFN_DMACSR_C_WAIT | 962 HIFN_DMACSR_ENGINE | 963 ((sc->sc_flags & HIFN_HAS_PUBLIC) ? 964 HIFN_DMACSR_PUBDONE : 0) | 965 ((sc->sc_flags & HIFN_IS_7811) ? 966 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0)); 967 968 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0; 969 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 970 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 971 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 972 HIFN_DMAIER_ENGINE | 973 ((sc->sc_flags & HIFN_IS_7811) ? 974 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0); 975 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 976 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 977 CLR_LED(sc, HIFN_MIPSRST_LED0 | HIFN_MIPSRST_LED1 | HIFN_MIPSRST_LED2); 978 979 if (sc->sc_flags & HIFN_IS_7956) { 980 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 981 HIFN_PUCNFG_TCALLPHASES | 982 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32); 983 WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956); 984 } else { 985 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 986 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 987 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 988 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM)); 989 } 990 991 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 992 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 993 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 994 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 995 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 996 } 997 998 /* 999 * The maximum number of sessions supported by the card 1000 * is dependent on the amount of context ram, which 1001 * encryption algorithms are enabled, and how compression 1002 * is configured. This should be configured before this 1003 * routine is called. 1004 */ 1005 static void 1006 hifn_sessions(struct hifn_softc *sc) 1007 { 1008 uint32_t pucnfg; 1009 int ctxsize; 1010 1011 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG); 1012 1013 if (pucnfg & HIFN_PUCNFG_COMPSING) { 1014 if (pucnfg & HIFN_PUCNFG_ENCCNFG) 1015 ctxsize = 128; 1016 else 1017 ctxsize = 512; 1018 /* 1019 * 7955/7956 has internal context memory of 32K 1020 */ 1021 if (sc->sc_flags & HIFN_IS_7956) 1022 sc->sc_maxses = 32768 / ctxsize; 1023 else 1024 sc->sc_maxses = 1 + 1025 ((sc->sc_ramsize - 32768) / ctxsize); 1026 } else 1027 sc->sc_maxses = sc->sc_ramsize / 16384; 1028 1029 if (sc->sc_maxses > 2048) 1030 sc->sc_maxses = 2048; 1031 } 1032 1033 /* 1034 * Determine ram type (sram or dram). Board should be just out of a reset 1035 * state when this is called. 1036 */ 1037 static int 1038 hifn_ramtype(struct hifn_softc *sc) 1039 { 1040 uint8_t data[8], dataexpect[8]; 1041 size_t i; 1042 1043 for (i = 0; i < sizeof(data); i++) 1044 data[i] = dataexpect[i] = 0x55; 1045 if (hifn_writeramaddr(sc, 0, data)) 1046 return (-1); 1047 if (hifn_readramaddr(sc, 0, data)) 1048 return (-1); 1049 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1050 sc->sc_drammodel = 1; 1051 return (0); 1052 } 1053 1054 for (i = 0; i < sizeof(data); i++) 1055 data[i] = dataexpect[i] = 0xaa; 1056 if (hifn_writeramaddr(sc, 0, data)) 1057 return (-1); 1058 if (hifn_readramaddr(sc, 0, data)) 1059 return (-1); 1060 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1061 sc->sc_drammodel = 1; 1062 return (0); 1063 } 1064 1065 return (0); 1066 } 1067 1068 #define HIFN_SRAM_MAX (32 << 20) 1069 #define HIFN_SRAM_STEP_SIZE 16384 1070 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE) 1071 1072 static int 1073 hifn_sramsize(struct hifn_softc *sc) 1074 { 1075 uint32_t a, b; 1076 uint8_t data[8]; 1077 uint8_t dataexpect[sizeof(data)]; 1078 size_t i; 1079 1080 for (i = 0; i < sizeof(data); i++) 1081 data[i] = dataexpect[i] = i ^ 0x5a; 1082 1083 a = HIFN_SRAM_GRANULARITY * HIFN_SRAM_STEP_SIZE; 1084 b = HIFN_SRAM_GRANULARITY; 1085 for (i = 0; i < HIFN_SRAM_GRANULARITY; ++i) { 1086 a -= HIFN_SRAM_STEP_SIZE; 1087 b -= 1; 1088 le32enc(data, b); 1089 hifn_writeramaddr(sc, a, data); 1090 } 1091 1092 a = 0; 1093 b = 0; 1094 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) { 1095 le32enc(dataexpect, b); 1096 if (hifn_readramaddr(sc, a, data) < 0) 1097 return (0); 1098 if (memcmp(data, dataexpect, sizeof(data)) != 0) 1099 return (0); 1100 1101 a += HIFN_SRAM_STEP_SIZE; 1102 b += 1; 1103 sc->sc_ramsize = a; 1104 } 1105 1106 return (0); 1107 } 1108 1109 /* 1110 * XXX For dram boards, one should really try all of the 1111 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG 1112 * is already set up correctly. 1113 */ 1114 static int 1115 hifn_dramsize(struct hifn_softc *sc) 1116 { 1117 uint32_t cnfg; 1118 1119 if (sc->sc_flags & HIFN_IS_7956) { 1120 /* 1121 * 7955/7956 have a fixed internal ram of only 32K. 1122 */ 1123 sc->sc_ramsize = 32768; 1124 } else { 1125 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) & 1126 HIFN_PUCNFG_DRAMMASK; 1127 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18); 1128 } 1129 return (0); 1130 } 1131 1132 static void 1133 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, 1134 int *resp) 1135 { 1136 struct hifn_dma *dma = sc->sc_dma; 1137 1138 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1139 dma->cmdi = 0; 1140 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1141 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1142 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1143 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1144 } 1145 *cmdp = dma->cmdi++; 1146 dma->cmdk = dma->cmdi; 1147 1148 if (dma->srci == HIFN_D_SRC_RSIZE) { 1149 dma->srci = 0; 1150 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID | 1151 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1152 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1153 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1154 } 1155 *srcp = dma->srci++; 1156 dma->srck = dma->srci; 1157 1158 if (dma->dsti == HIFN_D_DST_RSIZE) { 1159 dma->dsti = 0; 1160 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID | 1161 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1162 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE, 1163 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1164 } 1165 *dstp = dma->dsti++; 1166 dma->dstk = dma->dsti; 1167 1168 if (dma->resi == HIFN_D_RES_RSIZE) { 1169 dma->resi = 0; 1170 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1171 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1172 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1173 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1174 } 1175 *resp = dma->resi++; 1176 dma->resk = dma->resi; 1177 } 1178 1179 static int 1180 hifn_writeramaddr(struct hifn_softc *sc, int addr, uint8_t *data) 1181 { 1182 struct hifn_dma *dma = sc->sc_dma; 1183 struct hifn_base_command wc; 1184 const uint32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1185 int r, cmdi, resi, srci, dsti; 1186 1187 wc.masks = htole16(3 << 13); 1188 wc.session_num = htole16(addr >> 14); 1189 wc.total_source_count = htole16(8); 1190 wc.total_dest_count = htole16(addr & 0x3fff); 1191 1192 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1193 1194 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1195 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1196 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1197 1198 /* build write command */ 1199 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1200 *(struct hifn_base_command *)dma->command_bufs[cmdi] = wc; 1201 memcpy(&dma->test_src, data, sizeof(dma->test_src)); 1202 1203 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1204 + offsetof(struct hifn_dma, test_src)); 1205 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1206 + offsetof(struct hifn_dma, test_dst)); 1207 1208 dma->cmdr[cmdi].l = htole32(16 | masks); 1209 dma->srcr[srci].l = htole32(8 | masks); 1210 dma->dstr[dsti].l = htole32(4 | masks); 1211 dma->resr[resi].l = htole32(4 | masks); 1212 1213 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1214 0, sc->sc_dmamap->dm_mapsize, 1215 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1216 1217 for (r = 10000; r >= 0; r--) { 1218 DELAY(10); 1219 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1220 0, sc->sc_dmamap->dm_mapsize, 1221 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1222 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1223 break; 1224 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1225 0, sc->sc_dmamap->dm_mapsize, 1226 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1227 } 1228 if (r == 0) { 1229 printf("%s: writeramaddr -- " 1230 "result[%d](addr %d) still valid\n", 1231 device_xname(sc->sc_dv), resi, addr); 1232 r = -1; 1233 return (-1); 1234 } else 1235 r = 0; 1236 1237 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1238 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1239 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1240 1241 return (r); 1242 } 1243 1244 static int 1245 hifn_readramaddr(struct hifn_softc *sc, int addr, uint8_t *data) 1246 { 1247 struct hifn_dma *dma = sc->sc_dma; 1248 struct hifn_base_command rc; 1249 const uint32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1250 int r, cmdi, srci, dsti, resi; 1251 1252 rc.masks = htole16(2 << 13); 1253 rc.session_num = htole16(addr >> 14); 1254 rc.total_source_count = htole16(addr & 0x3fff); 1255 rc.total_dest_count = htole16(8); 1256 1257 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1258 1259 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1260 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1261 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1262 1263 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1264 *(struct hifn_base_command *)dma->command_bufs[cmdi] = rc; 1265 1266 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1267 offsetof(struct hifn_dma, test_src)); 1268 dma->test_src = 0; 1269 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1270 offsetof(struct hifn_dma, test_dst)); 1271 dma->test_dst = 0; 1272 dma->cmdr[cmdi].l = htole32(8 | masks); 1273 dma->srcr[srci].l = htole32(8 | masks); 1274 dma->dstr[dsti].l = htole32(8 | masks); 1275 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks); 1276 1277 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1278 0, sc->sc_dmamap->dm_mapsize, 1279 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1280 1281 for (r = 10000; r >= 0; r--) { 1282 DELAY(10); 1283 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1284 0, sc->sc_dmamap->dm_mapsize, 1285 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1286 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1287 break; 1288 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1289 0, sc->sc_dmamap->dm_mapsize, 1290 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1291 } 1292 if (r == 0) { 1293 printf("%s: readramaddr -- " 1294 "result[%d](addr %d) still valid\n", 1295 device_xname(sc->sc_dv), resi, addr); 1296 r = -1; 1297 } else { 1298 r = 0; 1299 memcpy(data, &dma->test_dst, sizeof(dma->test_dst)); 1300 } 1301 1302 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1303 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1304 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1305 1306 return (r); 1307 } 1308 1309 /* 1310 * Initialize the descriptor rings. 1311 */ 1312 static void 1313 hifn_init_dma(struct hifn_softc *sc) 1314 { 1315 struct hifn_dma *dma = sc->sc_dma; 1316 int i; 1317 1318 hifn_set_retry(sc); 1319 1320 /* initialize static pointer values */ 1321 for (i = 0; i < HIFN_D_CMD_RSIZE; i++) 1322 dma->cmdr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1323 offsetof(struct hifn_dma, command_bufs[i][0])); 1324 for (i = 0; i < HIFN_D_RES_RSIZE; i++) 1325 dma->resr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1326 offsetof(struct hifn_dma, result_bufs[i][0])); 1327 1328 dma->cmdr[HIFN_D_CMD_RSIZE].p = 1329 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1330 offsetof(struct hifn_dma, cmdr[0])); 1331 dma->srcr[HIFN_D_SRC_RSIZE].p = 1332 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1333 offsetof(struct hifn_dma, srcr[0])); 1334 dma->dstr[HIFN_D_DST_RSIZE].p = 1335 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1336 offsetof(struct hifn_dma, dstr[0])); 1337 dma->resr[HIFN_D_RES_RSIZE].p = 1338 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1339 offsetof(struct hifn_dma, resr[0])); 1340 1341 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 1342 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 1343 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 1344 } 1345 1346 /* 1347 * Writes out the raw command buffer space. Returns the 1348 * command buffer size. 1349 */ 1350 static u_int 1351 hifn_write_command(struct hifn_command *cmd, uint8_t *buf) 1352 { 1353 uint8_t *buf_pos; 1354 struct hifn_base_command *base_cmd; 1355 struct hifn_mac_command *mac_cmd; 1356 struct hifn_crypt_command *cry_cmd; 1357 struct hifn_comp_command *comp_cmd; 1358 int using_mac, using_crypt, using_comp, len, ivlen; 1359 uint32_t dlen, slen; 1360 1361 buf_pos = buf; 1362 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC; 1363 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT; 1364 using_comp = cmd->base_masks & HIFN_BASE_CMD_COMP; 1365 1366 base_cmd = (struct hifn_base_command *)buf_pos; 1367 base_cmd->masks = htole16(cmd->base_masks); 1368 slen = cmd->src_map->dm_mapsize; 1369 if (cmd->sloplen) 1370 dlen = cmd->dst_map->dm_mapsize - cmd->sloplen + 1371 sizeof(uint32_t); 1372 else 1373 dlen = cmd->dst_map->dm_mapsize; 1374 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO); 1375 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1376 dlen >>= 16; 1377 slen >>= 16; 1378 base_cmd->session_num = htole16(cmd->session_num | 1379 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1380 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1381 buf_pos += sizeof(struct hifn_base_command); 1382 1383 if (using_comp) { 1384 comp_cmd = (struct hifn_comp_command *)buf_pos; 1385 dlen = cmd->compcrd->crd_len; 1386 comp_cmd->source_count = htole16(dlen & 0xffff); 1387 dlen >>= 16; 1388 comp_cmd->masks = htole16(cmd->comp_masks | 1389 ((dlen << HIFN_COMP_CMD_SRCLEN_S) & HIFN_COMP_CMD_SRCLEN_M)); 1390 comp_cmd->header_skip = htole16(cmd->compcrd->crd_skip); 1391 comp_cmd->reserved = 0; 1392 buf_pos += sizeof(struct hifn_comp_command); 1393 } 1394 1395 if (using_mac) { 1396 mac_cmd = (struct hifn_mac_command *)buf_pos; 1397 dlen = cmd->maccrd->crd_len; 1398 mac_cmd->source_count = htole16(dlen & 0xffff); 1399 dlen >>= 16; 1400 mac_cmd->masks = htole16(cmd->mac_masks | 1401 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M)); 1402 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip); 1403 mac_cmd->reserved = 0; 1404 buf_pos += sizeof(struct hifn_mac_command); 1405 } 1406 1407 if (using_crypt) { 1408 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1409 dlen = cmd->enccrd->crd_len; 1410 cry_cmd->source_count = htole16(dlen & 0xffff); 1411 dlen >>= 16; 1412 cry_cmd->masks = htole16(cmd->cry_masks | 1413 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M)); 1414 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip); 1415 cry_cmd->reserved = 0; 1416 buf_pos += sizeof(struct hifn_crypt_command); 1417 } 1418 1419 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) { 1420 memcpy(buf_pos, cmd->mac, HIFN_MAC_KEY_LENGTH); 1421 buf_pos += HIFN_MAC_KEY_LENGTH; 1422 } 1423 1424 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) { 1425 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1426 case HIFN_CRYPT_CMD_ALG_3DES: 1427 memcpy(buf_pos, cmd->ck, HIFN_3DES_KEY_LENGTH); 1428 buf_pos += HIFN_3DES_KEY_LENGTH; 1429 break; 1430 case HIFN_CRYPT_CMD_ALG_DES: 1431 memcpy(buf_pos, cmd->ck, HIFN_DES_KEY_LENGTH); 1432 buf_pos += HIFN_DES_KEY_LENGTH; 1433 break; 1434 case HIFN_CRYPT_CMD_ALG_RC4: 1435 len = 256; 1436 do { 1437 int clen; 1438 1439 clen = MIN(cmd->cklen, len); 1440 memcpy(buf_pos, cmd->ck, clen); 1441 len -= clen; 1442 buf_pos += clen; 1443 } while (len > 0); 1444 memset(buf_pos, 0, 4); 1445 buf_pos += 4; 1446 break; 1447 case HIFN_CRYPT_CMD_ALG_AES: 1448 /* 1449 * AES keys are variable 128, 192 and 1450 * 256 bits (16, 24 and 32 bytes). 1451 */ 1452 memcpy(buf_pos, cmd->ck, cmd->cklen); 1453 buf_pos += cmd->cklen; 1454 break; 1455 } 1456 } 1457 1458 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) { 1459 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1460 case HIFN_CRYPT_CMD_ALG_AES: 1461 ivlen = HIFN_AES_IV_LENGTH; 1462 break; 1463 default: 1464 ivlen = HIFN_IV_LENGTH; 1465 break; 1466 } 1467 memcpy(buf_pos, cmd->iv, ivlen); 1468 buf_pos += ivlen; 1469 } 1470 1471 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC | HIFN_BASE_CMD_CRYPT | 1472 HIFN_BASE_CMD_COMP)) == 0) { 1473 memset(buf_pos, 0, 8); 1474 buf_pos += 8; 1475 } 1476 1477 return (buf_pos - buf); 1478 } 1479 1480 static int 1481 hifn_dmamap_aligned(bus_dmamap_t map) 1482 { 1483 int i; 1484 1485 for (i = 0; i < map->dm_nsegs; i++) { 1486 if (map->dm_segs[i].ds_addr & 3) 1487 return (0); 1488 if ((i != (map->dm_nsegs - 1)) && 1489 (map->dm_segs[i].ds_len & 3)) 1490 return (0); 1491 } 1492 return (1); 1493 } 1494 1495 static int 1496 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd) 1497 { 1498 struct hifn_dma *dma = sc->sc_dma; 1499 bus_dmamap_t map = cmd->dst_map; 1500 uint32_t p, l; 1501 int idx, used = 0, i; 1502 1503 idx = dma->dsti; 1504 for (i = 0; i < map->dm_nsegs - 1; i++) { 1505 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1506 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1507 HIFN_D_MASKDONEIRQ | map->dm_segs[i].ds_len); 1508 HIFN_DSTR_SYNC(sc, idx, 1509 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1510 used++; 1511 1512 if (++idx == HIFN_D_DST_RSIZE) { 1513 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1514 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1515 HIFN_DSTR_SYNC(sc, idx, 1516 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1517 idx = 0; 1518 } 1519 } 1520 1521 if (cmd->sloplen == 0) { 1522 p = map->dm_segs[i].ds_addr; 1523 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1524 map->dm_segs[i].ds_len; 1525 } else { 1526 p = sc->sc_dmamap->dm_segs[0].ds_addr + 1527 offsetof(struct hifn_dma, slop[cmd->slopidx]); 1528 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1529 sizeof(uint32_t); 1530 1531 if ((map->dm_segs[i].ds_len - cmd->sloplen) != 0) { 1532 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1533 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1534 HIFN_D_MASKDONEIRQ | 1535 (map->dm_segs[i].ds_len - cmd->sloplen)); 1536 HIFN_DSTR_SYNC(sc, idx, 1537 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1538 used++; 1539 1540 if (++idx == HIFN_D_DST_RSIZE) { 1541 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1542 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1543 HIFN_DSTR_SYNC(sc, idx, 1544 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1545 idx = 0; 1546 } 1547 } 1548 } 1549 dma->dstr[idx].p = htole32(p); 1550 dma->dstr[idx].l = htole32(l); 1551 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1552 used++; 1553 1554 if (++idx == HIFN_D_DST_RSIZE) { 1555 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP | 1556 HIFN_D_MASKDONEIRQ); 1557 HIFN_DSTR_SYNC(sc, idx, 1558 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1559 idx = 0; 1560 } 1561 1562 dma->dsti = idx; 1563 dma->dstu += used; 1564 return (idx); 1565 } 1566 1567 static int 1568 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd) 1569 { 1570 struct hifn_dma *dma = sc->sc_dma; 1571 bus_dmamap_t map = cmd->src_map; 1572 int idx, i; 1573 uint32_t last = 0; 1574 1575 idx = dma->srci; 1576 for (i = 0; i < map->dm_nsegs; i++) { 1577 if (i == map->dm_nsegs - 1) 1578 last = HIFN_D_LAST; 1579 1580 dma->srcr[idx].p = htole32(map->dm_segs[i].ds_addr); 1581 dma->srcr[idx].l = htole32(map->dm_segs[i].ds_len | 1582 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last); 1583 HIFN_SRCR_SYNC(sc, idx, 1584 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1585 1586 if (++idx == HIFN_D_SRC_RSIZE) { 1587 dma->srcr[idx].l = htole32(HIFN_D_VALID | 1588 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1589 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1590 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1591 idx = 0; 1592 } 1593 } 1594 dma->srci = idx; 1595 dma->srcu += map->dm_nsegs; 1596 return (idx); 1597 } 1598 1599 static int 1600 hifn_crypto(struct hifn_softc *sc, struct hifn_command *cmd, 1601 struct cryptop *crp, int hint) 1602 { 1603 struct hifn_dma *dma = sc->sc_dma; 1604 uint32_t cmdlen; 1605 int cmdi, resi, err = 0; 1606 1607 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 1608 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) 1609 return (ENOMEM); 1610 1611 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1612 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 1613 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 1614 err = ENOMEM; 1615 goto err_srcmap1; 1616 } 1617 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1618 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 1619 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 1620 err = ENOMEM; 1621 goto err_srcmap1; 1622 } 1623 } else { 1624 err = EINVAL; 1625 goto err_srcmap1; 1626 } 1627 1628 if (hifn_dmamap_aligned(cmd->src_map)) { 1629 cmd->sloplen = cmd->src_map->dm_mapsize & 3; 1630 if (crp->crp_flags & CRYPTO_F_IOV) 1631 cmd->dstu.dst_io = cmd->srcu.src_io; 1632 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1633 cmd->dstu.dst_m = cmd->srcu.src_m; 1634 cmd->dst_map = cmd->src_map; 1635 } else { 1636 if (crp->crp_flags & CRYPTO_F_IOV) { 1637 err = EINVAL; 1638 goto err_srcmap; 1639 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1640 int totlen, len; 1641 struct mbuf *m, *m0, *mlast; 1642 1643 totlen = cmd->src_map->dm_mapsize; 1644 if (cmd->srcu.src_m->m_flags & M_PKTHDR) { 1645 len = MHLEN; 1646 MGETHDR(m0, M_DONTWAIT, MT_DATA); 1647 } else { 1648 len = MLEN; 1649 MGET(m0, M_DONTWAIT, MT_DATA); 1650 } 1651 if (m0 == NULL) { 1652 err = ENOMEM; 1653 goto err_srcmap; 1654 } 1655 if (len == MHLEN) 1656 m_copy_pkthdr(m0, cmd->srcu.src_m); 1657 if (totlen >= MINCLSIZE) { 1658 MCLGET(m0, M_DONTWAIT); 1659 if (m0->m_flags & M_EXT) 1660 len = MCLBYTES; 1661 } 1662 totlen -= len; 1663 m0->m_pkthdr.len = m0->m_len = len; 1664 mlast = m0; 1665 1666 while (totlen > 0) { 1667 MGET(m, M_DONTWAIT, MT_DATA); 1668 if (m == NULL) { 1669 err = ENOMEM; 1670 m_freem(m0); 1671 goto err_srcmap; 1672 } 1673 len = MLEN; 1674 if (totlen >= MINCLSIZE) { 1675 MCLGET(m, M_DONTWAIT); 1676 if (m->m_flags & M_EXT) 1677 len = MCLBYTES; 1678 } 1679 1680 m->m_len = len; 1681 if (m0->m_flags & M_PKTHDR) 1682 m0->m_pkthdr.len += len; 1683 totlen -= len; 1684 1685 mlast->m_next = m; 1686 mlast = m; 1687 } 1688 cmd->dstu.dst_m = m0; 1689 } 1690 } 1691 1692 if (cmd->dst_map == NULL) { 1693 if (bus_dmamap_create(sc->sc_dmat, 1694 HIFN_MAX_SEGLEN * MAX_SCATTER, MAX_SCATTER, 1695 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 1696 err = ENOMEM; 1697 goto err_srcmap; 1698 } 1699 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1700 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 1701 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 1702 err = ENOMEM; 1703 goto err_dstmap1; 1704 } 1705 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1706 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 1707 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 1708 err = ENOMEM; 1709 goto err_dstmap1; 1710 } 1711 } 1712 } 1713 1714 #ifdef HIFN_DEBUG 1715 if (hifn_debug) 1716 printf("%s: Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n", 1717 device_xname(sc->sc_dv), 1718 READ_REG_1(sc, HIFN_1_DMA_CSR), 1719 READ_REG_1(sc, HIFN_1_DMA_IER), 1720 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1721 cmd->src_map->dm_nsegs, cmd->dst_map->dm_nsegs); 1722 #endif 1723 1724 if (cmd->src_map == cmd->dst_map) 1725 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1726 0, cmd->src_map->dm_mapsize, 1727 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 1728 else { 1729 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1730 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1731 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 1732 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 1733 } 1734 1735 /* 1736 * need 1 cmd, and 1 res 1737 * need N src, and N dst 1738 */ 1739 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 1740 (dma->resu + 1) > HIFN_D_RES_RSIZE) { 1741 err = ENOMEM; 1742 goto err_dstmap; 1743 } 1744 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 1745 (dma->dstu + cmd->dst_map->dm_nsegs + 1) > HIFN_D_DST_RSIZE) { 1746 err = ENOMEM; 1747 goto err_dstmap; 1748 } 1749 1750 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1751 dma->cmdi = 0; 1752 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1753 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1754 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1755 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1756 } 1757 cmdi = dma->cmdi++; 1758 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 1759 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 1760 1761 /* .p for command/result already set */ 1762 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 1763 HIFN_D_MASKDONEIRQ); 1764 HIFN_CMDR_SYNC(sc, cmdi, 1765 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1766 dma->cmdu++; 1767 if (sc->sc_c_busy == 0) { 1768 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1769 sc->sc_c_busy = 1; 1770 SET_LED(sc, HIFN_MIPSRST_LED0); 1771 } 1772 1773 /* 1774 * Always enable the command wait interrupt. We are obviously 1775 * missing an interrupt or two somewhere. Enabling the command wait 1776 * interrupt will guarantee we get called periodically until all 1777 * of the queues are drained and thus work around this. 1778 */ 1779 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 1780 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1781 1782 hifnstats.hst_ipackets++; 1783 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 1784 1785 hifn_dmamap_load_src(sc, cmd); 1786 if (sc->sc_s_busy == 0) { 1787 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1788 sc->sc_s_busy = 1; 1789 SET_LED(sc, HIFN_MIPSRST_LED1); 1790 } 1791 1792 /* 1793 * Unlike other descriptors, we don't mask done interrupt from 1794 * result descriptor. 1795 */ 1796 #ifdef HIFN_DEBUG 1797 if (hifn_debug) 1798 printf("load res\n"); 1799 #endif 1800 if (dma->resi == HIFN_D_RES_RSIZE) { 1801 dma->resi = 0; 1802 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1803 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1804 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1805 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1806 } 1807 resi = dma->resi++; 1808 dma->hifn_commands[resi] = cmd; 1809 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 1810 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 1811 HIFN_D_VALID | HIFN_D_LAST); 1812 HIFN_RESR_SYNC(sc, resi, 1813 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1814 dma->resu++; 1815 if (sc->sc_r_busy == 0) { 1816 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1817 sc->sc_r_busy = 1; 1818 SET_LED(sc, HIFN_MIPSRST_LED2); 1819 } 1820 1821 if (cmd->sloplen) 1822 cmd->slopidx = resi; 1823 1824 hifn_dmamap_load_dst(sc, cmd); 1825 1826 if (sc->sc_d_busy == 0) { 1827 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1828 sc->sc_d_busy = 1; 1829 } 1830 1831 #ifdef HIFN_DEBUG 1832 if (hifn_debug) 1833 printf("%s: command: stat %8x ier %8x\n", 1834 device_xname(sc->sc_dv), 1835 READ_REG_1(sc, HIFN_1_DMA_CSR), 1836 READ_REG_1(sc, HIFN_1_DMA_IER)); 1837 #endif 1838 1839 sc->sc_active = 5; 1840 return (err); /* success */ 1841 1842 err_dstmap: 1843 if (cmd->src_map != cmd->dst_map) 1844 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 1845 err_dstmap1: 1846 if (cmd->src_map != cmd->dst_map) 1847 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 1848 err_srcmap: 1849 if (crp->crp_flags & CRYPTO_F_IMBUF && 1850 cmd->srcu.src_m != cmd->dstu.dst_m) 1851 m_freem(cmd->dstu.dst_m); 1852 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 1853 err_srcmap1: 1854 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 1855 return (err); 1856 } 1857 1858 static void 1859 hifn_tick(void *vsc) 1860 { 1861 struct hifn_softc *sc = vsc; 1862 1863 mutex_spin_enter(&sc->sc_mtx); 1864 if (sc->sc_active == 0) { 1865 struct hifn_dma *dma = sc->sc_dma; 1866 uint32_t r = 0; 1867 1868 if (dma->cmdu == 0 && sc->sc_c_busy) { 1869 sc->sc_c_busy = 0; 1870 r |= HIFN_DMACSR_C_CTRL_DIS; 1871 CLR_LED(sc, HIFN_MIPSRST_LED0); 1872 } 1873 if (dma->srcu == 0 && sc->sc_s_busy) { 1874 sc->sc_s_busy = 0; 1875 r |= HIFN_DMACSR_S_CTRL_DIS; 1876 CLR_LED(sc, HIFN_MIPSRST_LED1); 1877 } 1878 if (dma->dstu == 0 && sc->sc_d_busy) { 1879 sc->sc_d_busy = 0; 1880 r |= HIFN_DMACSR_D_CTRL_DIS; 1881 } 1882 if (dma->resu == 0 && sc->sc_r_busy) { 1883 sc->sc_r_busy = 0; 1884 r |= HIFN_DMACSR_R_CTRL_DIS; 1885 CLR_LED(sc, HIFN_MIPSRST_LED2); 1886 } 1887 if (r) 1888 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r); 1889 } else 1890 sc->sc_active--; 1891 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 1892 mutex_spin_exit(&sc->sc_mtx); 1893 } 1894 1895 static int 1896 hifn_intr(void *arg) 1897 { 1898 struct hifn_softc *sc = arg; 1899 struct hifn_dma *dma = sc->sc_dma; 1900 uint32_t dmacsr, restart; 1901 int i, u; 1902 1903 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR); 1904 1905 #ifdef HIFN_DEBUG 1906 if (hifn_debug) 1907 printf("%s: irq: stat %08x ien %08x u %d/%d/%d/%d\n", 1908 device_xname(sc->sc_dv), 1909 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), 1910 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1911 #endif 1912 1913 mutex_spin_enter(&sc->sc_mtx); 1914 1915 /* Nothing in the DMA unit interrupted */ 1916 if ((dmacsr & sc->sc_dmaier) == 0) { 1917 mutex_spin_exit(&sc->sc_mtx); 1918 return (0); 1919 } 1920 1921 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier); 1922 1923 if (dmacsr & HIFN_DMACSR_ENGINE) 1924 WRITE_REG_0(sc, HIFN_0_PUISR, READ_REG_0(sc, HIFN_0_PUISR)); 1925 1926 if ((sc->sc_flags & HIFN_HAS_PUBLIC) && 1927 (dmacsr & HIFN_DMACSR_PUBDONE)) 1928 WRITE_REG_1(sc, HIFN_1_PUB_STATUS, 1929 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1930 1931 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1932 if (restart) 1933 printf("%s: overrun %x\n", device_xname(sc->sc_dv), dmacsr); 1934 1935 if (sc->sc_flags & HIFN_IS_7811) { 1936 if (dmacsr & HIFN_DMACSR_ILLR) 1937 printf("%s: illegal read\n", device_xname(sc->sc_dv)); 1938 if (dmacsr & HIFN_DMACSR_ILLW) 1939 printf("%s: illegal write\n", device_xname(sc->sc_dv)); 1940 } 1941 1942 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1943 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1944 if (restart) { 1945 printf("%s: abort, resetting.\n", device_xname(sc->sc_dv)); 1946 hifnstats.hst_abort++; 1947 hifn_abort(sc); 1948 goto out; 1949 } 1950 1951 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->resu == 0)) { 1952 /* 1953 * If no slots to process and we receive a "waiting on 1954 * command" interrupt, we disable the "waiting on command" 1955 * (by clearing it). 1956 */ 1957 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 1958 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1959 } 1960 1961 /* clear the rings */ 1962 i = dma->resk; 1963 while (dma->resu != 0) { 1964 HIFN_RESR_SYNC(sc, i, 1965 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1966 if (dma->resr[i].l & htole32(HIFN_D_VALID)) { 1967 HIFN_RESR_SYNC(sc, i, 1968 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1969 break; 1970 } 1971 1972 if (i != HIFN_D_RES_RSIZE) { 1973 struct hifn_command *cmd; 1974 1975 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD); 1976 cmd = dma->hifn_commands[i]; 1977 KASSERT(cmd != NULL 1978 /*("hifn_intr: null command slot %u", i)*/); 1979 dma->hifn_commands[i] = NULL; 1980 1981 hifn_callback(sc, cmd, dma->result_bufs[i]); 1982 hifnstats.hst_opackets++; 1983 } 1984 1985 if (++i == (HIFN_D_RES_RSIZE + 1)) 1986 i = 0; 1987 else 1988 dma->resu--; 1989 } 1990 dma->resk = i; 1991 1992 i = dma->srck; u = dma->srcu; 1993 while (u != 0) { 1994 HIFN_SRCR_SYNC(sc, i, 1995 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1996 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) { 1997 HIFN_SRCR_SYNC(sc, i, 1998 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1999 break; 2000 } 2001 if (++i == (HIFN_D_SRC_RSIZE + 1)) 2002 i = 0; 2003 else 2004 u--; 2005 } 2006 dma->srck = i; dma->srcu = u; 2007 2008 i = dma->cmdk; u = dma->cmdu; 2009 while (u != 0) { 2010 HIFN_CMDR_SYNC(sc, i, 2011 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2012 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) { 2013 HIFN_CMDR_SYNC(sc, i, 2014 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2015 break; 2016 } 2017 if (i != HIFN_D_CMD_RSIZE) { 2018 u--; 2019 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE); 2020 } 2021 if (++i == (HIFN_D_CMD_RSIZE + 1)) 2022 i = 0; 2023 } 2024 dma->cmdk = i; dma->cmdu = u; 2025 2026 out: 2027 mutex_spin_exit(&sc->sc_mtx); 2028 return (1); 2029 } 2030 2031 /* 2032 * Allocate a new 'session' and return an encoded session id. 'sidp' 2033 * contains our registration id, and should contain an encoded session 2034 * id on successful allocation. 2035 */ 2036 static int 2037 hifn_newsession(void *arg, uint32_t *sidp, struct cryptoini *cri) 2038 { 2039 struct cryptoini *c; 2040 struct hifn_softc *sc = arg; 2041 int i, mac = 0, cry = 0, comp = 0, retval = EINVAL; 2042 2043 KASSERT(sc != NULL /*, ("hifn_newsession: null softc")*/); 2044 if (sidp == NULL || cri == NULL || sc == NULL) 2045 return retval; 2046 2047 mutex_spin_enter(&sc->sc_mtx); 2048 2049 for (i = 0; i < sc->sc_maxses; i++) 2050 if (sc->sc_sessions[i].hs_state == HS_STATE_FREE) 2051 break; 2052 if (i == sc->sc_maxses) { 2053 retval = ENOMEM; 2054 goto out; 2055 } 2056 2057 for (c = cri; c != NULL; c = c->cri_next) { 2058 switch (c->cri_alg) { 2059 case CRYPTO_MD5: 2060 case CRYPTO_SHA1: 2061 case CRYPTO_MD5_HMAC_96: 2062 case CRYPTO_SHA1_HMAC_96: 2063 if (mac) { 2064 goto out; 2065 } 2066 mac = 1; 2067 break; 2068 case CRYPTO_DES_CBC: 2069 case CRYPTO_3DES_CBC: 2070 case CRYPTO_AES_CBC: 2071 /* Note that this is an initialization 2072 vector, not a cipher key; any function 2073 giving sufficient Hamming distance 2074 between outputs is fine. Use of RC4 2075 to generate IVs has been FIPS140-2 2076 certified by several labs. */ 2077 cprng_fast(sc->sc_sessions[i].hs_iv, 2078 c->cri_alg == CRYPTO_AES_CBC ? 2079 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2080 /*FALLTHROUGH*/ 2081 case CRYPTO_ARC4: 2082 if (cry) { 2083 goto out; 2084 } 2085 cry = 1; 2086 break; 2087 #ifdef CRYPTO_LZS_COMP 2088 case CRYPTO_LZS_COMP: 2089 if (comp) { 2090 goto out; 2091 } 2092 comp = 1; 2093 break; 2094 #endif 2095 default: 2096 goto out; 2097 } 2098 } 2099 if (mac == 0 && cry == 0 && comp == 0) { 2100 goto out; 2101 } 2102 2103 /* 2104 * XXX only want to support compression without chaining to 2105 * MAC/crypt engine right now 2106 */ 2107 if ((comp && mac) || (comp && cry)) { 2108 goto out; 2109 } 2110 2111 *sidp = HIFN_SID(device_unit(sc->sc_dv), i); 2112 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2113 2114 retval = 0; 2115 out: 2116 mutex_spin_exit(&sc->sc_mtx); 2117 return retval; 2118 } 2119 2120 /* 2121 * Deallocate a session. 2122 * XXX this routine should run a zero'd mac/encrypt key into context ram. 2123 * XXX to blow away any keys already stored there. 2124 */ 2125 static int 2126 hifn_freesession(void *arg, uint64_t tid) 2127 { 2128 struct hifn_softc *sc = arg; 2129 int session; 2130 uint32_t sid = ((uint32_t) tid) & 0xffffffff; 2131 2132 KASSERT(sc != NULL /*, ("hifn_freesession: null softc")*/); 2133 if (sc == NULL) 2134 return (EINVAL); 2135 2136 mutex_spin_enter(&sc->sc_mtx); 2137 session = HIFN_SESSION(sid); 2138 if (session >= sc->sc_maxses) { 2139 mutex_spin_exit(&sc->sc_mtx); 2140 return (EINVAL); 2141 } 2142 2143 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 2144 mutex_spin_exit(&sc->sc_mtx); 2145 return (0); 2146 } 2147 2148 static int 2149 hifn_process(void *arg, struct cryptop *crp, int hint) 2150 { 2151 struct hifn_softc *sc = arg; 2152 struct hifn_command *cmd = NULL; 2153 int session, err, ivlen; 2154 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 2155 2156 if (crp == NULL || crp->crp_callback == NULL) { 2157 hifnstats.hst_invalid++; 2158 return (EINVAL); 2159 } 2160 2161 mutex_spin_enter(&sc->sc_mtx); 2162 session = HIFN_SESSION(crp->crp_sid); 2163 2164 if (sc == NULL || session >= sc->sc_maxses) { 2165 err = EINVAL; 2166 goto errout; 2167 } 2168 2169 cmd = (struct hifn_command *)malloc(sizeof(struct hifn_command), 2170 M_DEVBUF, M_NOWAIT|M_ZERO); 2171 if (cmd == NULL) { 2172 hifnstats.hst_nomem++; 2173 err = ENOMEM; 2174 goto errout; 2175 } 2176 2177 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2178 cmd->srcu.src_m = (struct mbuf *)crp->crp_buf; 2179 cmd->dstu.dst_m = (struct mbuf *)crp->crp_buf; 2180 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2181 cmd->srcu.src_io = (struct uio *)crp->crp_buf; 2182 cmd->dstu.dst_io = (struct uio *)crp->crp_buf; 2183 } else { 2184 err = EINVAL; 2185 goto errout; /* XXX we don't handle contiguous buffers! */ 2186 } 2187 2188 crd1 = crp->crp_desc; 2189 if (crd1 == NULL) { 2190 err = EINVAL; 2191 goto errout; 2192 } 2193 crd2 = crd1->crd_next; 2194 2195 if (crd2 == NULL) { 2196 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2197 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2198 crd1->crd_alg == CRYPTO_SHA1 || 2199 crd1->crd_alg == CRYPTO_MD5) { 2200 maccrd = crd1; 2201 enccrd = NULL; 2202 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 2203 crd1->crd_alg == CRYPTO_3DES_CBC || 2204 crd1->crd_alg == CRYPTO_AES_CBC || 2205 crd1->crd_alg == CRYPTO_ARC4) { 2206 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0) 2207 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2208 maccrd = NULL; 2209 enccrd = crd1; 2210 #ifdef CRYPTO_LZS_COMP 2211 } else if (crd1->crd_alg == CRYPTO_LZS_COMP) { 2212 return (hifn_compression(sc, crp, cmd)); 2213 #endif 2214 } else { 2215 err = EINVAL; 2216 goto errout; 2217 } 2218 } else { 2219 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2220 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2221 crd1->crd_alg == CRYPTO_MD5 || 2222 crd1->crd_alg == CRYPTO_SHA1) && 2223 (crd2->crd_alg == CRYPTO_DES_CBC || 2224 crd2->crd_alg == CRYPTO_3DES_CBC || 2225 crd2->crd_alg == CRYPTO_AES_CBC || 2226 crd2->crd_alg == CRYPTO_ARC4) && 2227 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 2228 cmd->base_masks = HIFN_BASE_CMD_DECODE; 2229 maccrd = crd1; 2230 enccrd = crd2; 2231 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 2232 crd1->crd_alg == CRYPTO_ARC4 || 2233 crd1->crd_alg == CRYPTO_3DES_CBC || 2234 crd1->crd_alg == CRYPTO_AES_CBC) && 2235 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 2236 crd2->crd_alg == CRYPTO_SHA1_HMAC_96 || 2237 crd2->crd_alg == CRYPTO_MD5 || 2238 crd2->crd_alg == CRYPTO_SHA1) && 2239 (crd1->crd_flags & CRD_F_ENCRYPT)) { 2240 enccrd = crd1; 2241 maccrd = crd2; 2242 } else { 2243 /* 2244 * We cannot order the 7751 as requested 2245 */ 2246 err = EINVAL; 2247 goto errout; 2248 } 2249 } 2250 2251 if (enccrd) { 2252 cmd->enccrd = enccrd; 2253 cmd->base_masks |= HIFN_BASE_CMD_CRYPT; 2254 switch (enccrd->crd_alg) { 2255 case CRYPTO_ARC4: 2256 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4; 2257 if ((enccrd->crd_flags & CRD_F_ENCRYPT) 2258 != sc->sc_sessions[session].hs_prev_op) 2259 sc->sc_sessions[session].hs_state = 2260 HS_STATE_USED; 2261 break; 2262 case CRYPTO_DES_CBC: 2263 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES | 2264 HIFN_CRYPT_CMD_MODE_CBC | 2265 HIFN_CRYPT_CMD_NEW_IV; 2266 break; 2267 case CRYPTO_3DES_CBC: 2268 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES | 2269 HIFN_CRYPT_CMD_MODE_CBC | 2270 HIFN_CRYPT_CMD_NEW_IV; 2271 break; 2272 case CRYPTO_AES_CBC: 2273 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES | 2274 HIFN_CRYPT_CMD_MODE_CBC | 2275 HIFN_CRYPT_CMD_NEW_IV; 2276 break; 2277 default: 2278 err = EINVAL; 2279 goto errout; 2280 } 2281 if (enccrd->crd_alg != CRYPTO_ARC4) { 2282 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? 2283 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2284 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 2285 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2286 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2287 else 2288 bcopy(sc->sc_sessions[session].hs_iv, 2289 cmd->iv, ivlen); 2290 2291 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) 2292 == 0) { 2293 if (crp->crp_flags & CRYPTO_F_IMBUF) 2294 m_copyback(cmd->srcu.src_m, 2295 enccrd->crd_inject, 2296 ivlen, cmd->iv); 2297 else if (crp->crp_flags & CRYPTO_F_IOV) 2298 cuio_copyback(cmd->srcu.src_io, 2299 enccrd->crd_inject, 2300 ivlen, cmd->iv); 2301 } 2302 } else { 2303 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2304 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2305 else if (crp->crp_flags & CRYPTO_F_IMBUF) 2306 m_copydata(cmd->srcu.src_m, 2307 enccrd->crd_inject, ivlen, cmd->iv); 2308 else if (crp->crp_flags & CRYPTO_F_IOV) 2309 cuio_copydata(cmd->srcu.src_io, 2310 enccrd->crd_inject, ivlen, cmd->iv); 2311 } 2312 } 2313 2314 cmd->ck = enccrd->crd_key; 2315 cmd->cklen = enccrd->crd_klen >> 3; 2316 2317 /* 2318 * Need to specify the size for the AES key in the masks. 2319 */ 2320 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) == 2321 HIFN_CRYPT_CMD_ALG_AES) { 2322 switch (cmd->cklen) { 2323 case 16: 2324 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128; 2325 break; 2326 case 24: 2327 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192; 2328 break; 2329 case 32: 2330 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256; 2331 break; 2332 default: 2333 err = EINVAL; 2334 goto errout; 2335 } 2336 } 2337 2338 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2339 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; 2340 } 2341 2342 if (maccrd) { 2343 cmd->maccrd = maccrd; 2344 cmd->base_masks |= HIFN_BASE_CMD_MAC; 2345 2346 switch (maccrd->crd_alg) { 2347 case CRYPTO_MD5: 2348 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2349 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2350 HIFN_MAC_CMD_POS_IPSEC; 2351 break; 2352 case CRYPTO_MD5_HMAC_96: 2353 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2354 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2355 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2356 break; 2357 case CRYPTO_SHA1: 2358 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2359 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2360 HIFN_MAC_CMD_POS_IPSEC; 2361 break; 2362 case CRYPTO_SHA1_HMAC_96: 2363 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2364 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2365 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2366 break; 2367 } 2368 2369 if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC_96 || 2370 maccrd->crd_alg == CRYPTO_MD5_HMAC_96) && 2371 sc->sc_sessions[session].hs_state == HS_STATE_USED) { 2372 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY; 2373 memcpy(cmd->mac, maccrd->crd_key, maccrd->crd_klen >> 3); 2374 memset(cmd->mac + (maccrd->crd_klen >> 3), 0, 2375 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3)); 2376 } 2377 } 2378 2379 cmd->crp = crp; 2380 cmd->session_num = session; 2381 cmd->softc = sc; 2382 2383 err = hifn_crypto(sc, cmd, crp, hint); 2384 if (err == 0) { 2385 if (enccrd) 2386 sc->sc_sessions[session].hs_prev_op = 2387 enccrd->crd_flags & CRD_F_ENCRYPT; 2388 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2389 sc->sc_sessions[session].hs_state = HS_STATE_KEY; 2390 mutex_spin_exit(&sc->sc_mtx); 2391 return 0; 2392 } else if (err == ERESTART) { 2393 /* 2394 * There weren't enough resources to dispatch the request 2395 * to the part. Notify the caller so they'll requeue this 2396 * request and resubmit it again soon. 2397 */ 2398 #ifdef HIFN_DEBUG 2399 if (hifn_debug) 2400 printf("%s: requeue request\n", device_xname(sc->sc_dv)); 2401 #endif 2402 free(cmd, M_DEVBUF); 2403 sc->sc_needwakeup |= CRYPTO_SYMQ; 2404 mutex_spin_exit(&sc->sc_mtx); 2405 return (err); 2406 } 2407 2408 errout: 2409 if (cmd != NULL) 2410 free(cmd, M_DEVBUF); 2411 if (err == EINVAL) 2412 hifnstats.hst_invalid++; 2413 else 2414 hifnstats.hst_nomem++; 2415 crp->crp_etype = err; 2416 mutex_spin_exit(&sc->sc_mtx); 2417 crypto_done(crp); 2418 return (0); 2419 } 2420 2421 static void 2422 hifn_abort(struct hifn_softc *sc) 2423 { 2424 struct hifn_dma *dma = sc->sc_dma; 2425 struct hifn_command *cmd; 2426 struct cryptop *crp; 2427 int i, u; 2428 2429 i = dma->resk; u = dma->resu; 2430 while (u != 0) { 2431 cmd = dma->hifn_commands[i]; 2432 KASSERT(cmd != NULL /*, ("hifn_abort: null cmd slot %u", i)*/); 2433 dma->hifn_commands[i] = NULL; 2434 crp = cmd->crp; 2435 2436 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) { 2437 /* Salvage what we can. */ 2438 hifnstats.hst_opackets++; 2439 hifn_callback(sc, cmd, dma->result_bufs[i]); 2440 } else { 2441 if (cmd->src_map == cmd->dst_map) { 2442 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2443 0, cmd->src_map->dm_mapsize, 2444 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2445 } else { 2446 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2447 0, cmd->src_map->dm_mapsize, 2448 BUS_DMASYNC_POSTWRITE); 2449 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2450 0, cmd->dst_map->dm_mapsize, 2451 BUS_DMASYNC_POSTREAD); 2452 } 2453 2454 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2455 m_freem(cmd->srcu.src_m); 2456 crp->crp_buf = (void *)cmd->dstu.dst_m; 2457 } 2458 2459 /* non-shared buffers cannot be restarted */ 2460 if (cmd->src_map != cmd->dst_map) { 2461 /* 2462 * XXX should be EAGAIN, delayed until 2463 * after the reset. 2464 */ 2465 crp->crp_etype = ENOMEM; 2466 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2467 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2468 } else 2469 crp->crp_etype = ENOMEM; 2470 2471 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2472 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2473 2474 free(cmd, M_DEVBUF); 2475 if (crp->crp_etype != EAGAIN) 2476 crypto_done(crp); 2477 } 2478 2479 if (++i == HIFN_D_RES_RSIZE) 2480 i = 0; 2481 u--; 2482 } 2483 dma->resk = i; dma->resu = u; 2484 2485 /* Force upload of key next time */ 2486 for (i = 0; i < sc->sc_maxses; i++) 2487 if (sc->sc_sessions[i].hs_state == HS_STATE_KEY) 2488 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2489 2490 hifn_reset_board(sc, 1); 2491 hifn_init_dma(sc); 2492 hifn_init_pci_registers(sc); 2493 } 2494 2495 static void 2496 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, uint8_t *resbuf) 2497 { 2498 struct hifn_dma *dma = sc->sc_dma; 2499 struct cryptop *crp = cmd->crp; 2500 struct cryptodesc *crd; 2501 struct mbuf *m; 2502 int totlen, i, u, ivlen; 2503 2504 if (cmd->src_map == cmd->dst_map) 2505 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2506 0, cmd->src_map->dm_mapsize, 2507 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2508 else { 2509 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2510 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2511 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2512 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2513 } 2514 2515 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2516 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2517 crp->crp_buf = (void *)cmd->dstu.dst_m; 2518 totlen = cmd->src_map->dm_mapsize; 2519 for (m = cmd->dstu.dst_m; m != NULL; m = m->m_next) { 2520 if (totlen < m->m_len) { 2521 m->m_len = totlen; 2522 totlen = 0; 2523 } else 2524 totlen -= m->m_len; 2525 } 2526 cmd->dstu.dst_m->m_pkthdr.len = 2527 cmd->srcu.src_m->m_pkthdr.len; 2528 m_freem(cmd->srcu.src_m); 2529 } 2530 } 2531 2532 if (cmd->sloplen != 0) { 2533 if (crp->crp_flags & CRYPTO_F_IMBUF) 2534 m_copyback((struct mbuf *)crp->crp_buf, 2535 cmd->src_map->dm_mapsize - cmd->sloplen, 2536 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2537 else if (crp->crp_flags & CRYPTO_F_IOV) 2538 cuio_copyback((struct uio *)crp->crp_buf, 2539 cmd->src_map->dm_mapsize - cmd->sloplen, 2540 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2541 } 2542 2543 i = dma->dstk; u = dma->dstu; 2544 while (u != 0) { 2545 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2546 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2547 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2548 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2549 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2550 offsetof(struct hifn_dma, dstr[i]), 2551 sizeof(struct hifn_desc), 2552 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2553 break; 2554 } 2555 if (++i == (HIFN_D_DST_RSIZE + 1)) 2556 i = 0; 2557 else 2558 u--; 2559 } 2560 dma->dstk = i; dma->dstu = u; 2561 2562 hifnstats.hst_obytes += cmd->dst_map->dm_mapsize; 2563 2564 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) == 2565 HIFN_BASE_CMD_CRYPT) { 2566 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2567 if (crd->crd_alg != CRYPTO_DES_CBC && 2568 crd->crd_alg != CRYPTO_3DES_CBC && 2569 crd->crd_alg != CRYPTO_AES_CBC) 2570 continue; 2571 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? 2572 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2573 if (crp->crp_flags & CRYPTO_F_IMBUF) 2574 m_copydata((struct mbuf *)crp->crp_buf, 2575 crd->crd_skip + crd->crd_len - ivlen, 2576 ivlen, 2577 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2578 else if (crp->crp_flags & CRYPTO_F_IOV) { 2579 cuio_copydata((struct uio *)crp->crp_buf, 2580 crd->crd_skip + crd->crd_len - ivlen, 2581 ivlen, 2582 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2583 } 2584 /* XXX We do not handle contig data */ 2585 break; 2586 } 2587 } 2588 2589 if (cmd->base_masks & HIFN_BASE_CMD_MAC) { 2590 uint8_t *macbuf; 2591 2592 macbuf = resbuf + sizeof(struct hifn_base_result); 2593 if (cmd->base_masks & HIFN_BASE_CMD_COMP) 2594 macbuf += sizeof(struct hifn_comp_result); 2595 macbuf += sizeof(struct hifn_mac_result); 2596 2597 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2598 int len; 2599 2600 if (crd->crd_alg == CRYPTO_MD5) 2601 len = 16; 2602 else if (crd->crd_alg == CRYPTO_SHA1) 2603 len = 20; 2604 else if (crd->crd_alg == CRYPTO_MD5_HMAC_96 || 2605 crd->crd_alg == CRYPTO_SHA1_HMAC_96) 2606 len = 12; 2607 else 2608 continue; 2609 2610 if (crp->crp_flags & CRYPTO_F_IMBUF) 2611 m_copyback((struct mbuf *)crp->crp_buf, 2612 crd->crd_inject, len, macbuf); 2613 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac) 2614 memcpy(crp->crp_mac, (void *)macbuf, len); 2615 break; 2616 } 2617 } 2618 2619 if (cmd->src_map != cmd->dst_map) { 2620 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2621 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2622 } 2623 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2624 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2625 free(cmd, M_DEVBUF); 2626 crypto_done(crp); 2627 } 2628 2629 #ifdef CRYPTO_LZS_COMP 2630 2631 static int 2632 hifn_compression(struct hifn_softc *sc, struct cryptop *crp, 2633 struct hifn_command *cmd) 2634 { 2635 struct cryptodesc *crd = crp->crp_desc; 2636 int s, err = 0; 2637 2638 cmd->compcrd = crd; 2639 cmd->base_masks |= HIFN_BASE_CMD_COMP; 2640 2641 if ((crp->crp_flags & CRYPTO_F_IMBUF) == 0) { 2642 /* 2643 * XXX can only handle mbufs right now since we can 2644 * XXX dynamically resize them. 2645 */ 2646 err = EINVAL; 2647 return (ENOMEM); 2648 } 2649 2650 if ((crd->crd_flags & CRD_F_COMP) == 0) 2651 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2652 if (crd->crd_alg == CRYPTO_LZS_COMP) 2653 cmd->comp_masks |= HIFN_COMP_CMD_ALG_LZS | 2654 HIFN_COMP_CMD_CLEARHIST; 2655 2656 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2657 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) { 2658 err = ENOMEM; 2659 goto fail; 2660 } 2661 2662 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2663 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 2664 err = ENOMEM; 2665 goto fail; 2666 } 2667 2668 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2669 int len; 2670 2671 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 2672 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 2673 err = ENOMEM; 2674 goto fail; 2675 } 2676 2677 len = cmd->src_map->dm_mapsize / MCLBYTES; 2678 if ((cmd->src_map->dm_mapsize % MCLBYTES) != 0) 2679 len++; 2680 len *= MCLBYTES; 2681 2682 if ((crd->crd_flags & CRD_F_COMP) == 0) 2683 len *= 4; 2684 2685 if (len > HIFN_MAX_DMALEN) 2686 len = HIFN_MAX_DMALEN; 2687 2688 cmd->dstu.dst_m = hifn_mkmbuf_chain(len, cmd->srcu.src_m); 2689 if (cmd->dstu.dst_m == NULL) { 2690 err = ENOMEM; 2691 goto fail; 2692 } 2693 2694 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2695 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2696 err = ENOMEM; 2697 goto fail; 2698 } 2699 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2700 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 2701 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 2702 err = ENOMEM; 2703 goto fail; 2704 } 2705 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 2706 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 2707 err = ENOMEM; 2708 goto fail; 2709 } 2710 } 2711 2712 if (cmd->src_map == cmd->dst_map) 2713 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2714 0, cmd->src_map->dm_mapsize, 2715 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 2716 else { 2717 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2718 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2719 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2720 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2721 } 2722 2723 cmd->crp = crp; 2724 /* 2725 * Always use session 0. The modes of compression we use are 2726 * stateless and there is always at least one compression 2727 * context, zero. 2728 */ 2729 cmd->session_num = 0; 2730 cmd->softc = sc; 2731 2732 err = hifn_compress_enter(sc, cmd); 2733 2734 if (err != 0) 2735 goto fail; 2736 return (0); 2737 2738 fail: 2739 if (cmd->dst_map != NULL) { 2740 if (cmd->dst_map->dm_nsegs > 0) 2741 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2742 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2743 } 2744 if (cmd->src_map != NULL) { 2745 if (cmd->src_map->dm_nsegs > 0) 2746 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2747 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2748 } 2749 free(cmd, M_DEVBUF); 2750 if (err == EINVAL) 2751 hifnstats.hst_invalid++; 2752 else 2753 hifnstats.hst_nomem++; 2754 crp->crp_etype = err; 2755 crypto_done(crp); 2756 return (0); 2757 } 2758 2759 static int 2760 hifn_compress_enter(struct hifn_softc *sc, struct hifn_command *cmd) 2761 { 2762 struct hifn_dma *dma = sc->sc_dma; 2763 int cmdi, resi; 2764 uint32_t cmdlen; 2765 2766 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 2767 (dma->resu + 1) > HIFN_D_CMD_RSIZE) 2768 return (ENOMEM); 2769 2770 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 2771 (dma->dstu + cmd->dst_map->dm_nsegs) > HIFN_D_DST_RSIZE) 2772 return (ENOMEM); 2773 2774 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 2775 dma->cmdi = 0; 2776 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 2777 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2778 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 2779 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2780 } 2781 cmdi = dma->cmdi++; 2782 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 2783 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 2784 2785 /* .p for command/result already set */ 2786 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 2787 HIFN_D_MASKDONEIRQ); 2788 HIFN_CMDR_SYNC(sc, cmdi, 2789 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2790 dma->cmdu++; 2791 if (sc->sc_c_busy == 0) { 2792 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 2793 sc->sc_c_busy = 1; 2794 SET_LED(sc, HIFN_MIPSRST_LED0); 2795 } 2796 2797 /* 2798 * We don't worry about missing an interrupt (which a "command wait" 2799 * interrupt salvages us from), unless there is more than one command 2800 * in the queue. 2801 */ 2802 if (dma->cmdu > 1) { 2803 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 2804 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 2805 } 2806 2807 hifnstats.hst_ipackets++; 2808 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 2809 2810 hifn_dmamap_load_src(sc, cmd); 2811 if (sc->sc_s_busy == 0) { 2812 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 2813 sc->sc_s_busy = 1; 2814 SET_LED(sc, HIFN_MIPSRST_LED1); 2815 } 2816 2817 /* 2818 * Unlike other descriptors, we don't mask done interrupt from 2819 * result descriptor. 2820 */ 2821 if (dma->resi == HIFN_D_RES_RSIZE) { 2822 dma->resi = 0; 2823 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 2824 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2825 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 2826 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2827 } 2828 resi = dma->resi++; 2829 dma->hifn_commands[resi] = cmd; 2830 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 2831 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 2832 HIFN_D_VALID | HIFN_D_LAST); 2833 HIFN_RESR_SYNC(sc, resi, 2834 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2835 dma->resu++; 2836 if (sc->sc_r_busy == 0) { 2837 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 2838 sc->sc_r_busy = 1; 2839 SET_LED(sc, HIFN_MIPSRST_LED2); 2840 } 2841 2842 if (cmd->sloplen) 2843 cmd->slopidx = resi; 2844 2845 hifn_dmamap_load_dst(sc, cmd); 2846 2847 if (sc->sc_d_busy == 0) { 2848 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 2849 sc->sc_d_busy = 1; 2850 } 2851 sc->sc_active = 5; 2852 cmd->cmd_callback = hifn_callback_comp; 2853 return (0); 2854 } 2855 2856 static void 2857 hifn_callback_comp(struct hifn_softc *sc, struct hifn_command *cmd, 2858 uint8_t *resbuf) 2859 { 2860 struct hifn_base_result baseres; 2861 struct cryptop *crp = cmd->crp; 2862 struct hifn_dma *dma = sc->sc_dma; 2863 struct mbuf *m; 2864 int err = 0, i, u; 2865 uint32_t olen; 2866 bus_size_t dstsize; 2867 2868 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2869 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2870 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2871 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2872 2873 dstsize = cmd->dst_map->dm_mapsize; 2874 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2875 2876 memcpy(&baseres, resbuf, sizeof(struct hifn_base_result)); 2877 2878 i = dma->dstk; u = dma->dstu; 2879 while (u != 0) { 2880 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2881 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2882 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2883 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2884 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2885 offsetof(struct hifn_dma, dstr[i]), 2886 sizeof(struct hifn_desc), 2887 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2888 break; 2889 } 2890 if (++i == (HIFN_D_DST_RSIZE + 1)) 2891 i = 0; 2892 else 2893 u--; 2894 } 2895 dma->dstk = i; dma->dstu = u; 2896 2897 if (baseres.flags & htole16(HIFN_BASE_RES_DSTOVERRUN)) { 2898 bus_size_t xlen; 2899 2900 xlen = dstsize; 2901 2902 m_freem(cmd->dstu.dst_m); 2903 2904 if (xlen == HIFN_MAX_DMALEN) { 2905 /* We've done all we can. */ 2906 err = E2BIG; 2907 goto out; 2908 } 2909 2910 xlen += MCLBYTES; 2911 2912 if (xlen > HIFN_MAX_DMALEN) 2913 xlen = HIFN_MAX_DMALEN; 2914 2915 cmd->dstu.dst_m = hifn_mkmbuf_chain(xlen, 2916 cmd->srcu.src_m); 2917 if (cmd->dstu.dst_m == NULL) { 2918 err = ENOMEM; 2919 goto out; 2920 } 2921 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2922 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2923 err = ENOMEM; 2924 goto out; 2925 } 2926 2927 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2928 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2929 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2930 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2931 2932 err = hifn_compress_enter(sc, cmd); 2933 if (err != 0) 2934 goto out; 2935 return; 2936 } 2937 2938 olen = dstsize - (letoh16(baseres.dst_cnt) | 2939 (((letoh16(baseres.session) & HIFN_BASE_RES_DSTLEN_M) >> 2940 HIFN_BASE_RES_DSTLEN_S) << 16)); 2941 2942 crp->crp_olen = olen - cmd->compcrd->crd_skip; 2943 2944 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2945 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2946 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2947 2948 m = cmd->dstu.dst_m; 2949 if (m->m_flags & M_PKTHDR) 2950 m->m_pkthdr.len = olen; 2951 crp->crp_buf = (void *)m; 2952 for (; m != NULL; m = m->m_next) { 2953 if (olen >= m->m_len) 2954 olen -= m->m_len; 2955 else { 2956 m->m_len = olen; 2957 olen = 0; 2958 } 2959 } 2960 2961 m_freem(cmd->srcu.src_m); 2962 free(cmd, M_DEVBUF); 2963 crp->crp_etype = 0; 2964 crypto_done(crp); 2965 return; 2966 2967 out: 2968 if (cmd->dst_map != NULL) { 2969 if (cmd->src_map->dm_nsegs != 0) 2970 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2971 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2972 } 2973 if (cmd->src_map != NULL) { 2974 if (cmd->src_map->dm_nsegs != 0) 2975 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2976 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2977 } 2978 if (cmd->dstu.dst_m != NULL) 2979 m_freem(cmd->dstu.dst_m); 2980 free(cmd, M_DEVBUF); 2981 crp->crp_etype = err; 2982 crypto_done(crp); 2983 } 2984 2985 static struct mbuf * 2986 hifn_mkmbuf_chain(int totlen, struct mbuf *mtemplate) 2987 { 2988 int len; 2989 struct mbuf *m, *m0, *mlast; 2990 2991 if (mtemplate->m_flags & M_PKTHDR) { 2992 len = MHLEN; 2993 MGETHDR(m0, M_DONTWAIT, MT_DATA); 2994 } else { 2995 len = MLEN; 2996 MGET(m0, M_DONTWAIT, MT_DATA); 2997 } 2998 if (m0 == NULL) 2999 return (NULL); 3000 if (len == MHLEN) 3001 m_copy_pkthdr(m0, mtemplate); 3002 MCLGET(m0, M_DONTWAIT); 3003 if (!(m0->m_flags & M_EXT)) { 3004 m_freem(m0); 3005 return (NULL); 3006 } 3007 len = MCLBYTES; 3008 3009 totlen -= len; 3010 m0->m_pkthdr.len = m0->m_len = len; 3011 mlast = m0; 3012 3013 while (totlen > 0) { 3014 MGET(m, M_DONTWAIT, MT_DATA); 3015 if (m == NULL) { 3016 m_freem(m0); 3017 return (NULL); 3018 } 3019 MCLGET(m, M_DONTWAIT); 3020 if (!(m->m_flags & M_EXT)) { 3021 m_freem(m); 3022 m_freem(m0); 3023 return (NULL); 3024 } 3025 len = MCLBYTES; 3026 m->m_len = len; 3027 if (m0->m_flags & M_PKTHDR) 3028 m0->m_pkthdr.len += len; 3029 totlen -= len; 3030 3031 mlast->m_next = m; 3032 mlast = m; 3033 } 3034 3035 return (m0); 3036 } 3037 #endif /* CRYPTO_LZS_COMP */ 3038 3039 static void 3040 hifn_write_4(struct hifn_softc *sc, int reggrp, bus_size_t reg, uint32_t val) 3041 { 3042 /* 3043 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0 3044 * and Group 1 registers; avoid conditions that could create 3045 * burst writes by doing a read in between the writes. 3046 */ 3047 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3048 if (sc->sc_waw_lastgroup == reggrp && 3049 sc->sc_waw_lastreg == reg - 4) { 3050 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID); 3051 } 3052 sc->sc_waw_lastgroup = reggrp; 3053 sc->sc_waw_lastreg = reg; 3054 } 3055 if (reggrp == 0) 3056 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val); 3057 else 3058 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val); 3059 3060 } 3061 3062 static uint32_t 3063 hifn_read_4(struct hifn_softc *sc, int reggrp, bus_size_t reg) 3064 { 3065 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3066 sc->sc_waw_lastgroup = -1; 3067 sc->sc_waw_lastreg = 1; 3068 } 3069 if (reggrp == 0) 3070 return (bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg)); 3071 return (bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg)); 3072 } 3073
Indexes created Fri Sep 26 08:10:20 GMT 2025