hifn7751.c revision 1.51
1 /* $NetBSD: hifn7751.c,v 1.51 2012/10/27 17:18:32 chs Exp $ */ 2 /* $FreeBSD: hifn7751.c,v 1.5.2.7 2003/10/08 23:52:00 sam Exp $ */ 3 /* $OpenBSD: hifn7751.c,v 1.140 2003/08/01 17:55:54 deraadt Exp $ */ 4 5 /* 6 * Invertex AEON / Hifn 7751 driver 7 * Copyright (c) 1999 Invertex Inc. All rights reserved. 8 * Copyright (c) 1999 Theo de Raadt 9 * Copyright (c) 2000-2001 Network Security Technologies, Inc. 10 * http://www.netsec.net 11 * Copyright (c) 2003 Hifn Inc. 12 * 13 * This driver is based on a previous driver by Invertex, for which they 14 * requested: Please send any comments, feedback, bug-fixes, or feature 15 * requests to software (at) invertex.com. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * 1. Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * 2. Redistributions in binary form must reproduce the above copyright 24 * notice, this list of conditions and the following disclaimer in the 25 * documentation and/or other materials provided with the distribution. 26 * 3. The name of the author may not be used to endorse or promote products 27 * derived from this software without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 30 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 31 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 32 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 33 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 34 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 38 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Effort sponsored in part by the Defense Advanced Research Projects 41 * Agency (DARPA) and Air Force Research Laboratory, Air Force 42 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 43 * 44 */ 45 46 /* 47 * Driver for various Hifn pre-HIPP encryption processors. 48 */ 49 50 #include <sys/cdefs.h> 51 __KERNEL_RCSID(0, "$NetBSD: hifn7751.c,v 1.51 2012/10/27 17:18:32 chs Exp $"); 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/proc.h> 56 #include <sys/errno.h> 57 #include <sys/malloc.h> 58 #include <sys/kernel.h> 59 #include <sys/mbuf.h> 60 #include <sys/device.h> 61 62 #ifdef __OpenBSD__ 63 #include <crypto/crypto.h> 64 #include <dev/rndvar.h> 65 #else 66 #include <opencrypto/cryptodev.h> 67 #include <sys/cprng.h> 68 #include <sys/rnd.h> 69 #include <sys/sha1.h> 70 #endif 71 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcivar.h> 74 #include <dev/pci/pcidevs.h> 75 76 #include <dev/pci/hifn7751reg.h> 77 #include <dev/pci/hifn7751var.h> 78 79 #undef HIFN_DEBUG 80 81 #ifdef __NetBSD__ 82 #define M_DUP_PKTHDR M_COPY_PKTHDR /* XXX */ 83 #endif 84 85 #ifdef HIFN_DEBUG 86 extern int hifn_debug; /* patchable */ 87 int hifn_debug = 1; 88 #endif 89 90 #ifdef __OpenBSD__ 91 #define HAVE_CRYPTO_LZS /* OpenBSD OCF supports CRYPTO_COMP_LZS */ 92 #endif 93 94 /* 95 * Prototypes and count for the pci_device structure 96 */ 97 #ifdef __OpenBSD__ 98 static int hifn_probe((struct device *, void *, void *); 99 #else 100 static int hifn_probe(device_t, cfdata_t, void *); 101 #endif 102 static void hifn_attach(device_t, device_t, void *); 103 104 CFATTACH_DECL_NEW(hifn, sizeof(struct hifn_softc), 105 hifn_probe, hifn_attach, NULL, NULL); 106 107 #ifdef __OpenBSD__ 108 struct cfdriver hifn_cd = { 109 0, "hifn", DV_DULL 110 }; 111 #endif 112 113 static void hifn_reset_board(struct hifn_softc *, int); 114 static void hifn_reset_puc(struct hifn_softc *); 115 static void hifn_puc_wait(struct hifn_softc *); 116 static const char *hifn_enable_crypto(struct hifn_softc *, pcireg_t); 117 static void hifn_set_retry(struct hifn_softc *); 118 static void hifn_init_dma(struct hifn_softc *); 119 static void hifn_init_pci_registers(struct hifn_softc *); 120 static int hifn_sramsize(struct hifn_softc *); 121 static int hifn_dramsize(struct hifn_softc *); 122 static int hifn_ramtype(struct hifn_softc *); 123 static void hifn_sessions(struct hifn_softc *); 124 static int hifn_intr(void *); 125 static u_int hifn_write_command(struct hifn_command *, u_int8_t *); 126 static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt); 127 static int hifn_newsession(void*, u_int32_t *, struct cryptoini *); 128 static int hifn_freesession(void*, u_int64_t); 129 static int hifn_process(void*, struct cryptop *, int); 130 static void hifn_callback(struct hifn_softc *, struct hifn_command *, 131 u_int8_t *); 132 static int hifn_crypto(struct hifn_softc *, struct hifn_command *, 133 struct cryptop*, int); 134 static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *); 135 static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *); 136 static int hifn_dmamap_aligned(bus_dmamap_t); 137 static int hifn_dmamap_load_src(struct hifn_softc *, 138 struct hifn_command *); 139 static int hifn_dmamap_load_dst(struct hifn_softc *, 140 struct hifn_command *); 141 static int hifn_init_pubrng(struct hifn_softc *); 142 static void hifn_rng(void *); 143 static void hifn_tick(void *); 144 static void hifn_abort(struct hifn_softc *); 145 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, 146 int *); 147 static void hifn_write_4(struct hifn_softc *, int, bus_size_t, u_int32_t); 148 static u_int32_t hifn_read_4(struct hifn_softc *, int, bus_size_t); 149 #ifdef HAVE_CRYPTO_LZS 150 static int hifn_compression(struct hifn_softc *, struct cryptop *, 151 struct hifn_command *); 152 static struct mbuf *hifn_mkmbuf_chain(int, struct mbuf *); 153 static int hifn_compress_enter(struct hifn_softc *, struct hifn_command *); 154 static void hifn_callback_comp(struct hifn_softc *, struct hifn_command *, 155 u_int8_t *); 156 #endif /* HAVE_CRYPTO_LZS */ 157 158 159 struct hifn_stats hifnstats; 160 161 static const struct hifn_product { 162 pci_vendor_id_t hifn_vendor; 163 pci_product_id_t hifn_product; 164 int hifn_flags; 165 const char *hifn_name; 166 } hifn_products[] = { 167 { PCI_VENDOR_INVERTEX, PCI_PRODUCT_INVERTEX_AEON, 168 0, 169 "Invertex AEON", 170 }, 171 172 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7751, 173 0, 174 "Hifn 7751", 175 }, 176 { PCI_VENDOR_NETSEC, PCI_PRODUCT_NETSEC_7751, 177 0, 178 "Hifn 7751 (NetSec)" 179 }, 180 181 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7811, 182 HIFN_IS_7811 | HIFN_HAS_RNG | HIFN_HAS_LEDS | HIFN_NO_BURSTWRITE, 183 "Hifn 7811", 184 }, 185 186 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7951, 187 HIFN_HAS_RNG | HIFN_HAS_PUBLIC, 188 "Hifn 7951", 189 }, 190 191 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7955, 192 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 193 "Hifn 7955", 194 }, 195 196 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7956, 197 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 198 "Hifn 7956", 199 }, 200 201 202 { 0, 0, 203 0, 204 NULL 205 } 206 }; 207 208 static const struct hifn_product * 209 hifn_lookup(const struct pci_attach_args *pa) 210 { 211 const struct hifn_product *hp; 212 213 for (hp = hifn_products; hp->hifn_name != NULL; hp++) { 214 if (PCI_VENDOR(pa->pa_id) == hp->hifn_vendor && 215 PCI_PRODUCT(pa->pa_id) == hp->hifn_product) 216 return (hp); 217 } 218 return (NULL); 219 } 220 221 static int 222 hifn_probe(device_t parent, cfdata_t match, void *aux) 223 { 224 struct pci_attach_args *pa = aux; 225 226 if (hifn_lookup(pa) != NULL) 227 return 1; 228 229 return 0; 230 } 231 232 static void 233 hifn_attach(device_t parent, device_t self, void *aux) 234 { 235 struct hifn_softc *sc = device_private(self); 236 struct pci_attach_args *pa = aux; 237 const struct hifn_product *hp; 238 pci_chipset_tag_t pc = pa->pa_pc; 239 pci_intr_handle_t ih; 240 const char *intrstr = NULL; 241 const char *hifncap; 242 char rbase; 243 bus_size_t iosize0, iosize1; 244 u_int32_t cmd; 245 u_int16_t ena; 246 bus_dma_segment_t seg; 247 bus_dmamap_t dmamap; 248 int rseg; 249 void *kva; 250 251 hp = hifn_lookup(pa); 252 if (hp == NULL) { 253 printf("\n"); 254 panic("hifn_attach: impossible"); 255 } 256 257 pci_aprint_devinfo_fancy(pa, "Crypto processor", hp->hifn_name, 1); 258 259 sc->sc_dv = self; 260 sc->sc_pci_pc = pa->pa_pc; 261 sc->sc_pci_tag = pa->pa_tag; 262 263 sc->sc_flags = hp->hifn_flags; 264 265 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 266 cmd |= PCI_COMMAND_MASTER_ENABLE; 267 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 268 269 if (pci_mapreg_map(pa, HIFN_BAR0, PCI_MAPREG_TYPE_MEM, 0, 270 &sc->sc_st0, &sc->sc_sh0, NULL, &iosize0)) { 271 aprint_error_dev(sc->sc_dv, "can't map mem space %d\n", 0); 272 return; 273 } 274 275 if (pci_mapreg_map(pa, HIFN_BAR1, PCI_MAPREG_TYPE_MEM, 0, 276 &sc->sc_st1, &sc->sc_sh1, NULL, &iosize1)) { 277 aprint_error_dev(sc->sc_dv, "can't find mem space %d\n", 1); 278 goto fail_io0; 279 } 280 281 hifn_set_retry(sc); 282 283 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 284 sc->sc_waw_lastgroup = -1; 285 sc->sc_waw_lastreg = 1; 286 } 287 288 sc->sc_dmat = pa->pa_dmat; 289 if (bus_dmamem_alloc(sc->sc_dmat, sizeof(*sc->sc_dma), PAGE_SIZE, 0, 290 &seg, 1, &rseg, BUS_DMA_NOWAIT)) { 291 aprint_error_dev(sc->sc_dv, "can't alloc DMA buffer\n"); 292 goto fail_io1; 293 } 294 if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, sizeof(*sc->sc_dma), &kva, 295 BUS_DMA_NOWAIT)) { 296 aprint_error_dev(sc->sc_dv, "can't map DMA buffers (%lu bytes)\n", 297 (u_long)sizeof(*sc->sc_dma)); 298 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 299 goto fail_io1; 300 } 301 if (bus_dmamap_create(sc->sc_dmat, sizeof(*sc->sc_dma), 1, 302 sizeof(*sc->sc_dma), 0, BUS_DMA_NOWAIT, &dmamap)) { 303 aprint_error_dev(sc->sc_dv, "can't create DMA map\n"); 304 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 305 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 306 goto fail_io1; 307 } 308 if (bus_dmamap_load(sc->sc_dmat, dmamap, kva, sizeof(*sc->sc_dma), 309 NULL, BUS_DMA_NOWAIT)) { 310 aprint_error_dev(sc->sc_dv, "can't load DMA map\n"); 311 bus_dmamap_destroy(sc->sc_dmat, dmamap); 312 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 313 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 314 goto fail_io1; 315 } 316 sc->sc_dmamap = dmamap; 317 sc->sc_dma = (struct hifn_dma *)kva; 318 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 319 320 hifn_reset_board(sc, 0); 321 322 if ((hifncap = hifn_enable_crypto(sc, pa->pa_id)) == NULL) { 323 aprint_error_dev(sc->sc_dv, "crypto enabling failed\n"); 324 goto fail_mem; 325 } 326 hifn_reset_puc(sc); 327 328 hifn_init_dma(sc); 329 hifn_init_pci_registers(sc); 330 331 /* XXX can't dynamically determine ram type for 795x; force dram */ 332 if (sc->sc_flags & HIFN_IS_7956) 333 sc->sc_drammodel = 1; 334 else if (hifn_ramtype(sc)) 335 goto fail_mem; 336 337 if (sc->sc_drammodel == 0) 338 hifn_sramsize(sc); 339 else 340 hifn_dramsize(sc); 341 342 /* 343 * Workaround for NetSec 7751 rev A: half ram size because two 344 * of the address lines were left floating 345 */ 346 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETSEC && 347 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETSEC_7751 && 348 PCI_REVISION(pa->pa_class) == 0x61) 349 sc->sc_ramsize >>= 1; 350 351 if (pci_intr_map(pa, &ih)) { 352 aprint_error_dev(sc->sc_dv, "couldn't map interrupt\n"); 353 goto fail_mem; 354 } 355 intrstr = pci_intr_string(pc, ih); 356 #ifdef __OpenBSD__ 357 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc, 358 device_xname(self)); 359 #else 360 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc); 361 #endif 362 if (sc->sc_ih == NULL) { 363 aprint_error_dev(sc->sc_dv, "couldn't establish interrupt\n"); 364 if (intrstr != NULL) 365 aprint_error(" at %s", intrstr); 366 aprint_error("\n"); 367 goto fail_mem; 368 } 369 370 hifn_sessions(sc); 371 372 rseg = sc->sc_ramsize / 1024; 373 rbase = 'K'; 374 if (sc->sc_ramsize >= (1024 * 1024)) { 375 rbase = 'M'; 376 rseg /= 1024; 377 } 378 aprint_normal_dev(sc->sc_dv, "%s, %d%cB %cRAM, interrupting at %s\n", 379 hifncap, rseg, rbase, 380 sc->sc_drammodel ? 'D' : 'S', intrstr); 381 382 sc->sc_cid = crypto_get_driverid(0); 383 if (sc->sc_cid < 0) { 384 aprint_error_dev(sc->sc_dv, "couldn't get crypto driver id\n"); 385 goto fail_intr; 386 } 387 388 WRITE_REG_0(sc, HIFN_0_PUCNFG, 389 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID); 390 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 391 392 switch (ena) { 393 case HIFN_PUSTAT_ENA_2: 394 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 395 hifn_newsession, hifn_freesession, hifn_process, sc); 396 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0, 397 hifn_newsession, hifn_freesession, hifn_process, sc); 398 if (sc->sc_flags & HIFN_HAS_AES) 399 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 400 hifn_newsession, hifn_freesession, 401 hifn_process, sc); 402 /*FALLTHROUGH*/ 403 case HIFN_PUSTAT_ENA_1: 404 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0, 405 hifn_newsession, hifn_freesession, hifn_process, sc); 406 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0, 407 hifn_newsession, hifn_freesession, hifn_process, sc); 408 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 409 hifn_newsession, hifn_freesession, hifn_process, sc); 410 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 411 hifn_newsession, hifn_freesession, hifn_process, sc); 412 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 413 hifn_newsession, hifn_freesession, hifn_process, sc); 414 break; 415 } 416 417 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 0, 418 sc->sc_dmamap->dm_mapsize, 419 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 420 421 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) 422 hifn_init_pubrng(sc); 423 424 #ifdef __OpenBSD__ 425 timeout_set(&sc->sc_tickto, hifn_tick, sc); 426 timeout_add(&sc->sc_tickto, hz); 427 #else 428 callout_init(&sc->sc_tickto, 0); 429 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 430 #endif 431 return; 432 433 fail_intr: 434 pci_intr_disestablish(pc, sc->sc_ih); 435 fail_mem: 436 bus_dmamap_unload(sc->sc_dmat, dmamap); 437 bus_dmamap_destroy(sc->sc_dmat, dmamap); 438 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 439 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 440 441 /* Turn off DMA polling */ 442 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 443 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 444 445 fail_io1: 446 bus_space_unmap(sc->sc_st1, sc->sc_sh1, iosize1); 447 fail_io0: 448 bus_space_unmap(sc->sc_st0, sc->sc_sh0, iosize0); 449 } 450 451 static int 452 hifn_init_pubrng(struct hifn_softc *sc) 453 { 454 u_int32_t r; 455 int i; 456 457 if ((sc->sc_flags & HIFN_IS_7811) == 0) { 458 /* Reset 7951 public key/rng engine */ 459 WRITE_REG_1(sc, HIFN_1_PUB_RESET, 460 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET); 461 462 for (i = 0; i < 100; i++) { 463 DELAY(1000); 464 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) & 465 HIFN_PUBRST_RESET) == 0) 466 break; 467 } 468 469 if (i == 100) { 470 printf("%s: public key init failed\n", 471 device_xname(sc->sc_dv)); 472 return (1); 473 } 474 } 475 476 /* Enable the rng, if available */ 477 if (sc->sc_flags & HIFN_HAS_RNG) { 478 if (sc->sc_flags & HIFN_IS_7811) { 479 r = READ_REG_1(sc, HIFN_1_7811_RNGENA); 480 if (r & HIFN_7811_RNGENA_ENA) { 481 r &= ~HIFN_7811_RNGENA_ENA; 482 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 483 } 484 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG, 485 HIFN_7811_RNGCFG_DEFL); 486 r |= HIFN_7811_RNGENA_ENA; 487 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 488 } else 489 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG, 490 READ_REG_1(sc, HIFN_1_RNG_CONFIG) | 491 HIFN_RNGCFG_ENA); 492 493 /* 494 * The Hifn RNG documentation states that at their 495 * recommended "conservative" RNG config values, 496 * the RNG must warm up for 0.4s before providing 497 * data that meet their worst-case estimate of 0.06 498 * bits of random data per output register bit. 499 */ 500 DELAY(4000); 501 502 #ifdef __NetBSD__ 503 /* 504 * XXX Careful! The use of RND_FLAG_NO_ESTIMATE 505 * XXX here is unobvious: we later feed raw bits 506 * XXX into the "entropy pool" with rnd_add_data, 507 * XXX explicitly supplying an entropy estimate. 508 * XXX In this context, NO_ESTIMATE serves only 509 * XXX to prevent rnd_add_data from trying to 510 * XXX use the *time at which we added the data* 511 * XXX as entropy, which is not a good idea since 512 * XXX we add data periodically from a callout. 513 */ 514 rnd_attach_source(&sc->sc_rnd_source, device_xname(sc->sc_dv), 515 RND_TYPE_RNG, RND_FLAG_NO_ESTIMATE); 516 #endif 517 518 sc->sc_rngfirst = 1; 519 if (hz >= 100) 520 sc->sc_rnghz = hz / 100; 521 else 522 sc->sc_rnghz = 1; 523 #ifdef __OpenBSD__ 524 timeout_set(&sc->sc_rngto, hifn_rng, sc); 525 #else /* !__OpenBSD__ */ 526 callout_init(&sc->sc_rngto, 0); 527 #endif /* !__OpenBSD__ */ 528 } 529 530 /* Enable public key engine, if available */ 531 if (sc->sc_flags & HIFN_HAS_PUBLIC) { 532 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 533 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE; 534 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 535 } 536 537 /* Call directly into the RNG once to prime the pool. */ 538 hifn_rng(sc); /* Sets callout/timeout at end */ 539 540 return (0); 541 } 542 543 static void 544 hifn_rng(void *vsc) 545 { 546 struct hifn_softc *sc = vsc; 547 #ifdef __NetBSD__ 548 uint32_t num[64]; 549 #else 550 uint32_t num[2]; 551 #endif 552 uint32_t sts; 553 int i; 554 555 if (sc->sc_flags & HIFN_IS_7811) { 556 for (i = 0; i < 5; i++) { /* XXX why 5? */ 557 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS); 558 if (sts & HIFN_7811_RNGSTS_UFL) { 559 printf("%s: RNG underflow: disabling\n", 560 device_xname(sc->sc_dv)); 561 return; 562 } 563 if ((sts & HIFN_7811_RNGSTS_RDY) == 0) 564 break; 565 566 /* 567 * There are at least two words in the RNG FIFO 568 * at this point. 569 */ 570 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 571 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 572 573 if (sc->sc_rngfirst) 574 sc->sc_rngfirst = 0; 575 #ifdef __NetBSD__ 576 rnd_add_data(&sc->sc_rnd_source, num, 577 2 * sizeof(num[0]), 578 (2 * sizeof(num[0]) * NBBY) / 579 HIFN_RNG_BITSPER); 580 #else 581 /* 582 * XXX This is a really bad idea. 583 * XXX Hifn estimate as little as 0.06 584 * XXX actual bits of entropy per output 585 * XXX register bit. How can we tell the 586 * XXX kernel RNG subsystem we're handing 587 * XXX it 64 "true" random bits, for any 588 * XXX sane value of "true"? 589 * XXX 590 * XXX The right thing to do here, if we 591 * XXX cannot supply an estimate ourselves, 592 * XXX would be to hash the bits locally. 593 */ 594 add_true_randomness(num[0]); 595 add_true_randomness(num[1]); 596 #endif 597 598 } 599 } else { 600 #ifdef __NetBSD__ 601 /* First time through, try to help fill the pool. */ 602 int nwords = sc->sc_rngfirst ? 603 sizeof(num) / sizeof(num[0]) : 4; 604 #else 605 int nwords = 2; 606 #endif 607 /* 608 * We must be *extremely* careful here. The Hifn 609 * 795x differ from the published 6500 RNG design 610 * in more ways than the obvious lack of the output 611 * FIFO and LFSR control registers. In fact, there 612 * is only one LFSR, instead of the 6500's two, and 613 * it's 32 bits, not 31. 614 * 615 * Further, a block diagram obtained from Hifn shows 616 * a very curious latching of this register: the LFSR 617 * rotates at a frequency of RNG_Clk / 8, but the 618 * RNG_Data register is latched at a frequency of 619 * RNG_Clk, which means that it is possible for 620 * consecutive reads of the RNG_Data register to read 621 * identical state from the LFSR. The simplest 622 * workaround seems to be to read eight samples from 623 * the register for each one that we use. Since each 624 * read must require at least one PCI cycle, and 625 * RNG_Clk is at least PCI_Clk, this is safe. 626 */ 627 628 629 if (sc->sc_rngfirst) { 630 sc->sc_rngfirst = 0; 631 } 632 633 634 for(i = 0 ; i < nwords * 8; i++) 635 { 636 volatile u_int32_t regtmp; 637 regtmp = READ_REG_1(sc, HIFN_1_RNG_DATA); 638 num[i / 8] = regtmp; 639 } 640 #ifdef __NetBSD__ 641 rnd_add_data(&sc->sc_rnd_source, num, 642 nwords * sizeof(num[0]), 643 (nwords * sizeof(num[0]) * NBBY) / 644 HIFN_RNG_BITSPER); 645 #else 646 /* XXX a bad idea; see 7811 block above */ 647 add_true_randomness(num[0]); 648 #endif 649 } 650 651 #ifdef __OpenBSD__ 652 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 653 #else 654 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc); 655 #endif 656 } 657 658 static void 659 hifn_puc_wait(struct hifn_softc *sc) 660 { 661 int i; 662 663 for (i = 5000; i > 0; i--) { 664 DELAY(1); 665 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET)) 666 break; 667 } 668 if (!i) 669 printf("%s: proc unit did not reset\n", device_xname(sc->sc_dv)); 670 } 671 672 /* 673 * Reset the processing unit. 674 */ 675 static void 676 hifn_reset_puc(struct hifn_softc *sc) 677 { 678 /* Reset processing unit */ 679 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 680 hifn_puc_wait(sc); 681 } 682 683 static void 684 hifn_set_retry(struct hifn_softc *sc) 685 { 686 u_int32_t r; 687 688 r = pci_conf_read(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT); 689 r &= 0xffff0000; 690 pci_conf_write(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT, r); 691 } 692 693 /* 694 * Resets the board. Values in the regesters are left as is 695 * from the reset (i.e. initial values are assigned elsewhere). 696 */ 697 static void 698 hifn_reset_board(struct hifn_softc *sc, int full) 699 { 700 u_int32_t reg; 701 702 /* 703 * Set polling in the DMA configuration register to zero. 0x7 avoids 704 * resetting the board and zeros out the other fields. 705 */ 706 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 707 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 708 709 /* 710 * Now that polling has been disabled, we have to wait 1 ms 711 * before resetting the board. 712 */ 713 DELAY(1000); 714 715 /* Reset the DMA unit */ 716 if (full) { 717 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 718 DELAY(1000); 719 } else { 720 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, 721 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET); 722 hifn_reset_puc(sc); 723 } 724 725 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 726 727 /* Bring dma unit out of reset */ 728 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 729 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 730 731 hifn_puc_wait(sc); 732 733 hifn_set_retry(sc); 734 735 if (sc->sc_flags & HIFN_IS_7811) { 736 for (reg = 0; reg < 1000; reg++) { 737 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) & 738 HIFN_MIPSRST_CRAMINIT) 739 break; 740 DELAY(1000); 741 } 742 if (reg == 1000) 743 printf(": cram init timeout\n"); 744 } 745 } 746 747 static u_int32_t 748 hifn_next_signature(u_int32_t a, u_int cnt) 749 { 750 int i; 751 u_int32_t v; 752 753 for (i = 0; i < cnt; i++) { 754 755 /* get the parity */ 756 v = a & 0x80080125; 757 v ^= v >> 16; 758 v ^= v >> 8; 759 v ^= v >> 4; 760 v ^= v >> 2; 761 v ^= v >> 1; 762 763 a = (v & 1) ^ (a << 1); 764 } 765 766 return a; 767 } 768 769 static struct pci2id { 770 u_short pci_vendor; 771 u_short pci_prod; 772 char card_id[13]; 773 } const pci2id[] = { 774 { 775 PCI_VENDOR_HIFN, 776 PCI_PRODUCT_HIFN_7951, 777 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 778 0x00, 0x00, 0x00, 0x00, 0x00 } 779 }, { 780 PCI_VENDOR_HIFN, 781 PCI_PRODUCT_HIFN_7955, 782 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 783 0x00, 0x00, 0x00, 0x00, 0x00 } 784 }, { 785 PCI_VENDOR_HIFN, 786 PCI_PRODUCT_HIFN_7956, 787 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 788 0x00, 0x00, 0x00, 0x00, 0x00 } 789 }, { 790 PCI_VENDOR_NETSEC, 791 PCI_PRODUCT_NETSEC_7751, 792 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 793 0x00, 0x00, 0x00, 0x00, 0x00 } 794 }, { 795 PCI_VENDOR_INVERTEX, 796 PCI_PRODUCT_INVERTEX_AEON, 797 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 798 0x00, 0x00, 0x00, 0x00, 0x00 } 799 }, { 800 PCI_VENDOR_HIFN, 801 PCI_PRODUCT_HIFN_7811, 802 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 803 0x00, 0x00, 0x00, 0x00, 0x00 } 804 }, { 805 /* 806 * Other vendors share this PCI ID as well, such as 807 * http://www.powercrypt.com, and obviously they also 808 * use the same key. 809 */ 810 PCI_VENDOR_HIFN, 811 PCI_PRODUCT_HIFN_7751, 812 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 813 0x00, 0x00, 0x00, 0x00, 0x00 } 814 }, 815 }; 816 817 /* 818 * Checks to see if crypto is already enabled. If crypto isn't enable, 819 * "hifn_enable_crypto" is called to enable it. The check is important, 820 * as enabling crypto twice will lock the board. 821 */ 822 static const char * 823 hifn_enable_crypto(struct hifn_softc *sc, pcireg_t pciid) 824 { 825 u_int32_t dmacfg, ramcfg, encl, addr, i; 826 const char *offtbl = NULL; 827 828 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { 829 if (pci2id[i].pci_vendor == PCI_VENDOR(pciid) && 830 pci2id[i].pci_prod == PCI_PRODUCT(pciid)) { 831 offtbl = pci2id[i].card_id; 832 break; 833 } 834 } 835 836 if (offtbl == NULL) { 837 #ifdef HIFN_DEBUG 838 aprint_debug_dev(sc->sc_dv, "Unknown card!\n"); 839 #endif 840 return (NULL); 841 } 842 843 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG); 844 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG); 845 846 /* 847 * The RAM config register's encrypt level bit needs to be set before 848 * every read performed on the encryption level register. 849 */ 850 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 851 852 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 853 854 /* 855 * Make sure we don't re-unlock. Two unlocks kills chip until the 856 * next reboot. 857 */ 858 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) { 859 #ifdef HIFN_DEBUG 860 aprint_debug_dev(sc->sc_dv, "Strong Crypto already enabled!\n"); 861 #endif 862 goto report; 863 } 864 865 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) { 866 #ifdef HIFN_DEBUG 867 aprint_debug_dev(sc->sc_dv, "Unknown encryption level\n"); 868 #endif 869 return (NULL); 870 } 871 872 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK | 873 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 874 DELAY(1000); 875 addr = READ_REG_1(sc, HIFN_1_UNLOCK_SECRET1); 876 DELAY(1000); 877 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, 0); 878 DELAY(1000); 879 880 for (i = 0; i <= 12; i++) { 881 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 882 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, addr); 883 884 DELAY(1000); 885 } 886 887 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 888 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 889 890 #ifdef HIFN_DEBUG 891 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2) 892 aprint_debug("Encryption engine is permanently locked until next system reset."); 893 else 894 aprint_debug("Encryption engine enabled successfully!"); 895 #endif 896 897 report: 898 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg); 899 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg); 900 901 switch (encl) { 902 case HIFN_PUSTAT_ENA_0: 903 return ("LZS-only (no encr/auth)"); 904 905 case HIFN_PUSTAT_ENA_1: 906 return ("DES"); 907 908 case HIFN_PUSTAT_ENA_2: 909 if (sc->sc_flags & HIFN_HAS_AES) 910 return ("3DES/AES"); 911 else 912 return ("3DES"); 913 914 default: 915 return ("disabled"); 916 } 917 /* NOTREACHED */ 918 } 919 920 /* 921 * Give initial values to the registers listed in the "Register Space" 922 * section of the HIFN Software Development reference manual. 923 */ 924 static void 925 hifn_init_pci_registers(struct hifn_softc *sc) 926 { 927 /* write fixed values needed by the Initialization registers */ 928 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 929 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 930 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 931 932 /* write all 4 ring address registers */ 933 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 934 offsetof(struct hifn_dma, cmdr[0])); 935 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 936 offsetof(struct hifn_dma, srcr[0])); 937 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 938 offsetof(struct hifn_dma, dstr[0])); 939 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 940 offsetof(struct hifn_dma, resr[0])); 941 942 DELAY(2000); 943 944 /* write status register */ 945 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 946 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 947 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 948 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 949 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 950 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 951 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 952 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 953 HIFN_DMACSR_S_WAIT | 954 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 955 HIFN_DMACSR_C_WAIT | 956 HIFN_DMACSR_ENGINE | 957 ((sc->sc_flags & HIFN_HAS_PUBLIC) ? 958 HIFN_DMACSR_PUBDONE : 0) | 959 ((sc->sc_flags & HIFN_IS_7811) ? 960 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0)); 961 962 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0; 963 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 964 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 965 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 966 HIFN_DMAIER_ENGINE | 967 ((sc->sc_flags & HIFN_IS_7811) ? 968 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0); 969 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 970 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 971 CLR_LED(sc, HIFN_MIPSRST_LED0 | HIFN_MIPSRST_LED1 | HIFN_MIPSRST_LED2); 972 973 if (sc->sc_flags & HIFN_IS_7956) { 974 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 975 HIFN_PUCNFG_TCALLPHASES | 976 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32); 977 WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956); 978 } else { 979 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 980 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 981 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 982 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM)); 983 } 984 985 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 986 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 987 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 988 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 989 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 990 } 991 992 /* 993 * The maximum number of sessions supported by the card 994 * is dependent on the amount of context ram, which 995 * encryption algorithms are enabled, and how compression 996 * is configured. This should be configured before this 997 * routine is called. 998 */ 999 static void 1000 hifn_sessions(struct hifn_softc *sc) 1001 { 1002 u_int32_t pucnfg; 1003 int ctxsize; 1004 1005 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG); 1006 1007 if (pucnfg & HIFN_PUCNFG_COMPSING) { 1008 if (pucnfg & HIFN_PUCNFG_ENCCNFG) 1009 ctxsize = 128; 1010 else 1011 ctxsize = 512; 1012 /* 1013 * 7955/7956 has internal context memory of 32K 1014 */ 1015 if (sc->sc_flags & HIFN_IS_7956) 1016 sc->sc_maxses = 32768 / ctxsize; 1017 else 1018 sc->sc_maxses = 1 + 1019 ((sc->sc_ramsize - 32768) / ctxsize); 1020 } 1021 else 1022 sc->sc_maxses = sc->sc_ramsize / 16384; 1023 1024 if (sc->sc_maxses > 2048) 1025 sc->sc_maxses = 2048; 1026 } 1027 1028 /* 1029 * Determine ram type (sram or dram). Board should be just out of a reset 1030 * state when this is called. 1031 */ 1032 static int 1033 hifn_ramtype(struct hifn_softc *sc) 1034 { 1035 u_int8_t data[8], dataexpect[8]; 1036 int i; 1037 1038 for (i = 0; i < sizeof(data); i++) 1039 data[i] = dataexpect[i] = 0x55; 1040 if (hifn_writeramaddr(sc, 0, data)) 1041 return (-1); 1042 if (hifn_readramaddr(sc, 0, data)) 1043 return (-1); 1044 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1045 sc->sc_drammodel = 1; 1046 return (0); 1047 } 1048 1049 for (i = 0; i < sizeof(data); i++) 1050 data[i] = dataexpect[i] = 0xaa; 1051 if (hifn_writeramaddr(sc, 0, data)) 1052 return (-1); 1053 if (hifn_readramaddr(sc, 0, data)) 1054 return (-1); 1055 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1056 sc->sc_drammodel = 1; 1057 return (0); 1058 } 1059 1060 return (0); 1061 } 1062 1063 #define HIFN_SRAM_MAX (32 << 20) 1064 #define HIFN_SRAM_STEP_SIZE 16384 1065 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE) 1066 1067 static int 1068 hifn_sramsize(struct hifn_softc *sc) 1069 { 1070 u_int32_t a; 1071 u_int8_t data[8]; 1072 u_int8_t dataexpect[sizeof(data)]; 1073 int32_t i; 1074 1075 for (i = 0; i < sizeof(data); i++) 1076 data[i] = dataexpect[i] = i ^ 0x5a; 1077 1078 for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) { 1079 a = i * HIFN_SRAM_STEP_SIZE; 1080 memcpy(data, &i, sizeof(i)); 1081 hifn_writeramaddr(sc, a, data); 1082 } 1083 1084 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) { 1085 a = i * HIFN_SRAM_STEP_SIZE; 1086 memcpy(dataexpect, &i, sizeof(i)); 1087 if (hifn_readramaddr(sc, a, data) < 0) 1088 return (0); 1089 if (memcmp(data, dataexpect, sizeof(data)) != 0) 1090 return (0); 1091 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE; 1092 } 1093 1094 return (0); 1095 } 1096 1097 /* 1098 * XXX For dram boards, one should really try all of the 1099 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG 1100 * is already set up correctly. 1101 */ 1102 static int 1103 hifn_dramsize(struct hifn_softc *sc) 1104 { 1105 u_int32_t cnfg; 1106 1107 if (sc->sc_flags & HIFN_IS_7956) { 1108 /* 1109 * 7955/7956 have a fixed internal ram of only 32K. 1110 */ 1111 sc->sc_ramsize = 32768; 1112 } else { 1113 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) & 1114 HIFN_PUCNFG_DRAMMASK; 1115 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18); 1116 } 1117 return (0); 1118 } 1119 1120 static void 1121 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, 1122 int *resp) 1123 { 1124 struct hifn_dma *dma = sc->sc_dma; 1125 1126 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1127 dma->cmdi = 0; 1128 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1129 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1130 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1131 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1132 } 1133 *cmdp = dma->cmdi++; 1134 dma->cmdk = dma->cmdi; 1135 1136 if (dma->srci == HIFN_D_SRC_RSIZE) { 1137 dma->srci = 0; 1138 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID | 1139 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1140 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1141 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1142 } 1143 *srcp = dma->srci++; 1144 dma->srck = dma->srci; 1145 1146 if (dma->dsti == HIFN_D_DST_RSIZE) { 1147 dma->dsti = 0; 1148 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID | 1149 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1150 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE, 1151 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1152 } 1153 *dstp = dma->dsti++; 1154 dma->dstk = dma->dsti; 1155 1156 if (dma->resi == HIFN_D_RES_RSIZE) { 1157 dma->resi = 0; 1158 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1159 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1160 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1161 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1162 } 1163 *resp = dma->resi++; 1164 dma->resk = dma->resi; 1165 } 1166 1167 static int 1168 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1169 { 1170 struct hifn_dma *dma = sc->sc_dma; 1171 struct hifn_base_command wc; 1172 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1173 int r, cmdi, resi, srci, dsti; 1174 1175 wc.masks = htole16(3 << 13); 1176 wc.session_num = htole16(addr >> 14); 1177 wc.total_source_count = htole16(8); 1178 wc.total_dest_count = htole16(addr & 0x3fff); 1179 1180 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1181 1182 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1183 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1184 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1185 1186 /* build write command */ 1187 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1188 *(struct hifn_base_command *)dma->command_bufs[cmdi] = wc; 1189 memcpy(&dma->test_src, data, sizeof(dma->test_src)); 1190 1191 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1192 + offsetof(struct hifn_dma, test_src)); 1193 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1194 + offsetof(struct hifn_dma, test_dst)); 1195 1196 dma->cmdr[cmdi].l = htole32(16 | masks); 1197 dma->srcr[srci].l = htole32(8 | masks); 1198 dma->dstr[dsti].l = htole32(4 | masks); 1199 dma->resr[resi].l = htole32(4 | masks); 1200 1201 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1202 0, sc->sc_dmamap->dm_mapsize, 1203 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1204 1205 for (r = 10000; r >= 0; r--) { 1206 DELAY(10); 1207 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1208 0, sc->sc_dmamap->dm_mapsize, 1209 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1210 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1211 break; 1212 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1213 0, sc->sc_dmamap->dm_mapsize, 1214 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1215 } 1216 if (r == 0) { 1217 printf("%s: writeramaddr -- " 1218 "result[%d](addr %d) still valid\n", 1219 device_xname(sc->sc_dv), resi, addr); 1220 r = -1; 1221 return (-1); 1222 } else 1223 r = 0; 1224 1225 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1226 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1227 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1228 1229 return (r); 1230 } 1231 1232 static int 1233 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1234 { 1235 struct hifn_dma *dma = sc->sc_dma; 1236 struct hifn_base_command rc; 1237 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1238 int r, cmdi, srci, dsti, resi; 1239 1240 rc.masks = htole16(2 << 13); 1241 rc.session_num = htole16(addr >> 14); 1242 rc.total_source_count = htole16(addr & 0x3fff); 1243 rc.total_dest_count = htole16(8); 1244 1245 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1246 1247 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1248 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1249 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1250 1251 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1252 *(struct hifn_base_command *)dma->command_bufs[cmdi] = rc; 1253 1254 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1255 offsetof(struct hifn_dma, test_src)); 1256 dma->test_src = 0; 1257 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1258 offsetof(struct hifn_dma, test_dst)); 1259 dma->test_dst = 0; 1260 dma->cmdr[cmdi].l = htole32(8 | masks); 1261 dma->srcr[srci].l = htole32(8 | masks); 1262 dma->dstr[dsti].l = htole32(8 | masks); 1263 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks); 1264 1265 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1266 0, sc->sc_dmamap->dm_mapsize, 1267 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1268 1269 for (r = 10000; r >= 0; r--) { 1270 DELAY(10); 1271 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1272 0, sc->sc_dmamap->dm_mapsize, 1273 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1274 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1275 break; 1276 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1277 0, sc->sc_dmamap->dm_mapsize, 1278 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1279 } 1280 if (r == 0) { 1281 printf("%s: readramaddr -- " 1282 "result[%d](addr %d) still valid\n", 1283 device_xname(sc->sc_dv), resi, addr); 1284 r = -1; 1285 } else { 1286 r = 0; 1287 memcpy(data, &dma->test_dst, sizeof(dma->test_dst)); 1288 } 1289 1290 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1291 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1292 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1293 1294 return (r); 1295 } 1296 1297 /* 1298 * Initialize the descriptor rings. 1299 */ 1300 static void 1301 hifn_init_dma(struct hifn_softc *sc) 1302 { 1303 struct hifn_dma *dma = sc->sc_dma; 1304 int i; 1305 1306 hifn_set_retry(sc); 1307 1308 /* initialize static pointer values */ 1309 for (i = 0; i < HIFN_D_CMD_RSIZE; i++) 1310 dma->cmdr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1311 offsetof(struct hifn_dma, command_bufs[i][0])); 1312 for (i = 0; i < HIFN_D_RES_RSIZE; i++) 1313 dma->resr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1314 offsetof(struct hifn_dma, result_bufs[i][0])); 1315 1316 dma->cmdr[HIFN_D_CMD_RSIZE].p = 1317 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1318 offsetof(struct hifn_dma, cmdr[0])); 1319 dma->srcr[HIFN_D_SRC_RSIZE].p = 1320 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1321 offsetof(struct hifn_dma, srcr[0])); 1322 dma->dstr[HIFN_D_DST_RSIZE].p = 1323 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1324 offsetof(struct hifn_dma, dstr[0])); 1325 dma->resr[HIFN_D_RES_RSIZE].p = 1326 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1327 offsetof(struct hifn_dma, resr[0])); 1328 1329 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 1330 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 1331 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 1332 } 1333 1334 /* 1335 * Writes out the raw command buffer space. Returns the 1336 * command buffer size. 1337 */ 1338 static u_int 1339 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf) 1340 { 1341 u_int8_t *buf_pos; 1342 struct hifn_base_command *base_cmd; 1343 struct hifn_mac_command *mac_cmd; 1344 struct hifn_crypt_command *cry_cmd; 1345 struct hifn_comp_command *comp_cmd; 1346 int using_mac, using_crypt, using_comp, len, ivlen; 1347 u_int32_t dlen, slen; 1348 1349 buf_pos = buf; 1350 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC; 1351 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT; 1352 using_comp = cmd->base_masks & HIFN_BASE_CMD_COMP; 1353 1354 base_cmd = (struct hifn_base_command *)buf_pos; 1355 base_cmd->masks = htole16(cmd->base_masks); 1356 slen = cmd->src_map->dm_mapsize; 1357 if (cmd->sloplen) 1358 dlen = cmd->dst_map->dm_mapsize - cmd->sloplen + 1359 sizeof(u_int32_t); 1360 else 1361 dlen = cmd->dst_map->dm_mapsize; 1362 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO); 1363 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1364 dlen >>= 16; 1365 slen >>= 16; 1366 base_cmd->session_num = htole16(cmd->session_num | 1367 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1368 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1369 buf_pos += sizeof(struct hifn_base_command); 1370 1371 if (using_comp) { 1372 comp_cmd = (struct hifn_comp_command *)buf_pos; 1373 dlen = cmd->compcrd->crd_len; 1374 comp_cmd->source_count = htole16(dlen & 0xffff); 1375 dlen >>= 16; 1376 comp_cmd->masks = htole16(cmd->comp_masks | 1377 ((dlen << HIFN_COMP_CMD_SRCLEN_S) & HIFN_COMP_CMD_SRCLEN_M)); 1378 comp_cmd->header_skip = htole16(cmd->compcrd->crd_skip); 1379 comp_cmd->reserved = 0; 1380 buf_pos += sizeof(struct hifn_comp_command); 1381 } 1382 1383 if (using_mac) { 1384 mac_cmd = (struct hifn_mac_command *)buf_pos; 1385 dlen = cmd->maccrd->crd_len; 1386 mac_cmd->source_count = htole16(dlen & 0xffff); 1387 dlen >>= 16; 1388 mac_cmd->masks = htole16(cmd->mac_masks | 1389 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M)); 1390 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip); 1391 mac_cmd->reserved = 0; 1392 buf_pos += sizeof(struct hifn_mac_command); 1393 } 1394 1395 if (using_crypt) { 1396 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1397 dlen = cmd->enccrd->crd_len; 1398 cry_cmd->source_count = htole16(dlen & 0xffff); 1399 dlen >>= 16; 1400 cry_cmd->masks = htole16(cmd->cry_masks | 1401 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M)); 1402 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip); 1403 cry_cmd->reserved = 0; 1404 buf_pos += sizeof(struct hifn_crypt_command); 1405 } 1406 1407 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) { 1408 memcpy(buf_pos, cmd->mac, HIFN_MAC_KEY_LENGTH); 1409 buf_pos += HIFN_MAC_KEY_LENGTH; 1410 } 1411 1412 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) { 1413 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1414 case HIFN_CRYPT_CMD_ALG_3DES: 1415 memcpy(buf_pos, cmd->ck, HIFN_3DES_KEY_LENGTH); 1416 buf_pos += HIFN_3DES_KEY_LENGTH; 1417 break; 1418 case HIFN_CRYPT_CMD_ALG_DES: 1419 memcpy(buf_pos, cmd->ck, HIFN_DES_KEY_LENGTH); 1420 buf_pos += HIFN_DES_KEY_LENGTH; 1421 break; 1422 case HIFN_CRYPT_CMD_ALG_RC4: 1423 len = 256; 1424 do { 1425 int clen; 1426 1427 clen = MIN(cmd->cklen, len); 1428 memcpy(buf_pos, cmd->ck, clen); 1429 len -= clen; 1430 buf_pos += clen; 1431 } while (len > 0); 1432 memset(buf_pos, 0, 4); 1433 buf_pos += 4; 1434 break; 1435 case HIFN_CRYPT_CMD_ALG_AES: 1436 /* 1437 * AES keys are variable 128, 192 and 1438 * 256 bits (16, 24 and 32 bytes). 1439 */ 1440 memcpy(buf_pos, cmd->ck, cmd->cklen); 1441 buf_pos += cmd->cklen; 1442 break; 1443 } 1444 } 1445 1446 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) { 1447 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1448 case HIFN_CRYPT_CMD_ALG_AES: 1449 ivlen = HIFN_AES_IV_LENGTH; 1450 break; 1451 default: 1452 ivlen = HIFN_IV_LENGTH; 1453 break; 1454 } 1455 memcpy(buf_pos, cmd->iv, ivlen); 1456 buf_pos += ivlen; 1457 } 1458 1459 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC | HIFN_BASE_CMD_CRYPT | 1460 HIFN_BASE_CMD_COMP)) == 0) { 1461 memset(buf_pos, 0, 8); 1462 buf_pos += 8; 1463 } 1464 1465 return (buf_pos - buf); 1466 } 1467 1468 static int 1469 hifn_dmamap_aligned(bus_dmamap_t map) 1470 { 1471 int i; 1472 1473 for (i = 0; i < map->dm_nsegs; i++) { 1474 if (map->dm_segs[i].ds_addr & 3) 1475 return (0); 1476 if ((i != (map->dm_nsegs - 1)) && 1477 (map->dm_segs[i].ds_len & 3)) 1478 return (0); 1479 } 1480 return (1); 1481 } 1482 1483 static int 1484 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd) 1485 { 1486 struct hifn_dma *dma = sc->sc_dma; 1487 bus_dmamap_t map = cmd->dst_map; 1488 u_int32_t p, l; 1489 int idx, used = 0, i; 1490 1491 idx = dma->dsti; 1492 for (i = 0; i < map->dm_nsegs - 1; i++) { 1493 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1494 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1495 HIFN_D_MASKDONEIRQ | map->dm_segs[i].ds_len); 1496 HIFN_DSTR_SYNC(sc, idx, 1497 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1498 used++; 1499 1500 if (++idx == HIFN_D_DST_RSIZE) { 1501 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1502 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1503 HIFN_DSTR_SYNC(sc, idx, 1504 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1505 idx = 0; 1506 } 1507 } 1508 1509 if (cmd->sloplen == 0) { 1510 p = map->dm_segs[i].ds_addr; 1511 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1512 map->dm_segs[i].ds_len; 1513 } else { 1514 p = sc->sc_dmamap->dm_segs[0].ds_addr + 1515 offsetof(struct hifn_dma, slop[cmd->slopidx]); 1516 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1517 sizeof(u_int32_t); 1518 1519 if ((map->dm_segs[i].ds_len - cmd->sloplen) != 0) { 1520 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1521 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1522 HIFN_D_MASKDONEIRQ | 1523 (map->dm_segs[i].ds_len - cmd->sloplen)); 1524 HIFN_DSTR_SYNC(sc, idx, 1525 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1526 used++; 1527 1528 if (++idx == HIFN_D_DST_RSIZE) { 1529 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1530 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1531 HIFN_DSTR_SYNC(sc, idx, 1532 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1533 idx = 0; 1534 } 1535 } 1536 } 1537 dma->dstr[idx].p = htole32(p); 1538 dma->dstr[idx].l = htole32(l); 1539 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1540 used++; 1541 1542 if (++idx == HIFN_D_DST_RSIZE) { 1543 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP | 1544 HIFN_D_MASKDONEIRQ); 1545 HIFN_DSTR_SYNC(sc, idx, 1546 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1547 idx = 0; 1548 } 1549 1550 dma->dsti = idx; 1551 dma->dstu += used; 1552 return (idx); 1553 } 1554 1555 static int 1556 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd) 1557 { 1558 struct hifn_dma *dma = sc->sc_dma; 1559 bus_dmamap_t map = cmd->src_map; 1560 int idx, i; 1561 u_int32_t last = 0; 1562 1563 idx = dma->srci; 1564 for (i = 0; i < map->dm_nsegs; i++) { 1565 if (i == map->dm_nsegs - 1) 1566 last = HIFN_D_LAST; 1567 1568 dma->srcr[idx].p = htole32(map->dm_segs[i].ds_addr); 1569 dma->srcr[idx].l = htole32(map->dm_segs[i].ds_len | 1570 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last); 1571 HIFN_SRCR_SYNC(sc, idx, 1572 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1573 1574 if (++idx == HIFN_D_SRC_RSIZE) { 1575 dma->srcr[idx].l = htole32(HIFN_D_VALID | 1576 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1577 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1578 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1579 idx = 0; 1580 } 1581 } 1582 dma->srci = idx; 1583 dma->srcu += map->dm_nsegs; 1584 return (idx); 1585 } 1586 1587 static int 1588 hifn_crypto(struct hifn_softc *sc, struct hifn_command *cmd, 1589 struct cryptop *crp, int hint) 1590 { 1591 struct hifn_dma *dma = sc->sc_dma; 1592 u_int32_t cmdlen; 1593 int cmdi, resi, s, err = 0; 1594 1595 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 1596 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) 1597 return (ENOMEM); 1598 1599 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1600 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 1601 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 1602 err = ENOMEM; 1603 goto err_srcmap1; 1604 } 1605 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1606 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 1607 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 1608 err = ENOMEM; 1609 goto err_srcmap1; 1610 } 1611 } else { 1612 err = EINVAL; 1613 goto err_srcmap1; 1614 } 1615 1616 if (hifn_dmamap_aligned(cmd->src_map)) { 1617 cmd->sloplen = cmd->src_map->dm_mapsize & 3; 1618 if (crp->crp_flags & CRYPTO_F_IOV) 1619 cmd->dstu.dst_io = cmd->srcu.src_io; 1620 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1621 cmd->dstu.dst_m = cmd->srcu.src_m; 1622 cmd->dst_map = cmd->src_map; 1623 } else { 1624 if (crp->crp_flags & CRYPTO_F_IOV) { 1625 err = EINVAL; 1626 goto err_srcmap; 1627 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1628 int totlen, len; 1629 struct mbuf *m, *m0, *mlast; 1630 1631 totlen = cmd->src_map->dm_mapsize; 1632 if (cmd->srcu.src_m->m_flags & M_PKTHDR) { 1633 len = MHLEN; 1634 MGETHDR(m0, M_DONTWAIT, MT_DATA); 1635 } else { 1636 len = MLEN; 1637 MGET(m0, M_DONTWAIT, MT_DATA); 1638 } 1639 if (m0 == NULL) { 1640 err = ENOMEM; 1641 goto err_srcmap; 1642 } 1643 if (len == MHLEN) 1644 M_DUP_PKTHDR(m0, cmd->srcu.src_m); 1645 if (totlen >= MINCLSIZE) { 1646 MCLGET(m0, M_DONTWAIT); 1647 if (m0->m_flags & M_EXT) 1648 len = MCLBYTES; 1649 } 1650 totlen -= len; 1651 m0->m_pkthdr.len = m0->m_len = len; 1652 mlast = m0; 1653 1654 while (totlen > 0) { 1655 MGET(m, M_DONTWAIT, MT_DATA); 1656 if (m == NULL) { 1657 err = ENOMEM; 1658 m_freem(m0); 1659 goto err_srcmap; 1660 } 1661 len = MLEN; 1662 if (totlen >= MINCLSIZE) { 1663 MCLGET(m, M_DONTWAIT); 1664 if (m->m_flags & M_EXT) 1665 len = MCLBYTES; 1666 } 1667 1668 m->m_len = len; 1669 if (m0->m_flags & M_PKTHDR) 1670 m0->m_pkthdr.len += len; 1671 totlen -= len; 1672 1673 mlast->m_next = m; 1674 mlast = m; 1675 } 1676 cmd->dstu.dst_m = m0; 1677 } 1678 } 1679 1680 if (cmd->dst_map == NULL) { 1681 if (bus_dmamap_create(sc->sc_dmat, 1682 HIFN_MAX_SEGLEN * MAX_SCATTER, MAX_SCATTER, 1683 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 1684 err = ENOMEM; 1685 goto err_srcmap; 1686 } 1687 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1688 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 1689 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 1690 err = ENOMEM; 1691 goto err_dstmap1; 1692 } 1693 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1694 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 1695 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 1696 err = ENOMEM; 1697 goto err_dstmap1; 1698 } 1699 } 1700 } 1701 1702 #ifdef HIFN_DEBUG 1703 if (hifn_debug) 1704 printf("%s: Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n", 1705 device_xname(sc->sc_dv), 1706 READ_REG_1(sc, HIFN_1_DMA_CSR), 1707 READ_REG_1(sc, HIFN_1_DMA_IER), 1708 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1709 cmd->src_map->dm_nsegs, cmd->dst_map->dm_nsegs); 1710 #endif 1711 1712 if (cmd->src_map == cmd->dst_map) 1713 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1714 0, cmd->src_map->dm_mapsize, 1715 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 1716 else { 1717 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1718 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1719 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 1720 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 1721 } 1722 1723 s = splnet(); 1724 1725 /* 1726 * need 1 cmd, and 1 res 1727 * need N src, and N dst 1728 */ 1729 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 1730 (dma->resu + 1) > HIFN_D_RES_RSIZE) { 1731 splx(s); 1732 err = ENOMEM; 1733 goto err_dstmap; 1734 } 1735 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 1736 (dma->dstu + cmd->dst_map->dm_nsegs + 1) > HIFN_D_DST_RSIZE) { 1737 splx(s); 1738 err = ENOMEM; 1739 goto err_dstmap; 1740 } 1741 1742 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1743 dma->cmdi = 0; 1744 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1745 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1746 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1747 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1748 } 1749 cmdi = dma->cmdi++; 1750 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 1751 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 1752 1753 /* .p for command/result already set */ 1754 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 1755 HIFN_D_MASKDONEIRQ); 1756 HIFN_CMDR_SYNC(sc, cmdi, 1757 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1758 dma->cmdu++; 1759 if (sc->sc_c_busy == 0) { 1760 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1761 sc->sc_c_busy = 1; 1762 SET_LED(sc, HIFN_MIPSRST_LED0); 1763 } 1764 1765 /* 1766 * We don't worry about missing an interrupt (which a "command wait" 1767 * interrupt salvages us from), unless there is more than one command 1768 * in the queue. 1769 * 1770 * XXX We do seem to miss some interrupts. So we always enable 1771 * XXX command wait. From OpenBSD revision 1.149. 1772 * 1773 */ 1774 #if 0 1775 if (dma->cmdu > 1) { 1776 #endif 1777 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 1778 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1779 #if 0 1780 } 1781 #endif 1782 1783 hifnstats.hst_ipackets++; 1784 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 1785 1786 hifn_dmamap_load_src(sc, cmd); 1787 if (sc->sc_s_busy == 0) { 1788 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1789 sc->sc_s_busy = 1; 1790 SET_LED(sc, HIFN_MIPSRST_LED1); 1791 } 1792 1793 /* 1794 * Unlike other descriptors, we don't mask done interrupt from 1795 * result descriptor. 1796 */ 1797 #ifdef HIFN_DEBUG 1798 if (hifn_debug) 1799 printf("load res\n"); 1800 #endif 1801 if (dma->resi == HIFN_D_RES_RSIZE) { 1802 dma->resi = 0; 1803 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1804 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1805 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1806 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1807 } 1808 resi = dma->resi++; 1809 dma->hifn_commands[resi] = cmd; 1810 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 1811 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 1812 HIFN_D_VALID | HIFN_D_LAST); 1813 HIFN_RESR_SYNC(sc, resi, 1814 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1815 dma->resu++; 1816 if (sc->sc_r_busy == 0) { 1817 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1818 sc->sc_r_busy = 1; 1819 SET_LED(sc, HIFN_MIPSRST_LED2); 1820 } 1821 1822 if (cmd->sloplen) 1823 cmd->slopidx = resi; 1824 1825 hifn_dmamap_load_dst(sc, cmd); 1826 1827 if (sc->sc_d_busy == 0) { 1828 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1829 sc->sc_d_busy = 1; 1830 } 1831 1832 #ifdef HIFN_DEBUG 1833 if (hifn_debug) 1834 printf("%s: command: stat %8x ier %8x\n", 1835 device_xname(sc->sc_dv), 1836 READ_REG_1(sc, HIFN_1_DMA_CSR), READ_REG_1(sc, HIFN_1_DMA_IER)); 1837 #endif 1838 1839 sc->sc_active = 5; 1840 splx(s); 1841 return (err); /* success */ 1842 1843 err_dstmap: 1844 if (cmd->src_map != cmd->dst_map) 1845 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 1846 err_dstmap1: 1847 if (cmd->src_map != cmd->dst_map) 1848 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 1849 err_srcmap: 1850 if (crp->crp_flags & CRYPTO_F_IMBUF && 1851 cmd->srcu.src_m != cmd->dstu.dst_m) 1852 m_freem(cmd->dstu.dst_m); 1853 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 1854 err_srcmap1: 1855 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 1856 return (err); 1857 } 1858 1859 static void 1860 hifn_tick(void *vsc) 1861 { 1862 struct hifn_softc *sc = vsc; 1863 int s; 1864 1865 s = splnet(); 1866 if (sc->sc_active == 0) { 1867 struct hifn_dma *dma = sc->sc_dma; 1868 u_int32_t r = 0; 1869 1870 if (dma->cmdu == 0 && sc->sc_c_busy) { 1871 sc->sc_c_busy = 0; 1872 r |= HIFN_DMACSR_C_CTRL_DIS; 1873 CLR_LED(sc, HIFN_MIPSRST_LED0); 1874 } 1875 if (dma->srcu == 0 && sc->sc_s_busy) { 1876 sc->sc_s_busy = 0; 1877 r |= HIFN_DMACSR_S_CTRL_DIS; 1878 CLR_LED(sc, HIFN_MIPSRST_LED1); 1879 } 1880 if (dma->dstu == 0 && sc->sc_d_busy) { 1881 sc->sc_d_busy = 0; 1882 r |= HIFN_DMACSR_D_CTRL_DIS; 1883 } 1884 if (dma->resu == 0 && sc->sc_r_busy) { 1885 sc->sc_r_busy = 0; 1886 r |= HIFN_DMACSR_R_CTRL_DIS; 1887 CLR_LED(sc, HIFN_MIPSRST_LED2); 1888 } 1889 if (r) 1890 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r); 1891 } 1892 else 1893 sc->sc_active--; 1894 splx(s); 1895 #ifdef __OpenBSD__ 1896 timeout_add(&sc->sc_tickto, hz); 1897 #else 1898 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 1899 #endif 1900 } 1901 1902 static int 1903 hifn_intr(void *arg) 1904 { 1905 struct hifn_softc *sc = arg; 1906 struct hifn_dma *dma = sc->sc_dma; 1907 u_int32_t dmacsr, restart; 1908 int i, u; 1909 1910 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR); 1911 1912 #ifdef HIFN_DEBUG 1913 if (hifn_debug) 1914 printf("%s: irq: stat %08x ien %08x u %d/%d/%d/%d\n", 1915 device_xname(sc->sc_dv), 1916 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), 1917 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1918 #endif 1919 1920 /* Nothing in the DMA unit interrupted */ 1921 if ((dmacsr & sc->sc_dmaier) == 0) 1922 return (0); 1923 1924 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier); 1925 1926 if (dmacsr & HIFN_DMACSR_ENGINE) 1927 WRITE_REG_0(sc, HIFN_0_PUISR, READ_REG_0(sc, HIFN_0_PUISR)); 1928 1929 if ((sc->sc_flags & HIFN_HAS_PUBLIC) && 1930 (dmacsr & HIFN_DMACSR_PUBDONE)) 1931 WRITE_REG_1(sc, HIFN_1_PUB_STATUS, 1932 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1933 1934 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1935 if (restart) 1936 printf("%s: overrun %x\n", device_xname(sc->sc_dv), dmacsr); 1937 1938 if (sc->sc_flags & HIFN_IS_7811) { 1939 if (dmacsr & HIFN_DMACSR_ILLR) 1940 printf("%s: illegal read\n", device_xname(sc->sc_dv)); 1941 if (dmacsr & HIFN_DMACSR_ILLW) 1942 printf("%s: illegal write\n", device_xname(sc->sc_dv)); 1943 } 1944 1945 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1946 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1947 if (restart) { 1948 printf("%s: abort, resetting.\n", device_xname(sc->sc_dv)); 1949 hifnstats.hst_abort++; 1950 hifn_abort(sc); 1951 return (1); 1952 } 1953 1954 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->resu == 0)) { 1955 /* 1956 * If no slots to process and we receive a "waiting on 1957 * command" interrupt, we disable the "waiting on command" 1958 * (by clearing it). 1959 */ 1960 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 1961 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1962 } 1963 1964 /* clear the rings */ 1965 i = dma->resk; 1966 while (dma->resu != 0) { 1967 HIFN_RESR_SYNC(sc, i, 1968 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1969 if (dma->resr[i].l & htole32(HIFN_D_VALID)) { 1970 HIFN_RESR_SYNC(sc, i, 1971 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1972 break; 1973 } 1974 1975 if (i != HIFN_D_RES_RSIZE) { 1976 struct hifn_command *cmd; 1977 1978 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD); 1979 cmd = dma->hifn_commands[i]; 1980 KASSERT(cmd != NULL 1981 /*("hifn_intr: null command slot %u", i)*/); 1982 dma->hifn_commands[i] = NULL; 1983 1984 hifn_callback(sc, cmd, dma->result_bufs[i]); 1985 hifnstats.hst_opackets++; 1986 } 1987 1988 if (++i == (HIFN_D_RES_RSIZE + 1)) 1989 i = 0; 1990 else 1991 dma->resu--; 1992 } 1993 dma->resk = i; 1994 1995 i = dma->srck; u = dma->srcu; 1996 while (u != 0) { 1997 HIFN_SRCR_SYNC(sc, i, 1998 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1999 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) { 2000 HIFN_SRCR_SYNC(sc, i, 2001 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2002 break; 2003 } 2004 if (++i == (HIFN_D_SRC_RSIZE + 1)) 2005 i = 0; 2006 else 2007 u--; 2008 } 2009 dma->srck = i; dma->srcu = u; 2010 2011 i = dma->cmdk; u = dma->cmdu; 2012 while (u != 0) { 2013 HIFN_CMDR_SYNC(sc, i, 2014 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2015 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) { 2016 HIFN_CMDR_SYNC(sc, i, 2017 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2018 break; 2019 } 2020 if (i != HIFN_D_CMD_RSIZE) { 2021 u--; 2022 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE); 2023 } 2024 if (++i == (HIFN_D_CMD_RSIZE + 1)) 2025 i = 0; 2026 } 2027 dma->cmdk = i; dma->cmdu = u; 2028 2029 return (1); 2030 } 2031 2032 /* 2033 * Allocate a new 'session' and return an encoded session id. 'sidp' 2034 * contains our registration id, and should contain an encoded session 2035 * id on successful allocation. 2036 */ 2037 static int 2038 hifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 2039 { 2040 struct cryptoini *c; 2041 struct hifn_softc *sc = arg; 2042 int i, mac = 0, cry = 0, comp = 0; 2043 2044 KASSERT(sc != NULL /*, ("hifn_newsession: null softc")*/); 2045 if (sidp == NULL || cri == NULL || sc == NULL) 2046 return (EINVAL); 2047 2048 for (i = 0; i < sc->sc_maxses; i++) 2049 if (sc->sc_sessions[i].hs_state == HS_STATE_FREE) 2050 break; 2051 if (i == sc->sc_maxses) 2052 return (ENOMEM); 2053 2054 for (c = cri; c != NULL; c = c->cri_next) { 2055 switch (c->cri_alg) { 2056 case CRYPTO_MD5: 2057 case CRYPTO_SHA1: 2058 case CRYPTO_MD5_HMAC_96: 2059 case CRYPTO_SHA1_HMAC_96: 2060 if (mac) 2061 return (EINVAL); 2062 mac = 1; 2063 break; 2064 case CRYPTO_DES_CBC: 2065 case CRYPTO_3DES_CBC: 2066 case CRYPTO_AES_CBC: 2067 /* Note that this is an initialization 2068 vector, not a cipher key; any function 2069 giving sufficient Hamming distance 2070 between outputs is fine. Use of RC4 2071 to generate IVs has been FIPS140-2 2072 certified by several labs. */ 2073 #ifdef __NetBSD__ 2074 cprng_fast(sc->sc_sessions[i].hs_iv, 2075 c->cri_alg == CRYPTO_AES_CBC ? 2076 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2077 #else /* FreeBSD and OpenBSD have get_random_bytes */ 2078 /* XXX this may read fewer, does it matter? */ 2079 get_random_bytes(sc->sc_sessions[i].hs_iv, 2080 c->cri_alg == CRYPTO_AES_CBC ? 2081 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2082 #endif 2083 /*FALLTHROUGH*/ 2084 case CRYPTO_ARC4: 2085 if (cry) 2086 return (EINVAL); 2087 cry = 1; 2088 break; 2089 #ifdef HAVE_CRYPTO_LZS 2090 case CRYPTO_LZS_COMP: 2091 if (comp) 2092 return (EINVAL); 2093 comp = 1; 2094 break; 2095 #endif 2096 default: 2097 return (EINVAL); 2098 } 2099 } 2100 if (mac == 0 && cry == 0 && comp == 0) 2101 return (EINVAL); 2102 2103 /* 2104 * XXX only want to support compression without chaining to 2105 * MAC/crypt engine right now 2106 */ 2107 if ((comp && mac) || (comp && cry)) 2108 return (EINVAL); 2109 2110 *sidp = HIFN_SID(device_unit(sc->sc_dv), i); 2111 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2112 2113 return (0); 2114 } 2115 2116 /* 2117 * Deallocate a session. 2118 * XXX this routine should run a zero'd mac/encrypt key into context ram. 2119 * XXX to blow away any keys already stored there. 2120 */ 2121 static int 2122 hifn_freesession(void *arg, u_int64_t tid) 2123 { 2124 struct hifn_softc *sc = arg; 2125 int session; 2126 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 2127 2128 KASSERT(sc != NULL /*, ("hifn_freesession: null softc")*/); 2129 if (sc == NULL) 2130 return (EINVAL); 2131 2132 session = HIFN_SESSION(sid); 2133 if (session >= sc->sc_maxses) 2134 return (EINVAL); 2135 2136 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 2137 return (0); 2138 } 2139 2140 static int 2141 hifn_process(void *arg, struct cryptop *crp, int hint) 2142 { 2143 struct hifn_softc *sc = arg; 2144 struct hifn_command *cmd = NULL; 2145 int session, err, ivlen; 2146 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 2147 2148 if (crp == NULL || crp->crp_callback == NULL) { 2149 hifnstats.hst_invalid++; 2150 return (EINVAL); 2151 } 2152 session = HIFN_SESSION(crp->crp_sid); 2153 2154 if (sc == NULL || session >= sc->sc_maxses) { 2155 err = EINVAL; 2156 goto errout; 2157 } 2158 2159 cmd = (struct hifn_command *)malloc(sizeof(struct hifn_command), 2160 M_DEVBUF, M_NOWAIT|M_ZERO); 2161 if (cmd == NULL) { 2162 hifnstats.hst_nomem++; 2163 err = ENOMEM; 2164 goto errout; 2165 } 2166 2167 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2168 cmd->srcu.src_m = (struct mbuf *)crp->crp_buf; 2169 cmd->dstu.dst_m = (struct mbuf *)crp->crp_buf; 2170 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2171 cmd->srcu.src_io = (struct uio *)crp->crp_buf; 2172 cmd->dstu.dst_io = (struct uio *)crp->crp_buf; 2173 } else { 2174 err = EINVAL; 2175 goto errout; /* XXX we don't handle contiguous buffers! */ 2176 } 2177 2178 crd1 = crp->crp_desc; 2179 if (crd1 == NULL) { 2180 err = EINVAL; 2181 goto errout; 2182 } 2183 crd2 = crd1->crd_next; 2184 2185 if (crd2 == NULL) { 2186 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2187 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2188 crd1->crd_alg == CRYPTO_SHA1 || 2189 crd1->crd_alg == CRYPTO_MD5) { 2190 maccrd = crd1; 2191 enccrd = NULL; 2192 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 2193 crd1->crd_alg == CRYPTO_3DES_CBC || 2194 crd1->crd_alg == CRYPTO_AES_CBC || 2195 crd1->crd_alg == CRYPTO_ARC4) { 2196 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0) 2197 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2198 maccrd = NULL; 2199 enccrd = crd1; 2200 #ifdef HAVE_CRYPTO_LZS 2201 } else if (crd1->crd_alg == CRYPTO_LZS_COMP) { 2202 return (hifn_compression(sc, crp, cmd)); 2203 #endif 2204 } else { 2205 err = EINVAL; 2206 goto errout; 2207 } 2208 } else { 2209 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2210 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2211 crd1->crd_alg == CRYPTO_MD5 || 2212 crd1->crd_alg == CRYPTO_SHA1) && 2213 (crd2->crd_alg == CRYPTO_DES_CBC || 2214 crd2->crd_alg == CRYPTO_3DES_CBC || 2215 crd2->crd_alg == CRYPTO_AES_CBC || 2216 crd2->crd_alg == CRYPTO_ARC4) && 2217 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 2218 cmd->base_masks = HIFN_BASE_CMD_DECODE; 2219 maccrd = crd1; 2220 enccrd = crd2; 2221 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 2222 crd1->crd_alg == CRYPTO_ARC4 || 2223 crd1->crd_alg == CRYPTO_3DES_CBC || 2224 crd1->crd_alg == CRYPTO_AES_CBC) && 2225 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 2226 crd2->crd_alg == CRYPTO_SHA1_HMAC_96 || 2227 crd2->crd_alg == CRYPTO_MD5 || 2228 crd2->crd_alg == CRYPTO_SHA1) && 2229 (crd1->crd_flags & CRD_F_ENCRYPT)) { 2230 enccrd = crd1; 2231 maccrd = crd2; 2232 } else { 2233 /* 2234 * We cannot order the 7751 as requested 2235 */ 2236 err = EINVAL; 2237 goto errout; 2238 } 2239 } 2240 2241 if (enccrd) { 2242 cmd->enccrd = enccrd; 2243 cmd->base_masks |= HIFN_BASE_CMD_CRYPT; 2244 switch (enccrd->crd_alg) { 2245 case CRYPTO_ARC4: 2246 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4; 2247 if ((enccrd->crd_flags & CRD_F_ENCRYPT) 2248 != sc->sc_sessions[session].hs_prev_op) 2249 sc->sc_sessions[session].hs_state = 2250 HS_STATE_USED; 2251 break; 2252 case CRYPTO_DES_CBC: 2253 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES | 2254 HIFN_CRYPT_CMD_MODE_CBC | 2255 HIFN_CRYPT_CMD_NEW_IV; 2256 break; 2257 case CRYPTO_3DES_CBC: 2258 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES | 2259 HIFN_CRYPT_CMD_MODE_CBC | 2260 HIFN_CRYPT_CMD_NEW_IV; 2261 break; 2262 case CRYPTO_AES_CBC: 2263 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES | 2264 HIFN_CRYPT_CMD_MODE_CBC | 2265 HIFN_CRYPT_CMD_NEW_IV; 2266 break; 2267 default: 2268 err = EINVAL; 2269 goto errout; 2270 } 2271 if (enccrd->crd_alg != CRYPTO_ARC4) { 2272 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? 2273 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2274 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 2275 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2276 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2277 else 2278 bcopy(sc->sc_sessions[session].hs_iv, 2279 cmd->iv, ivlen); 2280 2281 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) 2282 == 0) { 2283 if (crp->crp_flags & CRYPTO_F_IMBUF) 2284 m_copyback(cmd->srcu.src_m, 2285 enccrd->crd_inject, 2286 ivlen, cmd->iv); 2287 else if (crp->crp_flags & CRYPTO_F_IOV) 2288 cuio_copyback(cmd->srcu.src_io, 2289 enccrd->crd_inject, 2290 ivlen, cmd->iv); 2291 } 2292 } else { 2293 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2294 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2295 else if (crp->crp_flags & CRYPTO_F_IMBUF) 2296 m_copydata(cmd->srcu.src_m, 2297 enccrd->crd_inject, ivlen, cmd->iv); 2298 else if (crp->crp_flags & CRYPTO_F_IOV) 2299 cuio_copydata(cmd->srcu.src_io, 2300 enccrd->crd_inject, ivlen, cmd->iv); 2301 } 2302 } 2303 2304 cmd->ck = enccrd->crd_key; 2305 cmd->cklen = enccrd->crd_klen >> 3; 2306 2307 /* 2308 * Need to specify the size for the AES key in the masks. 2309 */ 2310 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) == 2311 HIFN_CRYPT_CMD_ALG_AES) { 2312 switch (cmd->cklen) { 2313 case 16: 2314 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128; 2315 break; 2316 case 24: 2317 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192; 2318 break; 2319 case 32: 2320 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256; 2321 break; 2322 default: 2323 err = EINVAL; 2324 goto errout; 2325 } 2326 } 2327 2328 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2329 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; 2330 } 2331 2332 if (maccrd) { 2333 cmd->maccrd = maccrd; 2334 cmd->base_masks |= HIFN_BASE_CMD_MAC; 2335 2336 switch (maccrd->crd_alg) { 2337 case CRYPTO_MD5: 2338 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2339 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2340 HIFN_MAC_CMD_POS_IPSEC; 2341 break; 2342 case CRYPTO_MD5_HMAC_96: 2343 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2344 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2345 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2346 break; 2347 case CRYPTO_SHA1: 2348 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2349 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2350 HIFN_MAC_CMD_POS_IPSEC; 2351 break; 2352 case CRYPTO_SHA1_HMAC_96: 2353 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2354 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2355 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2356 break; 2357 } 2358 2359 if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC_96 || 2360 maccrd->crd_alg == CRYPTO_MD5_HMAC_96) && 2361 sc->sc_sessions[session].hs_state == HS_STATE_USED) { 2362 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY; 2363 memcpy(cmd->mac, maccrd->crd_key, maccrd->crd_klen >> 3); 2364 memset(cmd->mac + (maccrd->crd_klen >> 3), 0, 2365 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3)); 2366 } 2367 } 2368 2369 cmd->crp = crp; 2370 cmd->session_num = session; 2371 cmd->softc = sc; 2372 2373 err = hifn_crypto(sc, cmd, crp, hint); 2374 if (err == 0) { 2375 if (enccrd) 2376 sc->sc_sessions[session].hs_prev_op = 2377 enccrd->crd_flags & CRD_F_ENCRYPT; 2378 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2379 sc->sc_sessions[session].hs_state = HS_STATE_KEY; 2380 return 0; 2381 } else if (err == ERESTART) { 2382 /* 2383 * There weren't enough resources to dispatch the request 2384 * to the part. Notify the caller so they'll requeue this 2385 * request and resubmit it again soon. 2386 */ 2387 #ifdef HIFN_DEBUG 2388 if (hifn_debug) 2389 printf("%s: requeue request\n", device_xname(sc->sc_dv)); 2390 #endif 2391 free(cmd, M_DEVBUF); 2392 sc->sc_needwakeup |= CRYPTO_SYMQ; 2393 return (err); 2394 } 2395 2396 errout: 2397 if (cmd != NULL) 2398 free(cmd, M_DEVBUF); 2399 if (err == EINVAL) 2400 hifnstats.hst_invalid++; 2401 else 2402 hifnstats.hst_nomem++; 2403 crp->crp_etype = err; 2404 crypto_done(crp); 2405 return (0); 2406 } 2407 2408 static void 2409 hifn_abort(struct hifn_softc *sc) 2410 { 2411 struct hifn_dma *dma = sc->sc_dma; 2412 struct hifn_command *cmd; 2413 struct cryptop *crp; 2414 int i, u; 2415 2416 i = dma->resk; u = dma->resu; 2417 while (u != 0) { 2418 cmd = dma->hifn_commands[i]; 2419 KASSERT(cmd != NULL /*, ("hifn_abort: null cmd slot %u", i)*/); 2420 dma->hifn_commands[i] = NULL; 2421 crp = cmd->crp; 2422 2423 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) { 2424 /* Salvage what we can. */ 2425 hifnstats.hst_opackets++; 2426 hifn_callback(sc, cmd, dma->result_bufs[i]); 2427 } else { 2428 if (cmd->src_map == cmd->dst_map) { 2429 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2430 0, cmd->src_map->dm_mapsize, 2431 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2432 } else { 2433 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2434 0, cmd->src_map->dm_mapsize, 2435 BUS_DMASYNC_POSTWRITE); 2436 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2437 0, cmd->dst_map->dm_mapsize, 2438 BUS_DMASYNC_POSTREAD); 2439 } 2440 2441 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2442 m_freem(cmd->srcu.src_m); 2443 crp->crp_buf = (void *)cmd->dstu.dst_m; 2444 } 2445 2446 /* non-shared buffers cannot be restarted */ 2447 if (cmd->src_map != cmd->dst_map) { 2448 /* 2449 * XXX should be EAGAIN, delayed until 2450 * after the reset. 2451 */ 2452 crp->crp_etype = ENOMEM; 2453 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2454 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2455 } else 2456 crp->crp_etype = ENOMEM; 2457 2458 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2459 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2460 2461 free(cmd, M_DEVBUF); 2462 if (crp->crp_etype != EAGAIN) 2463 crypto_done(crp); 2464 } 2465 2466 if (++i == HIFN_D_RES_RSIZE) 2467 i = 0; 2468 u--; 2469 } 2470 dma->resk = i; dma->resu = u; 2471 2472 /* Force upload of key next time */ 2473 for (i = 0; i < sc->sc_maxses; i++) 2474 if (sc->sc_sessions[i].hs_state == HS_STATE_KEY) 2475 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2476 2477 hifn_reset_board(sc, 1); 2478 hifn_init_dma(sc); 2479 hifn_init_pci_registers(sc); 2480 } 2481 2482 static void 2483 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *resbuf) 2484 { 2485 struct hifn_dma *dma = sc->sc_dma; 2486 struct cryptop *crp = cmd->crp; 2487 struct cryptodesc *crd; 2488 struct mbuf *m; 2489 int totlen, i, u, ivlen; 2490 2491 if (cmd->src_map == cmd->dst_map) 2492 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2493 0, cmd->src_map->dm_mapsize, 2494 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2495 else { 2496 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2497 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2498 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2499 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2500 } 2501 2502 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2503 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2504 crp->crp_buf = (void *)cmd->dstu.dst_m; 2505 totlen = cmd->src_map->dm_mapsize; 2506 for (m = cmd->dstu.dst_m; m != NULL; m = m->m_next) { 2507 if (totlen < m->m_len) { 2508 m->m_len = totlen; 2509 totlen = 0; 2510 } else 2511 totlen -= m->m_len; 2512 } 2513 cmd->dstu.dst_m->m_pkthdr.len = 2514 cmd->srcu.src_m->m_pkthdr.len; 2515 m_freem(cmd->srcu.src_m); 2516 } 2517 } 2518 2519 if (cmd->sloplen != 0) { 2520 if (crp->crp_flags & CRYPTO_F_IMBUF) 2521 m_copyback((struct mbuf *)crp->crp_buf, 2522 cmd->src_map->dm_mapsize - cmd->sloplen, 2523 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2524 else if (crp->crp_flags & CRYPTO_F_IOV) 2525 cuio_copyback((struct uio *)crp->crp_buf, 2526 cmd->src_map->dm_mapsize - cmd->sloplen, 2527 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2528 } 2529 2530 i = dma->dstk; u = dma->dstu; 2531 while (u != 0) { 2532 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2533 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2534 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2535 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2536 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2537 offsetof(struct hifn_dma, dstr[i]), 2538 sizeof(struct hifn_desc), 2539 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2540 break; 2541 } 2542 if (++i == (HIFN_D_DST_RSIZE + 1)) 2543 i = 0; 2544 else 2545 u--; 2546 } 2547 dma->dstk = i; dma->dstu = u; 2548 2549 hifnstats.hst_obytes += cmd->dst_map->dm_mapsize; 2550 2551 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) == 2552 HIFN_BASE_CMD_CRYPT) { 2553 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2554 if (crd->crd_alg != CRYPTO_DES_CBC && 2555 crd->crd_alg != CRYPTO_3DES_CBC && 2556 crd->crd_alg != CRYPTO_AES_CBC) 2557 continue; 2558 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? 2559 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2560 if (crp->crp_flags & CRYPTO_F_IMBUF) 2561 m_copydata((struct mbuf *)crp->crp_buf, 2562 crd->crd_skip + crd->crd_len - ivlen, 2563 ivlen, 2564 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2565 else if (crp->crp_flags & CRYPTO_F_IOV) { 2566 cuio_copydata((struct uio *)crp->crp_buf, 2567 crd->crd_skip + crd->crd_len - ivlen, 2568 ivlen, 2569 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2570 } 2571 /* XXX We do not handle contig data */ 2572 break; 2573 } 2574 } 2575 2576 if (cmd->base_masks & HIFN_BASE_CMD_MAC) { 2577 u_int8_t *macbuf; 2578 2579 macbuf = resbuf + sizeof(struct hifn_base_result); 2580 if (cmd->base_masks & HIFN_BASE_CMD_COMP) 2581 macbuf += sizeof(struct hifn_comp_result); 2582 macbuf += sizeof(struct hifn_mac_result); 2583 2584 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2585 int len; 2586 2587 if (crd->crd_alg == CRYPTO_MD5) 2588 len = 16; 2589 else if (crd->crd_alg == CRYPTO_SHA1) 2590 len = 20; 2591 else if (crd->crd_alg == CRYPTO_MD5_HMAC_96 || 2592 crd->crd_alg == CRYPTO_SHA1_HMAC_96) 2593 len = 12; 2594 else 2595 continue; 2596 2597 if (crp->crp_flags & CRYPTO_F_IMBUF) 2598 m_copyback((struct mbuf *)crp->crp_buf, 2599 crd->crd_inject, len, macbuf); 2600 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac) 2601 memcpy(crp->crp_mac, (void *)macbuf, len); 2602 break; 2603 } 2604 } 2605 2606 if (cmd->src_map != cmd->dst_map) { 2607 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2608 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2609 } 2610 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2611 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2612 free(cmd, M_DEVBUF); 2613 crypto_done(crp); 2614 } 2615 2616 #ifdef HAVE_CRYPTO_LZS 2617 2618 static int 2619 hifn_compression(struct hifn_softc *sc, struct cryptop *crp, 2620 struct hifn_command *cmd) 2621 { 2622 struct cryptodesc *crd = crp->crp_desc; 2623 int s, err = 0; 2624 2625 cmd->compcrd = crd; 2626 cmd->base_masks |= HIFN_BASE_CMD_COMP; 2627 2628 if ((crp->crp_flags & CRYPTO_F_IMBUF) == 0) { 2629 /* 2630 * XXX can only handle mbufs right now since we can 2631 * XXX dynamically resize them. 2632 */ 2633 err = EINVAL; 2634 return (ENOMEM); 2635 } 2636 2637 if ((crd->crd_flags & CRD_F_COMP) == 0) 2638 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2639 if (crd->crd_alg == CRYPTO_LZS_COMP) 2640 cmd->comp_masks |= HIFN_COMP_CMD_ALG_LZS | 2641 HIFN_COMP_CMD_CLEARHIST; 2642 2643 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2644 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) { 2645 err = ENOMEM; 2646 goto fail; 2647 } 2648 2649 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2650 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 2651 err = ENOMEM; 2652 goto fail; 2653 } 2654 2655 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2656 int len; 2657 2658 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 2659 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 2660 err = ENOMEM; 2661 goto fail; 2662 } 2663 2664 len = cmd->src_map->dm_mapsize / MCLBYTES; 2665 if ((cmd->src_map->dm_mapsize % MCLBYTES) != 0) 2666 len++; 2667 len *= MCLBYTES; 2668 2669 if ((crd->crd_flags & CRD_F_COMP) == 0) 2670 len *= 4; 2671 2672 if (len > HIFN_MAX_DMALEN) 2673 len = HIFN_MAX_DMALEN; 2674 2675 cmd->dstu.dst_m = hifn_mkmbuf_chain(len, cmd->srcu.src_m); 2676 if (cmd->dstu.dst_m == NULL) { 2677 err = ENOMEM; 2678 goto fail; 2679 } 2680 2681 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2682 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2683 err = ENOMEM; 2684 goto fail; 2685 } 2686 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2687 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 2688 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 2689 err = ENOMEM; 2690 goto fail; 2691 } 2692 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 2693 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 2694 err = ENOMEM; 2695 goto fail; 2696 } 2697 } 2698 2699 if (cmd->src_map == cmd->dst_map) 2700 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2701 0, cmd->src_map->dm_mapsize, 2702 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 2703 else { 2704 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2705 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2706 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2707 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2708 } 2709 2710 cmd->crp = crp; 2711 /* 2712 * Always use session 0. The modes of compression we use are 2713 * stateless and there is always at least one compression 2714 * context, zero. 2715 */ 2716 cmd->session_num = 0; 2717 cmd->softc = sc; 2718 2719 s = splnet(); 2720 err = hifn_compress_enter(sc, cmd); 2721 splx(s); 2722 2723 if (err != 0) 2724 goto fail; 2725 return (0); 2726 2727 fail: 2728 if (cmd->dst_map != NULL) { 2729 if (cmd->dst_map->dm_nsegs > 0) 2730 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2731 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2732 } 2733 if (cmd->src_map != NULL) { 2734 if (cmd->src_map->dm_nsegs > 0) 2735 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2736 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2737 } 2738 free(cmd, M_DEVBUF); 2739 if (err == EINVAL) 2740 hifnstats.hst_invalid++; 2741 else 2742 hifnstats.hst_nomem++; 2743 crp->crp_etype = err; 2744 crypto_done(crp); 2745 return (0); 2746 } 2747 2748 /* 2749 * must be called at splnet() 2750 */ 2751 static int 2752 hifn_compress_enter(struct hifn_softc *sc, struct hifn_command *cmd) 2753 { 2754 struct hifn_dma *dma = sc->sc_dma; 2755 int cmdi, resi; 2756 u_int32_t cmdlen; 2757 2758 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 2759 (dma->resu + 1) > HIFN_D_CMD_RSIZE) 2760 return (ENOMEM); 2761 2762 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 2763 (dma->dstu + cmd->dst_map->dm_nsegs) > HIFN_D_DST_RSIZE) 2764 return (ENOMEM); 2765 2766 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 2767 dma->cmdi = 0; 2768 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 2769 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2770 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 2771 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2772 } 2773 cmdi = dma->cmdi++; 2774 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 2775 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 2776 2777 /* .p for command/result already set */ 2778 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 2779 HIFN_D_MASKDONEIRQ); 2780 HIFN_CMDR_SYNC(sc, cmdi, 2781 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2782 dma->cmdu++; 2783 if (sc->sc_c_busy == 0) { 2784 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 2785 sc->sc_c_busy = 1; 2786 SET_LED(sc, HIFN_MIPSRST_LED0); 2787 } 2788 2789 /* 2790 * We don't worry about missing an interrupt (which a "command wait" 2791 * interrupt salvages us from), unless there is more than one command 2792 * in the queue. 2793 */ 2794 if (dma->cmdu > 1) { 2795 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 2796 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 2797 } 2798 2799 hifnstats.hst_ipackets++; 2800 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 2801 2802 hifn_dmamap_load_src(sc, cmd); 2803 if (sc->sc_s_busy == 0) { 2804 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 2805 sc->sc_s_busy = 1; 2806 SET_LED(sc, HIFN_MIPSRST_LED1); 2807 } 2808 2809 /* 2810 * Unlike other descriptors, we don't mask done interrupt from 2811 * result descriptor. 2812 */ 2813 if (dma->resi == HIFN_D_RES_RSIZE) { 2814 dma->resi = 0; 2815 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 2816 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2817 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 2818 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2819 } 2820 resi = dma->resi++; 2821 dma->hifn_commands[resi] = cmd; 2822 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 2823 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 2824 HIFN_D_VALID | HIFN_D_LAST); 2825 HIFN_RESR_SYNC(sc, resi, 2826 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2827 dma->resu++; 2828 if (sc->sc_r_busy == 0) { 2829 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 2830 sc->sc_r_busy = 1; 2831 SET_LED(sc, HIFN_MIPSRST_LED2); 2832 } 2833 2834 if (cmd->sloplen) 2835 cmd->slopidx = resi; 2836 2837 hifn_dmamap_load_dst(sc, cmd); 2838 2839 if (sc->sc_d_busy == 0) { 2840 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 2841 sc->sc_d_busy = 1; 2842 } 2843 sc->sc_active = 5; 2844 cmd->cmd_callback = hifn_callback_comp; 2845 return (0); 2846 } 2847 2848 static void 2849 hifn_callback_comp(struct hifn_softc *sc, struct hifn_command *cmd, 2850 u_int8_t *resbuf) 2851 { 2852 struct hifn_base_result baseres; 2853 struct cryptop *crp = cmd->crp; 2854 struct hifn_dma *dma = sc->sc_dma; 2855 struct mbuf *m; 2856 int err = 0, i, u; 2857 u_int32_t olen; 2858 bus_size_t dstsize; 2859 2860 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2861 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2862 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2863 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2864 2865 dstsize = cmd->dst_map->dm_mapsize; 2866 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2867 2868 memcpy(&baseres, resbuf, sizeof(struct hifn_base_result)); 2869 2870 i = dma->dstk; u = dma->dstu; 2871 while (u != 0) { 2872 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2873 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2874 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2875 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2876 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2877 offsetof(struct hifn_dma, dstr[i]), 2878 sizeof(struct hifn_desc), 2879 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2880 break; 2881 } 2882 if (++i == (HIFN_D_DST_RSIZE + 1)) 2883 i = 0; 2884 else 2885 u--; 2886 } 2887 dma->dstk = i; dma->dstu = u; 2888 2889 if (baseres.flags & htole16(HIFN_BASE_RES_DSTOVERRUN)) { 2890 bus_size_t xlen; 2891 2892 xlen = dstsize; 2893 2894 m_freem(cmd->dstu.dst_m); 2895 2896 if (xlen == HIFN_MAX_DMALEN) { 2897 /* We've done all we can. */ 2898 err = E2BIG; 2899 goto out; 2900 } 2901 2902 xlen += MCLBYTES; 2903 2904 if (xlen > HIFN_MAX_DMALEN) 2905 xlen = HIFN_MAX_DMALEN; 2906 2907 cmd->dstu.dst_m = hifn_mkmbuf_chain(xlen, 2908 cmd->srcu.src_m); 2909 if (cmd->dstu.dst_m == NULL) { 2910 err = ENOMEM; 2911 goto out; 2912 } 2913 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2914 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2915 err = ENOMEM; 2916 goto out; 2917 } 2918 2919 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2920 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2921 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2922 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2923 2924 /* already at splnet... */ 2925 err = hifn_compress_enter(sc, cmd); 2926 if (err != 0) 2927 goto out; 2928 return; 2929 } 2930 2931 olen = dstsize - (letoh16(baseres.dst_cnt) | 2932 (((letoh16(baseres.session) & HIFN_BASE_RES_DSTLEN_M) >> 2933 HIFN_BASE_RES_DSTLEN_S) << 16)); 2934 2935 crp->crp_olen = olen - cmd->compcrd->crd_skip; 2936 2937 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2938 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2939 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2940 2941 m = cmd->dstu.dst_m; 2942 if (m->m_flags & M_PKTHDR) 2943 m->m_pkthdr.len = olen; 2944 crp->crp_buf = (void *)m; 2945 for (; m != NULL; m = m->m_next) { 2946 if (olen >= m->m_len) 2947 olen -= m->m_len; 2948 else { 2949 m->m_len = olen; 2950 olen = 0; 2951 } 2952 } 2953 2954 m_freem(cmd->srcu.src_m); 2955 free(cmd, M_DEVBUF); 2956 crp->crp_etype = 0; 2957 crypto_done(crp); 2958 return; 2959 2960 out: 2961 if (cmd->dst_map != NULL) { 2962 if (cmd->src_map->dm_nsegs != 0) 2963 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2964 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2965 } 2966 if (cmd->src_map != NULL) { 2967 if (cmd->src_map->dm_nsegs != 0) 2968 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2969 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2970 } 2971 if (cmd->dstu.dst_m != NULL) 2972 m_freem(cmd->dstu.dst_m); 2973 free(cmd, M_DEVBUF); 2974 crp->crp_etype = err; 2975 crypto_done(crp); 2976 } 2977 2978 static struct mbuf * 2979 hifn_mkmbuf_chain(int totlen, struct mbuf *mtemplate) 2980 { 2981 int len; 2982 struct mbuf *m, *m0, *mlast; 2983 2984 if (mtemplate->m_flags & M_PKTHDR) { 2985 len = MHLEN; 2986 MGETHDR(m0, M_DONTWAIT, MT_DATA); 2987 } else { 2988 len = MLEN; 2989 MGET(m0, M_DONTWAIT, MT_DATA); 2990 } 2991 if (m0 == NULL) 2992 return (NULL); 2993 if (len == MHLEN) 2994 M_DUP_PKTHDR(m0, mtemplate); 2995 MCLGET(m0, M_DONTWAIT); 2996 if (!(m0->m_flags & M_EXT)) 2997 m_freem(m0); 2998 len = MCLBYTES; 2999 3000 totlen -= len; 3001 m0->m_pkthdr.len = m0->m_len = len; 3002 mlast = m0; 3003 3004 while (totlen > 0) { 3005 MGET(m, M_DONTWAIT, MT_DATA); 3006 if (m == NULL) { 3007 m_freem(m0); 3008 return (NULL); 3009 } 3010 MCLGET(m, M_DONTWAIT); 3011 if (!(m->m_flags & M_EXT)) { 3012 m_freem(m0); 3013 return (NULL); 3014 } 3015 len = MCLBYTES; 3016 m->m_len = len; 3017 if (m0->m_flags & M_PKTHDR) 3018 m0->m_pkthdr.len += len; 3019 totlen -= len; 3020 3021 mlast->m_next = m; 3022 mlast = m; 3023 } 3024 3025 return (m0); 3026 } 3027 #endif /* HAVE_CRYPTO_LZS */ 3028 3029 static void 3030 hifn_write_4(struct hifn_softc *sc, int reggrp, bus_size_t reg, u_int32_t val) 3031 { 3032 /* 3033 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0 3034 * and Group 1 registers; avoid conditions that could create 3035 * burst writes by doing a read in between the writes. 3036 */ 3037 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3038 if (sc->sc_waw_lastgroup == reggrp && 3039 sc->sc_waw_lastreg == reg - 4) { 3040 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID); 3041 } 3042 sc->sc_waw_lastgroup = reggrp; 3043 sc->sc_waw_lastreg = reg; 3044 } 3045 if (reggrp == 0) 3046 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val); 3047 else 3048 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val); 3049 3050 } 3051 3052 static u_int32_t 3053 hifn_read_4(struct hifn_softc *sc, int reggrp, bus_size_t reg) 3054 { 3055 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3056 sc->sc_waw_lastgroup = -1; 3057 sc->sc_waw_lastreg = 1; 3058 } 3059 if (reggrp == 0) 3060 return (bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg)); 3061 return (bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg)); 3062 } 3063
Indexes created Fri Sep 26 20:09:58 GMT 2025