hifn7751.c revision 1.41
1 /* $NetBSD: hifn7751.c,v 1.41 2009/04/18 14:58:03 tsutsui Exp $ */ 2 /* $FreeBSD: hifn7751.c,v 1.5.2.7 2003/10/08 23:52:00 sam Exp $ */ 3 /* $OpenBSD: hifn7751.c,v 1.140 2003/08/01 17:55:54 deraadt Exp $ */ 4 5 /* 6 * Invertex AEON / Hifn 7751 driver 7 * Copyright (c) 1999 Invertex Inc. All rights reserved. 8 * Copyright (c) 1999 Theo de Raadt 9 * Copyright (c) 2000-2001 Network Security Technologies, Inc. 10 * http://www.netsec.net 11 * Copyright (c) 2003 Hifn Inc. 12 * 13 * This driver is based on a previous driver by Invertex, for which they 14 * requested: Please send any comments, feedback, bug-fixes, or feature 15 * requests to software (at) invertex.com. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * 1. Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * 2. Redistributions in binary form must reproduce the above copyright 24 * notice, this list of conditions and the following disclaimer in the 25 * documentation and/or other materials provided with the distribution. 26 * 3. The name of the author may not be used to endorse or promote products 27 * derived from this software without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 30 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 31 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 32 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 33 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 34 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 38 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Effort sponsored in part by the Defense Advanced Research Projects 41 * Agency (DARPA) and Air Force Research Laboratory, Air Force 42 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 43 * 44 */ 45 46 /* 47 * Driver for various Hifn pre-HIPP encryption processors. 48 */ 49 50 #include <sys/cdefs.h> 51 __KERNEL_RCSID(0, "$NetBSD: hifn7751.c,v 1.41 2009/04/18 14:58:03 tsutsui Exp $"); 52 53 #include "rnd.h" 54 55 #if NRND == 0 56 #error hifn7751 requires rnd pseudo-devices 57 #endif 58 59 60 #include <sys/param.h> 61 #include <sys/systm.h> 62 #include <sys/proc.h> 63 #include <sys/errno.h> 64 #include <sys/malloc.h> 65 #include <sys/kernel.h> 66 #include <sys/mbuf.h> 67 #include <sys/device.h> 68 69 #include <uvm/uvm_extern.h> 70 71 72 #ifdef __OpenBSD__ 73 #include <crypto/crypto.h> 74 #include <dev/rndvar.h> 75 #else 76 #include <opencrypto/cryptodev.h> 77 #include <sys/rnd.h> 78 #endif 79 80 #include <dev/pci/pcireg.h> 81 #include <dev/pci/pcivar.h> 82 #include <dev/pci/pcidevs.h> 83 84 #include <dev/pci/hifn7751reg.h> 85 #include <dev/pci/hifn7751var.h> 86 87 #undef HIFN_DEBUG 88 89 #ifdef __NetBSD__ 90 #define M_DUP_PKTHDR M_COPY_PKTHDR /* XXX */ 91 #endif 92 93 #ifdef HIFN_DEBUG 94 extern int hifn_debug; /* patchable */ 95 int hifn_debug = 1; 96 #endif 97 98 #ifdef __OpenBSD__ 99 #define HAVE_CRYPTO_LZS /* OpenBSD OCF supports CRYPTO_COMP_LZS */ 100 #endif 101 102 /* 103 * Prototypes and count for the pci_device structure 104 */ 105 #ifdef __OpenBSD__ 106 static int hifn_probe((struct device *, void *, void *); 107 #else 108 static int hifn_probe(struct device *, struct cfdata *, void *); 109 #endif 110 static void hifn_attach(struct device *, struct device *, void *); 111 112 CFATTACH_DECL(hifn, sizeof(struct hifn_softc), 113 hifn_probe, hifn_attach, NULL, NULL); 114 115 #ifdef __OpenBSD__ 116 struct cfdriver hifn_cd = { 117 0, "hifn", DV_DULL 118 }; 119 #endif 120 121 static void hifn_reset_board(struct hifn_softc *, int); 122 static void hifn_reset_puc(struct hifn_softc *); 123 static void hifn_puc_wait(struct hifn_softc *); 124 static const char *hifn_enable_crypto(struct hifn_softc *, pcireg_t); 125 static void hifn_set_retry(struct hifn_softc *); 126 static void hifn_init_dma(struct hifn_softc *); 127 static void hifn_init_pci_registers(struct hifn_softc *); 128 static int hifn_sramsize(struct hifn_softc *); 129 static int hifn_dramsize(struct hifn_softc *); 130 static int hifn_ramtype(struct hifn_softc *); 131 static void hifn_sessions(struct hifn_softc *); 132 static int hifn_intr(void *); 133 static u_int hifn_write_command(struct hifn_command *, u_int8_t *); 134 static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt); 135 static int hifn_newsession(void*, u_int32_t *, struct cryptoini *); 136 static int hifn_freesession(void*, u_int64_t); 137 static int hifn_process(void*, struct cryptop *, int); 138 static void hifn_callback(struct hifn_softc *, struct hifn_command *, 139 u_int8_t *); 140 static int hifn_crypto(struct hifn_softc *, struct hifn_command *, 141 struct cryptop*, int); 142 static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *); 143 static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *); 144 static int hifn_dmamap_aligned(bus_dmamap_t); 145 static int hifn_dmamap_load_src(struct hifn_softc *, 146 struct hifn_command *); 147 static int hifn_dmamap_load_dst(struct hifn_softc *, 148 struct hifn_command *); 149 static int hifn_init_pubrng(struct hifn_softc *); 150 static void hifn_rng(void *); 151 static void hifn_tick(void *); 152 static void hifn_abort(struct hifn_softc *); 153 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, 154 int *); 155 static void hifn_write_4(struct hifn_softc *, int, bus_size_t, u_int32_t); 156 static u_int32_t hifn_read_4(struct hifn_softc *, int, bus_size_t); 157 #ifdef HAVE_CRYPTO_LZS 158 static int hifn_compression(struct hifn_softc *, struct cryptop *, 159 struct hifn_command *); 160 static struct mbuf *hifn_mkmbuf_chain(int, struct mbuf *); 161 static int hifn_compress_enter(struct hifn_softc *, struct hifn_command *); 162 static void hifn_callback_comp(struct hifn_softc *, struct hifn_command *, 163 u_int8_t *); 164 #endif /* HAVE_CRYPTO_LZS */ 165 166 167 struct hifn_stats hifnstats; 168 169 static const struct hifn_product { 170 pci_vendor_id_t hifn_vendor; 171 pci_product_id_t hifn_product; 172 int hifn_flags; 173 const char *hifn_name; 174 } hifn_products[] = { 175 { PCI_VENDOR_INVERTEX, PCI_PRODUCT_INVERTEX_AEON, 176 0, 177 "Invertex AEON", 178 }, 179 180 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7751, 181 0, 182 "Hifn 7751", 183 }, 184 { PCI_VENDOR_NETSEC, PCI_PRODUCT_NETSEC_7751, 185 0, 186 "Hifn 7751 (NetSec)" 187 }, 188 189 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7811, 190 HIFN_IS_7811 | HIFN_HAS_RNG | HIFN_HAS_LEDS | HIFN_NO_BURSTWRITE, 191 "Hifn 7811", 192 }, 193 194 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7951, 195 HIFN_HAS_RNG | HIFN_HAS_PUBLIC, 196 "Hifn 7951", 197 }, 198 199 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7955, 200 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 201 "Hifn 7955", 202 }, 203 204 { PCI_VENDOR_HIFN, PCI_PRODUCT_HIFN_7956, 205 HIFN_HAS_RNG | HIFN_HAS_PUBLIC | HIFN_IS_7956 | HIFN_HAS_AES, 206 "Hifn 7956", 207 }, 208 209 210 { 0, 0, 211 0, 212 NULL 213 } 214 }; 215 216 static const struct hifn_product * 217 hifn_lookup(const struct pci_attach_args *pa) 218 { 219 const struct hifn_product *hp; 220 221 for (hp = hifn_products; hp->hifn_name != NULL; hp++) { 222 if (PCI_VENDOR(pa->pa_id) == hp->hifn_vendor && 223 PCI_PRODUCT(pa->pa_id) == hp->hifn_product) 224 return (hp); 225 } 226 return (NULL); 227 } 228 229 static int 230 hifn_probe(struct device *parent, struct cfdata *match, 231 void *aux) 232 { 233 struct pci_attach_args *pa = (struct pci_attach_args *) aux; 234 235 if (hifn_lookup(pa) != NULL) 236 return (1); 237 238 return (0); 239 } 240 241 static void 242 hifn_attach(struct device *parent, struct device *self, void *aux) 243 { 244 struct hifn_softc *sc = (struct hifn_softc *)self; 245 struct pci_attach_args *pa = aux; 246 const struct hifn_product *hp; 247 pci_chipset_tag_t pc = pa->pa_pc; 248 pci_intr_handle_t ih; 249 const char *intrstr = NULL; 250 const char *hifncap; 251 char rbase; 252 bus_size_t iosize0, iosize1; 253 u_int32_t cmd; 254 u_int16_t ena; 255 bus_dma_segment_t seg; 256 bus_dmamap_t dmamap; 257 int rseg; 258 void *kva; 259 260 hp = hifn_lookup(pa); 261 if (hp == NULL) { 262 printf("\n"); 263 panic("hifn_attach: impossible"); 264 } 265 266 aprint_naive(": Crypto processor\n"); 267 aprint_normal(": %s, rev. %d\n", hp->hifn_name, 268 PCI_REVISION(pa->pa_class)); 269 270 sc->sc_pci_pc = pa->pa_pc; 271 sc->sc_pci_tag = pa->pa_tag; 272 273 sc->sc_flags = hp->hifn_flags; 274 275 cmd = pci_conf_read(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 276 cmd |= PCI_COMMAND_MASTER_ENABLE; 277 pci_conf_write(pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd); 278 279 if (pci_mapreg_map(pa, HIFN_BAR0, PCI_MAPREG_TYPE_MEM, 0, 280 &sc->sc_st0, &sc->sc_sh0, NULL, &iosize0)) { 281 aprint_error_dev(&sc->sc_dv, "can't map mem space %d\n", 0); 282 return; 283 } 284 285 if (pci_mapreg_map(pa, HIFN_BAR1, PCI_MAPREG_TYPE_MEM, 0, 286 &sc->sc_st1, &sc->sc_sh1, NULL, &iosize1)) { 287 aprint_error_dev(&sc->sc_dv, "can't find mem space %d\n", 1); 288 goto fail_io0; 289 } 290 291 hifn_set_retry(sc); 292 293 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 294 sc->sc_waw_lastgroup = -1; 295 sc->sc_waw_lastreg = 1; 296 } 297 298 sc->sc_dmat = pa->pa_dmat; 299 if (bus_dmamem_alloc(sc->sc_dmat, sizeof(*sc->sc_dma), PAGE_SIZE, 0, 300 &seg, 1, &rseg, BUS_DMA_NOWAIT)) { 301 aprint_error_dev(&sc->sc_dv, "can't alloc DMA buffer\n"); 302 goto fail_io1; 303 } 304 if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, sizeof(*sc->sc_dma), &kva, 305 BUS_DMA_NOWAIT)) { 306 aprint_error_dev(&sc->sc_dv, "can't map DMA buffers (%lu bytes)\n", 307 (u_long)sizeof(*sc->sc_dma)); 308 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 309 goto fail_io1; 310 } 311 if (bus_dmamap_create(sc->sc_dmat, sizeof(*sc->sc_dma), 1, 312 sizeof(*sc->sc_dma), 0, BUS_DMA_NOWAIT, &dmamap)) { 313 aprint_error_dev(&sc->sc_dv, "can't create DMA map\n"); 314 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 315 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 316 goto fail_io1; 317 } 318 if (bus_dmamap_load(sc->sc_dmat, dmamap, kva, sizeof(*sc->sc_dma), 319 NULL, BUS_DMA_NOWAIT)) { 320 aprint_error_dev(&sc->sc_dv, "can't load DMA map\n"); 321 bus_dmamap_destroy(sc->sc_dmat, dmamap); 322 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 323 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 324 goto fail_io1; 325 } 326 sc->sc_dmamap = dmamap; 327 sc->sc_dma = (struct hifn_dma *)kva; 328 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 329 330 hifn_reset_board(sc, 0); 331 332 if ((hifncap = hifn_enable_crypto(sc, pa->pa_id)) == NULL) { 333 aprint_error_dev(&sc->sc_dv, "crypto enabling failed\n"); 334 goto fail_mem; 335 } 336 hifn_reset_puc(sc); 337 338 hifn_init_dma(sc); 339 hifn_init_pci_registers(sc); 340 341 /* XXX can't dynamically determine ram type for 795x; force dram */ 342 if (sc->sc_flags & HIFN_IS_7956) 343 sc->sc_drammodel = 1; 344 else if (hifn_ramtype(sc)) 345 goto fail_mem; 346 347 if (sc->sc_drammodel == 0) 348 hifn_sramsize(sc); 349 else 350 hifn_dramsize(sc); 351 352 /* 353 * Workaround for NetSec 7751 rev A: half ram size because two 354 * of the address lines were left floating 355 */ 356 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETSEC && 357 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETSEC_7751 && 358 PCI_REVISION(pa->pa_class) == 0x61) 359 sc->sc_ramsize >>= 1; 360 361 if (pci_intr_map(pa, &ih)) { 362 aprint_error_dev(&sc->sc_dv, "couldn't map interrupt\n"); 363 goto fail_mem; 364 } 365 intrstr = pci_intr_string(pc, ih); 366 #ifdef __OpenBSD__ 367 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc, 368 self->dv_xname); 369 #else 370 sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, hifn_intr, sc); 371 #endif 372 if (sc->sc_ih == NULL) { 373 aprint_error_dev(&sc->sc_dv, "couldn't establish interrupt\n"); 374 if (intrstr != NULL) 375 aprint_normal(" at %s", intrstr); 376 aprint_normal("\n"); 377 goto fail_mem; 378 } 379 380 hifn_sessions(sc); 381 382 rseg = sc->sc_ramsize / 1024; 383 rbase = 'K'; 384 if (sc->sc_ramsize >= (1024 * 1024)) { 385 rbase = 'M'; 386 rseg /= 1024; 387 } 388 aprint_normal_dev(&sc->sc_dv, "%s, %d%cB %cram, interrupting at %s\n", 389 hifncap, rseg, rbase, 390 sc->sc_drammodel ? 'd' : 's', intrstr); 391 392 sc->sc_cid = crypto_get_driverid(0); 393 if (sc->sc_cid < 0) { 394 aprint_error_dev(&sc->sc_dv, "couldn't get crypto driver id\n"); 395 goto fail_intr; 396 } 397 398 WRITE_REG_0(sc, HIFN_0_PUCNFG, 399 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID); 400 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 401 402 switch (ena) { 403 case HIFN_PUSTAT_ENA_2: 404 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 405 hifn_newsession, hifn_freesession, hifn_process, sc); 406 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0, 407 hifn_newsession, hifn_freesession, hifn_process, sc); 408 if (sc->sc_flags & HIFN_HAS_AES) 409 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0, 410 hifn_newsession, hifn_freesession, 411 hifn_process, sc); 412 /*FALLTHROUGH*/ 413 case HIFN_PUSTAT_ENA_1: 414 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0, 415 hifn_newsession, hifn_freesession, hifn_process, sc); 416 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0, 417 hifn_newsession, hifn_freesession, hifn_process, sc); 418 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC_96, 0, 0, 419 hifn_newsession, hifn_freesession, hifn_process, sc); 420 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC_96, 0, 0, 421 hifn_newsession, hifn_freesession, hifn_process, sc); 422 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 423 hifn_newsession, hifn_freesession, hifn_process, sc); 424 break; 425 } 426 427 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 0, 428 sc->sc_dmamap->dm_mapsize, 429 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 430 431 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG)) 432 hifn_init_pubrng(sc); 433 434 #ifdef __OpenBSD__ 435 timeout_set(&sc->sc_tickto, hifn_tick, sc); 436 timeout_add(&sc->sc_tickto, hz); 437 #else 438 callout_init(&sc->sc_tickto, 0); 439 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 440 #endif 441 return; 442 443 fail_intr: 444 pci_intr_disestablish(pc, sc->sc_ih); 445 fail_mem: 446 bus_dmamap_unload(sc->sc_dmat, dmamap); 447 bus_dmamap_destroy(sc->sc_dmat, dmamap); 448 bus_dmamem_unmap(sc->sc_dmat, kva, sizeof(*sc->sc_dma)); 449 bus_dmamem_free(sc->sc_dmat, &seg, rseg); 450 451 /* Turn off DMA polling */ 452 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 453 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 454 455 fail_io1: 456 bus_space_unmap(sc->sc_st1, sc->sc_sh1, iosize1); 457 fail_io0: 458 bus_space_unmap(sc->sc_st0, sc->sc_sh0, iosize0); 459 } 460 461 static int 462 hifn_init_pubrng(struct hifn_softc *sc) 463 { 464 u_int32_t r; 465 int i; 466 467 if ((sc->sc_flags & HIFN_IS_7811) == 0) { 468 /* Reset 7951 public key/rng engine */ 469 WRITE_REG_1(sc, HIFN_1_PUB_RESET, 470 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET); 471 472 for (i = 0; i < 100; i++) { 473 DELAY(1000); 474 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) & 475 HIFN_PUBRST_RESET) == 0) 476 break; 477 } 478 479 if (i == 100) { 480 printf("%s: public key init failed\n", 481 device_xname(&sc->sc_dv)); 482 return (1); 483 } 484 } 485 486 /* Enable the rng, if available */ 487 if (sc->sc_flags & HIFN_HAS_RNG) { 488 if (sc->sc_flags & HIFN_IS_7811) { 489 r = READ_REG_1(sc, HIFN_1_7811_RNGENA); 490 if (r & HIFN_7811_RNGENA_ENA) { 491 r &= ~HIFN_7811_RNGENA_ENA; 492 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 493 } 494 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG, 495 HIFN_7811_RNGCFG_DEFL); 496 r |= HIFN_7811_RNGENA_ENA; 497 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r); 498 } else 499 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG, 500 READ_REG_1(sc, HIFN_1_RNG_CONFIG) | 501 HIFN_RNGCFG_ENA); 502 503 /* 504 * The Hifn RNG documentation states that at their 505 * recommended "conservative" RNG config values, 506 * the RNG must warm up for 0.4s before providing 507 * data that meet their worst-case estimate of 0.06 508 * bits of random data per output register bit. 509 */ 510 DELAY(4000); 511 512 #ifdef __NetBSD__ 513 /* 514 * XXX Careful! The use of RND_FLAG_NO_ESTIMATE 515 * XXX here is unobvious: we later feed raw bits 516 * XXX into the "entropy pool" with rnd_add_data, 517 * XXX explicitly supplying an entropy estimate. 518 * XXX In this context, NO_ESTIMATE serves only 519 * XXX to prevent rnd_add_data from trying to 520 * XXX use the *time at which we added the data* 521 * XXX as entropy, which is not a good idea since 522 * XXX we add data periodically from a callout. 523 */ 524 rnd_attach_source(&sc->sc_rnd_source, device_xname(&sc->sc_dv), 525 RND_TYPE_RNG, RND_FLAG_NO_ESTIMATE); 526 #endif 527 528 sc->sc_rngfirst = 1; 529 if (hz >= 100) 530 sc->sc_rnghz = hz / 100; 531 else 532 sc->sc_rnghz = 1; 533 #ifdef __OpenBSD__ 534 timeout_set(&sc->sc_rngto, hifn_rng, sc); 535 #else /* !__OpenBSD__ */ 536 callout_init(&sc->sc_rngto, 0); 537 #endif /* !__OpenBSD__ */ 538 } 539 540 /* Enable public key engine, if available */ 541 if (sc->sc_flags & HIFN_HAS_PUBLIC) { 542 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 543 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE; 544 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 545 } 546 547 /* Call directly into the RNG once to prime the pool. */ 548 hifn_rng(sc); /* Sets callout/timeout at end */ 549 550 return (0); 551 } 552 553 static void 554 hifn_rng(void *vsc) 555 { 556 struct hifn_softc *sc = vsc; 557 #ifdef __NetBSD__ 558 u_int32_t num[HIFN_RNG_BITSPER * RND_ENTROPY_THRESHOLD]; 559 #else 560 u_int32_t num[2]; 561 #endif 562 u_int32_t sts; 563 int i; 564 565 if (sc->sc_flags & HIFN_IS_7811) { 566 for (i = 0; i < 5; i++) { /* XXX why 5? */ 567 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS); 568 if (sts & HIFN_7811_RNGSTS_UFL) { 569 printf("%s: RNG underflow: disabling\n", 570 device_xname(&sc->sc_dv)); 571 return; 572 } 573 if ((sts & HIFN_7811_RNGSTS_RDY) == 0) 574 break; 575 576 /* 577 * There are at least two words in the RNG FIFO 578 * at this point. 579 */ 580 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 581 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT); 582 583 if (sc->sc_rngfirst) 584 sc->sc_rngfirst = 0; 585 #ifdef __NetBSD__ 586 rnd_add_data(&sc->sc_rnd_source, num, 587 2 * sizeof(num[0]), 588 (2 * sizeof(num[0]) * NBBY) / 589 HIFN_RNG_BITSPER); 590 #else 591 /* 592 * XXX This is a really bad idea. 593 * XXX Hifn estimate as little as 0.06 594 * XXX actual bits of entropy per output 595 * XXX register bit. How can we tell the 596 * XXX kernel RNG subsystem we're handing 597 * XXX it 64 "true" random bits, for any 598 * XXX sane value of "true"? 599 * XXX 600 * XXX The right thing to do here, if we 601 * XXX cannot supply an estimate ourselves, 602 * XXX would be to hash the bits locally. 603 */ 604 add_true_randomness(num[0]); 605 add_true_randomness(num[1]); 606 #endif 607 608 } 609 } else { 610 #ifdef __NetBSD__ 611 /* First time through, try to help fill the pool. */ 612 int nwords = sc->sc_rngfirst ? 613 sizeof(num) / sizeof(num[0]) : 4; 614 #else 615 int nwords = 2; 616 #endif 617 /* 618 * We must be *extremely* careful here. The Hifn 619 * 795x differ from the published 6500 RNG design 620 * in more ways than the obvious lack of the output 621 * FIFO and LFSR control registers. In fact, there 622 * is only one LFSR, instead of the 6500's two, and 623 * it's 32 bits, not 31. 624 * 625 * Further, a block diagram obtained from Hifn shows 626 * a very curious latching of this register: the LFSR 627 * rotates at a frequency of RNG_Clk / 8, but the 628 * RNG_Data register is latched at a frequency of 629 * RNG_Clk, which means that it is possible for 630 * consecutive reads of the RNG_Data register to read 631 * identical state from the LFSR. The simplest 632 * workaround seems to be to read eight samples from 633 * the register for each one that we use. Since each 634 * read must require at least one PCI cycle, and 635 * RNG_Clk is at least PCI_Clk, this is safe. 636 */ 637 638 639 if (sc->sc_rngfirst) { 640 sc->sc_rngfirst = 0; 641 } 642 643 644 for(i = 0 ; i < nwords * 8; i++) 645 { 646 volatile u_int32_t regtmp; 647 regtmp = READ_REG_1(sc, HIFN_1_RNG_DATA); 648 num[i / 8] = regtmp; 649 } 650 #ifdef __NetBSD__ 651 rnd_add_data(&sc->sc_rnd_source, num, 652 nwords * sizeof(num[0]), 653 (nwords * sizeof(num[0]) * NBBY) / 654 HIFN_RNG_BITSPER); 655 #else 656 /* XXX a bad idea; see 7811 block above */ 657 add_true_randomness(num[0]); 658 #endif 659 } 660 661 #ifdef __OpenBSD__ 662 timeout_add(&sc->sc_rngto, sc->sc_rnghz); 663 #else 664 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc); 665 #endif 666 } 667 668 static void 669 hifn_puc_wait(struct hifn_softc *sc) 670 { 671 int i; 672 673 for (i = 5000; i > 0; i--) { 674 DELAY(1); 675 if (!(READ_REG_0(sc, HIFN_0_PUCTRL) & HIFN_PUCTRL_RESET)) 676 break; 677 } 678 if (!i) 679 printf("%s: proc unit did not reset\n", device_xname(&sc->sc_dv)); 680 } 681 682 /* 683 * Reset the processing unit. 684 */ 685 static void 686 hifn_reset_puc(struct hifn_softc *sc) 687 { 688 /* Reset processing unit */ 689 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 690 hifn_puc_wait(sc); 691 } 692 693 static void 694 hifn_set_retry(struct hifn_softc *sc) 695 { 696 u_int32_t r; 697 698 r = pci_conf_read(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT); 699 r &= 0xffff0000; 700 pci_conf_write(sc->sc_pci_pc, sc->sc_pci_tag, HIFN_TRDY_TIMEOUT, r); 701 } 702 703 /* 704 * Resets the board. Values in the regesters are left as is 705 * from the reset (i.e. initial values are assigned elsewhere). 706 */ 707 static void 708 hifn_reset_board(struct hifn_softc *sc, int full) 709 { 710 u_int32_t reg; 711 712 /* 713 * Set polling in the DMA configuration register to zero. 0x7 avoids 714 * resetting the board and zeros out the other fields. 715 */ 716 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 717 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 718 719 /* 720 * Now that polling has been disabled, we have to wait 1 ms 721 * before resetting the board. 722 */ 723 DELAY(1000); 724 725 /* Reset the DMA unit */ 726 if (full) { 727 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 728 DELAY(1000); 729 } else { 730 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, 731 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET); 732 hifn_reset_puc(sc); 733 } 734 735 memset(sc->sc_dma, 0, sizeof(*sc->sc_dma)); 736 737 /* Bring dma unit out of reset */ 738 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 739 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 740 741 hifn_puc_wait(sc); 742 743 hifn_set_retry(sc); 744 745 if (sc->sc_flags & HIFN_IS_7811) { 746 for (reg = 0; reg < 1000; reg++) { 747 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) & 748 HIFN_MIPSRST_CRAMINIT) 749 break; 750 DELAY(1000); 751 } 752 if (reg == 1000) 753 printf(": cram init timeout\n"); 754 } 755 } 756 757 static u_int32_t 758 hifn_next_signature(u_int32_t a, u_int cnt) 759 { 760 int i; 761 u_int32_t v; 762 763 for (i = 0; i < cnt; i++) { 764 765 /* get the parity */ 766 v = a & 0x80080125; 767 v ^= v >> 16; 768 v ^= v >> 8; 769 v ^= v >> 4; 770 v ^= v >> 2; 771 v ^= v >> 1; 772 773 a = (v & 1) ^ (a << 1); 774 } 775 776 return a; 777 } 778 779 static struct pci2id { 780 u_short pci_vendor; 781 u_short pci_prod; 782 char card_id[13]; 783 } const pci2id[] = { 784 { 785 PCI_VENDOR_HIFN, 786 PCI_PRODUCT_HIFN_7951, 787 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 788 0x00, 0x00, 0x00, 0x00, 0x00 } 789 }, { 790 PCI_VENDOR_HIFN, 791 PCI_PRODUCT_HIFN_7955, 792 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 793 0x00, 0x00, 0x00, 0x00, 0x00 } 794 }, { 795 PCI_VENDOR_HIFN, 796 PCI_PRODUCT_HIFN_7956, 797 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 798 0x00, 0x00, 0x00, 0x00, 0x00 } 799 }, { 800 PCI_VENDOR_NETSEC, 801 PCI_PRODUCT_NETSEC_7751, 802 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 803 0x00, 0x00, 0x00, 0x00, 0x00 } 804 }, { 805 PCI_VENDOR_INVERTEX, 806 PCI_PRODUCT_INVERTEX_AEON, 807 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 808 0x00, 0x00, 0x00, 0x00, 0x00 } 809 }, { 810 PCI_VENDOR_HIFN, 811 PCI_PRODUCT_HIFN_7811, 812 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 813 0x00, 0x00, 0x00, 0x00, 0x00 } 814 }, { 815 /* 816 * Other vendors share this PCI ID as well, such as 817 * http://www.powercrypt.com, and obviously they also 818 * use the same key. 819 */ 820 PCI_VENDOR_HIFN, 821 PCI_PRODUCT_HIFN_7751, 822 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 823 0x00, 0x00, 0x00, 0x00, 0x00 } 824 }, 825 }; 826 827 /* 828 * Checks to see if crypto is already enabled. If crypto isn't enable, 829 * "hifn_enable_crypto" is called to enable it. The check is important, 830 * as enabling crypto twice will lock the board. 831 */ 832 static const char * 833 hifn_enable_crypto(struct hifn_softc *sc, pcireg_t pciid) 834 { 835 u_int32_t dmacfg, ramcfg, encl, addr, i; 836 const char *offtbl = NULL; 837 838 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { 839 if (pci2id[i].pci_vendor == PCI_VENDOR(pciid) && 840 pci2id[i].pci_prod == PCI_PRODUCT(pciid)) { 841 offtbl = pci2id[i].card_id; 842 break; 843 } 844 } 845 846 if (offtbl == NULL) { 847 #ifdef HIFN_DEBUG 848 aprint_debug_dev(&sc->sc_dv, "Unknown card!\n"); 849 #endif 850 return (NULL); 851 } 852 853 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG); 854 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG); 855 856 /* 857 * The RAM config register's encrypt level bit needs to be set before 858 * every read performed on the encryption level register. 859 */ 860 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 861 862 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 863 864 /* 865 * Make sure we don't re-unlock. Two unlocks kills chip until the 866 * next reboot. 867 */ 868 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) { 869 #ifdef HIFN_DEBUG 870 aprint_debug_dev(&sc->sc_dv, "Strong Crypto already enabled!\n"); 871 #endif 872 goto report; 873 } 874 875 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) { 876 #ifdef HIFN_DEBUG 877 aprint_debug_dev(&sc->sc_dv, "Unknown encryption level\n"); 878 #endif 879 return (NULL); 880 } 881 882 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK | 883 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 884 DELAY(1000); 885 addr = READ_REG_1(sc, HIFN_1_UNLOCK_SECRET1); 886 DELAY(1000); 887 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, 0); 888 DELAY(1000); 889 890 for (i = 0; i <= 12; i++) { 891 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 892 WRITE_REG_1(sc, HIFN_1_UNLOCK_SECRET2, addr); 893 894 DELAY(1000); 895 } 896 897 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID); 898 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA; 899 900 #ifdef HIFN_DEBUG 901 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2) 902 aprint_debug("Encryption engine is permanently locked until next system reset."); 903 else 904 aprint_debug("Encryption engine enabled successfully!"); 905 #endif 906 907 report: 908 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg); 909 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg); 910 911 switch (encl) { 912 case HIFN_PUSTAT_ENA_0: 913 return ("LZS-only (no encr/auth)"); 914 915 case HIFN_PUSTAT_ENA_1: 916 return ("DES"); 917 918 case HIFN_PUSTAT_ENA_2: 919 if (sc->sc_flags & HIFN_HAS_AES) 920 return ("3DES/AES"); 921 else 922 return ("3DES"); 923 924 default: 925 return ("disabled"); 926 } 927 /* NOTREACHED */ 928 } 929 930 /* 931 * Give initial values to the registers listed in the "Register Space" 932 * section of the HIFN Software Development reference manual. 933 */ 934 static void 935 hifn_init_pci_registers(struct hifn_softc *sc) 936 { 937 /* write fixed values needed by the Initialization registers */ 938 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 939 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 940 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 941 942 /* write all 4 ring address registers */ 943 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 944 offsetof(struct hifn_dma, cmdr[0])); 945 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 946 offsetof(struct hifn_dma, srcr[0])); 947 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 948 offsetof(struct hifn_dma, dstr[0])); 949 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dmamap->dm_segs[0].ds_addr + 950 offsetof(struct hifn_dma, resr[0])); 951 952 DELAY(2000); 953 954 /* write status register */ 955 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 956 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 957 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 958 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 959 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 960 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 961 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 962 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 963 HIFN_DMACSR_S_WAIT | 964 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 965 HIFN_DMACSR_C_WAIT | 966 HIFN_DMACSR_ENGINE | 967 ((sc->sc_flags & HIFN_HAS_PUBLIC) ? 968 HIFN_DMACSR_PUBDONE : 0) | 969 ((sc->sc_flags & HIFN_IS_7811) ? 970 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0)); 971 972 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0; 973 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 974 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 975 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 976 HIFN_DMAIER_ENGINE | 977 ((sc->sc_flags & HIFN_IS_7811) ? 978 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0); 979 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 980 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 981 CLR_LED(sc, HIFN_MIPSRST_LED0 | HIFN_MIPSRST_LED1 | HIFN_MIPSRST_LED2); 982 983 if (sc->sc_flags & HIFN_IS_7956) { 984 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 985 HIFN_PUCNFG_TCALLPHASES | 986 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32); 987 WRITE_REG_1(sc, HIFN_1_PLL, HIFN_PLL_7956); 988 } else { 989 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING | 990 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 991 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 992 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM)); 993 } 994 995 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 996 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 997 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 998 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 999 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1000 } 1001 1002 /* 1003 * The maximum number of sessions supported by the card 1004 * is dependent on the amount of context ram, which 1005 * encryption algorithms are enabled, and how compression 1006 * is configured. This should be configured before this 1007 * routine is called. 1008 */ 1009 static void 1010 hifn_sessions(struct hifn_softc *sc) 1011 { 1012 u_int32_t pucnfg; 1013 int ctxsize; 1014 1015 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG); 1016 1017 if (pucnfg & HIFN_PUCNFG_COMPSING) { 1018 if (pucnfg & HIFN_PUCNFG_ENCCNFG) 1019 ctxsize = 128; 1020 else 1021 ctxsize = 512; 1022 /* 1023 * 7955/7956 has internal context memory of 32K 1024 */ 1025 if (sc->sc_flags & HIFN_IS_7956) 1026 sc->sc_maxses = 32768 / ctxsize; 1027 else 1028 sc->sc_maxses = 1 + 1029 ((sc->sc_ramsize - 32768) / ctxsize); 1030 } 1031 else 1032 sc->sc_maxses = sc->sc_ramsize / 16384; 1033 1034 if (sc->sc_maxses > 2048) 1035 sc->sc_maxses = 2048; 1036 } 1037 1038 /* 1039 * Determine ram type (sram or dram). Board should be just out of a reset 1040 * state when this is called. 1041 */ 1042 static int 1043 hifn_ramtype(struct hifn_softc *sc) 1044 { 1045 u_int8_t data[8], dataexpect[8]; 1046 int i; 1047 1048 for (i = 0; i < sizeof(data); i++) 1049 data[i] = dataexpect[i] = 0x55; 1050 if (hifn_writeramaddr(sc, 0, data)) 1051 return (-1); 1052 if (hifn_readramaddr(sc, 0, data)) 1053 return (-1); 1054 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1055 sc->sc_drammodel = 1; 1056 return (0); 1057 } 1058 1059 for (i = 0; i < sizeof(data); i++) 1060 data[i] = dataexpect[i] = 0xaa; 1061 if (hifn_writeramaddr(sc, 0, data)) 1062 return (-1); 1063 if (hifn_readramaddr(sc, 0, data)) 1064 return (-1); 1065 if (memcmp(data, dataexpect, sizeof(data)) != 0) { 1066 sc->sc_drammodel = 1; 1067 return (0); 1068 } 1069 1070 return (0); 1071 } 1072 1073 #define HIFN_SRAM_MAX (32 << 20) 1074 #define HIFN_SRAM_STEP_SIZE 16384 1075 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE) 1076 1077 static int 1078 hifn_sramsize(struct hifn_softc *sc) 1079 { 1080 u_int32_t a; 1081 u_int8_t data[8]; 1082 u_int8_t dataexpect[sizeof(data)]; 1083 int32_t i; 1084 1085 for (i = 0; i < sizeof(data); i++) 1086 data[i] = dataexpect[i] = i ^ 0x5a; 1087 1088 for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) { 1089 a = i * HIFN_SRAM_STEP_SIZE; 1090 memcpy(data, &i, sizeof(i)); 1091 hifn_writeramaddr(sc, a, data); 1092 } 1093 1094 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) { 1095 a = i * HIFN_SRAM_STEP_SIZE; 1096 memcpy(dataexpect, &i, sizeof(i)); 1097 if (hifn_readramaddr(sc, a, data) < 0) 1098 return (0); 1099 if (memcmp(data, dataexpect, sizeof(data)) != 0) 1100 return (0); 1101 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE; 1102 } 1103 1104 return (0); 1105 } 1106 1107 /* 1108 * XXX For dram boards, one should really try all of the 1109 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG 1110 * is already set up correctly. 1111 */ 1112 static int 1113 hifn_dramsize(struct hifn_softc *sc) 1114 { 1115 u_int32_t cnfg; 1116 1117 if (sc->sc_flags & HIFN_IS_7956) { 1118 /* 1119 * 7955/7956 have a fixed internal ram of only 32K. 1120 */ 1121 sc->sc_ramsize = 32768; 1122 } else { 1123 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) & 1124 HIFN_PUCNFG_DRAMMASK; 1125 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18); 1126 } 1127 return (0); 1128 } 1129 1130 static void 1131 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, 1132 int *resp) 1133 { 1134 struct hifn_dma *dma = sc->sc_dma; 1135 1136 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1137 dma->cmdi = 0; 1138 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1139 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1140 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1141 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1142 } 1143 *cmdp = dma->cmdi++; 1144 dma->cmdk = dma->cmdi; 1145 1146 if (dma->srci == HIFN_D_SRC_RSIZE) { 1147 dma->srci = 0; 1148 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID | 1149 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1150 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1151 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1152 } 1153 *srcp = dma->srci++; 1154 dma->srck = dma->srci; 1155 1156 if (dma->dsti == HIFN_D_DST_RSIZE) { 1157 dma->dsti = 0; 1158 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID | 1159 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1160 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE, 1161 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1162 } 1163 *dstp = dma->dsti++; 1164 dma->dstk = dma->dsti; 1165 1166 if (dma->resi == HIFN_D_RES_RSIZE) { 1167 dma->resi = 0; 1168 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1169 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1170 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1171 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1172 } 1173 *resp = dma->resi++; 1174 dma->resk = dma->resi; 1175 } 1176 1177 static int 1178 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1179 { 1180 struct hifn_dma *dma = sc->sc_dma; 1181 struct hifn_base_command wc; 1182 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1183 int r, cmdi, resi, srci, dsti; 1184 1185 wc.masks = htole16(3 << 13); 1186 wc.session_num = htole16(addr >> 14); 1187 wc.total_source_count = htole16(8); 1188 wc.total_dest_count = htole16(addr & 0x3fff); 1189 1190 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1191 1192 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1193 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1194 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1195 1196 /* build write command */ 1197 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1198 *(struct hifn_base_command *)dma->command_bufs[cmdi] = wc; 1199 memcpy(&dma->test_src, data, sizeof(dma->test_src)); 1200 1201 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1202 + offsetof(struct hifn_dma, test_src)); 1203 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr 1204 + offsetof(struct hifn_dma, test_dst)); 1205 1206 dma->cmdr[cmdi].l = htole32(16 | masks); 1207 dma->srcr[srci].l = htole32(8 | masks); 1208 dma->dstr[dsti].l = htole32(4 | masks); 1209 dma->resr[resi].l = htole32(4 | masks); 1210 1211 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1212 0, sc->sc_dmamap->dm_mapsize, 1213 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1214 1215 for (r = 10000; r >= 0; r--) { 1216 DELAY(10); 1217 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1218 0, sc->sc_dmamap->dm_mapsize, 1219 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1220 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1221 break; 1222 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1223 0, sc->sc_dmamap->dm_mapsize, 1224 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1225 } 1226 if (r == 0) { 1227 printf("%s: writeramaddr -- " 1228 "result[%d](addr %d) still valid\n", 1229 device_xname(&sc->sc_dv), resi, addr); 1230 r = -1; 1231 return (-1); 1232 } else 1233 r = 0; 1234 1235 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1236 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1237 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1238 1239 return (r); 1240 } 1241 1242 static int 1243 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data) 1244 { 1245 struct hifn_dma *dma = sc->sc_dma; 1246 struct hifn_base_command rc; 1247 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ; 1248 int r, cmdi, srci, dsti, resi; 1249 1250 rc.masks = htole16(2 << 13); 1251 rc.session_num = htole16(addr >> 14); 1252 rc.total_source_count = htole16(addr & 0x3fff); 1253 rc.total_dest_count = htole16(8); 1254 1255 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi); 1256 1257 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1258 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1259 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA); 1260 1261 memset(dma->command_bufs[cmdi], 0, HIFN_MAX_COMMAND); 1262 *(struct hifn_base_command *)dma->command_bufs[cmdi] = rc; 1263 1264 dma->srcr[srci].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1265 offsetof(struct hifn_dma, test_src)); 1266 dma->test_src = 0; 1267 dma->dstr[dsti].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1268 offsetof(struct hifn_dma, test_dst)); 1269 dma->test_dst = 0; 1270 dma->cmdr[cmdi].l = htole32(8 | masks); 1271 dma->srcr[srci].l = htole32(8 | masks); 1272 dma->dstr[dsti].l = htole32(8 | masks); 1273 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks); 1274 1275 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1276 0, sc->sc_dmamap->dm_mapsize, 1277 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1278 1279 for (r = 10000; r >= 0; r--) { 1280 DELAY(10); 1281 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1282 0, sc->sc_dmamap->dm_mapsize, 1283 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1284 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0) 1285 break; 1286 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 1287 0, sc->sc_dmamap->dm_mapsize, 1288 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1289 } 1290 if (r == 0) { 1291 printf("%s: readramaddr -- " 1292 "result[%d](addr %d) still valid\n", 1293 device_xname(&sc->sc_dv), resi, addr); 1294 r = -1; 1295 } else { 1296 r = 0; 1297 memcpy(data, &dma->test_dst, sizeof(dma->test_dst)); 1298 } 1299 1300 WRITE_REG_1(sc, HIFN_1_DMA_CSR, 1301 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS | 1302 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS); 1303 1304 return (r); 1305 } 1306 1307 /* 1308 * Initialize the descriptor rings. 1309 */ 1310 static void 1311 hifn_init_dma(struct hifn_softc *sc) 1312 { 1313 struct hifn_dma *dma = sc->sc_dma; 1314 int i; 1315 1316 hifn_set_retry(sc); 1317 1318 /* initialize static pointer values */ 1319 for (i = 0; i < HIFN_D_CMD_RSIZE; i++) 1320 dma->cmdr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1321 offsetof(struct hifn_dma, command_bufs[i][0])); 1322 for (i = 0; i < HIFN_D_RES_RSIZE; i++) 1323 dma->resr[i].p = htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1324 offsetof(struct hifn_dma, result_bufs[i][0])); 1325 1326 dma->cmdr[HIFN_D_CMD_RSIZE].p = 1327 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1328 offsetof(struct hifn_dma, cmdr[0])); 1329 dma->srcr[HIFN_D_SRC_RSIZE].p = 1330 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1331 offsetof(struct hifn_dma, srcr[0])); 1332 dma->dstr[HIFN_D_DST_RSIZE].p = 1333 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1334 offsetof(struct hifn_dma, dstr[0])); 1335 dma->resr[HIFN_D_RES_RSIZE].p = 1336 htole32(sc->sc_dmamap->dm_segs[0].ds_addr + 1337 offsetof(struct hifn_dma, resr[0])); 1338 1339 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 1340 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 1341 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 1342 } 1343 1344 /* 1345 * Writes out the raw command buffer space. Returns the 1346 * command buffer size. 1347 */ 1348 static u_int 1349 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf) 1350 { 1351 u_int8_t *buf_pos; 1352 struct hifn_base_command *base_cmd; 1353 struct hifn_mac_command *mac_cmd; 1354 struct hifn_crypt_command *cry_cmd; 1355 struct hifn_comp_command *comp_cmd; 1356 int using_mac, using_crypt, using_comp, len, ivlen; 1357 u_int32_t dlen, slen; 1358 1359 buf_pos = buf; 1360 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC; 1361 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT; 1362 using_comp = cmd->base_masks & HIFN_BASE_CMD_COMP; 1363 1364 base_cmd = (struct hifn_base_command *)buf_pos; 1365 base_cmd->masks = htole16(cmd->base_masks); 1366 slen = cmd->src_map->dm_mapsize; 1367 if (cmd->sloplen) 1368 dlen = cmd->dst_map->dm_mapsize - cmd->sloplen + 1369 sizeof(u_int32_t); 1370 else 1371 dlen = cmd->dst_map->dm_mapsize; 1372 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO); 1373 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1374 dlen >>= 16; 1375 slen >>= 16; 1376 base_cmd->session_num = htole16(cmd->session_num | 1377 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1378 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1379 buf_pos += sizeof(struct hifn_base_command); 1380 1381 if (using_comp) { 1382 comp_cmd = (struct hifn_comp_command *)buf_pos; 1383 dlen = cmd->compcrd->crd_len; 1384 comp_cmd->source_count = htole16(dlen & 0xffff); 1385 dlen >>= 16; 1386 comp_cmd->masks = htole16(cmd->comp_masks | 1387 ((dlen << HIFN_COMP_CMD_SRCLEN_S) & HIFN_COMP_CMD_SRCLEN_M)); 1388 comp_cmd->header_skip = htole16(cmd->compcrd->crd_skip); 1389 comp_cmd->reserved = 0; 1390 buf_pos += sizeof(struct hifn_comp_command); 1391 } 1392 1393 if (using_mac) { 1394 mac_cmd = (struct hifn_mac_command *)buf_pos; 1395 dlen = cmd->maccrd->crd_len; 1396 mac_cmd->source_count = htole16(dlen & 0xffff); 1397 dlen >>= 16; 1398 mac_cmd->masks = htole16(cmd->mac_masks | 1399 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M)); 1400 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip); 1401 mac_cmd->reserved = 0; 1402 buf_pos += sizeof(struct hifn_mac_command); 1403 } 1404 1405 if (using_crypt) { 1406 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1407 dlen = cmd->enccrd->crd_len; 1408 cry_cmd->source_count = htole16(dlen & 0xffff); 1409 dlen >>= 16; 1410 cry_cmd->masks = htole16(cmd->cry_masks | 1411 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M)); 1412 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip); 1413 cry_cmd->reserved = 0; 1414 buf_pos += sizeof(struct hifn_crypt_command); 1415 } 1416 1417 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) { 1418 memcpy(buf_pos, cmd->mac, HIFN_MAC_KEY_LENGTH); 1419 buf_pos += HIFN_MAC_KEY_LENGTH; 1420 } 1421 1422 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) { 1423 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1424 case HIFN_CRYPT_CMD_ALG_3DES: 1425 memcpy(buf_pos, cmd->ck, HIFN_3DES_KEY_LENGTH); 1426 buf_pos += HIFN_3DES_KEY_LENGTH; 1427 break; 1428 case HIFN_CRYPT_CMD_ALG_DES: 1429 memcpy(buf_pos, cmd->ck, HIFN_DES_KEY_LENGTH); 1430 buf_pos += HIFN_DES_KEY_LENGTH; 1431 break; 1432 case HIFN_CRYPT_CMD_ALG_RC4: 1433 len = 256; 1434 do { 1435 int clen; 1436 1437 clen = MIN(cmd->cklen, len); 1438 memcpy(buf_pos, cmd->ck, clen); 1439 len -= clen; 1440 buf_pos += clen; 1441 } while (len > 0); 1442 memset(buf_pos, 0, 4); 1443 buf_pos += 4; 1444 break; 1445 case HIFN_CRYPT_CMD_ALG_AES: 1446 /* 1447 * AES keys are variable 128, 192 and 1448 * 256 bits (16, 24 and 32 bytes). 1449 */ 1450 memcpy(buf_pos, cmd->ck, cmd->cklen); 1451 buf_pos += cmd->cklen; 1452 break; 1453 } 1454 } 1455 1456 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) { 1457 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) { 1458 case HIFN_CRYPT_CMD_ALG_AES: 1459 ivlen = HIFN_AES_IV_LENGTH; 1460 break; 1461 default: 1462 ivlen = HIFN_IV_LENGTH; 1463 break; 1464 } 1465 memcpy(buf_pos, cmd->iv, ivlen); 1466 buf_pos += ivlen; 1467 } 1468 1469 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC | HIFN_BASE_CMD_CRYPT | 1470 HIFN_BASE_CMD_COMP)) == 0) { 1471 memset(buf_pos, 0, 8); 1472 buf_pos += 8; 1473 } 1474 1475 return (buf_pos - buf); 1476 } 1477 1478 static int 1479 hifn_dmamap_aligned(bus_dmamap_t map) 1480 { 1481 int i; 1482 1483 for (i = 0; i < map->dm_nsegs; i++) { 1484 if (map->dm_segs[i].ds_addr & 3) 1485 return (0); 1486 if ((i != (map->dm_nsegs - 1)) && 1487 (map->dm_segs[i].ds_len & 3)) 1488 return (0); 1489 } 1490 return (1); 1491 } 1492 1493 static int 1494 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd) 1495 { 1496 struct hifn_dma *dma = sc->sc_dma; 1497 bus_dmamap_t map = cmd->dst_map; 1498 u_int32_t p, l; 1499 int idx, used = 0, i; 1500 1501 idx = dma->dsti; 1502 for (i = 0; i < map->dm_nsegs - 1; i++) { 1503 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1504 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1505 HIFN_D_MASKDONEIRQ | map->dm_segs[i].ds_len); 1506 HIFN_DSTR_SYNC(sc, idx, 1507 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1508 used++; 1509 1510 if (++idx == HIFN_D_DST_RSIZE) { 1511 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1512 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1513 HIFN_DSTR_SYNC(sc, idx, 1514 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1515 idx = 0; 1516 } 1517 } 1518 1519 if (cmd->sloplen == 0) { 1520 p = map->dm_segs[i].ds_addr; 1521 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1522 map->dm_segs[i].ds_len; 1523 } else { 1524 p = sc->sc_dmamap->dm_segs[0].ds_addr + 1525 offsetof(struct hifn_dma, slop[cmd->slopidx]); 1526 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST | 1527 sizeof(u_int32_t); 1528 1529 if ((map->dm_segs[i].ds_len - cmd->sloplen) != 0) { 1530 dma->dstr[idx].p = htole32(map->dm_segs[i].ds_addr); 1531 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1532 HIFN_D_MASKDONEIRQ | 1533 (map->dm_segs[i].ds_len - cmd->sloplen)); 1534 HIFN_DSTR_SYNC(sc, idx, 1535 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1536 used++; 1537 1538 if (++idx == HIFN_D_DST_RSIZE) { 1539 dma->dstr[idx].l = htole32(HIFN_D_VALID | 1540 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1541 HIFN_DSTR_SYNC(sc, idx, 1542 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1543 idx = 0; 1544 } 1545 } 1546 } 1547 dma->dstr[idx].p = htole32(p); 1548 dma->dstr[idx].l = htole32(l); 1549 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1550 used++; 1551 1552 if (++idx == HIFN_D_DST_RSIZE) { 1553 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP | 1554 HIFN_D_MASKDONEIRQ); 1555 HIFN_DSTR_SYNC(sc, idx, 1556 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1557 idx = 0; 1558 } 1559 1560 dma->dsti = idx; 1561 dma->dstu += used; 1562 return (idx); 1563 } 1564 1565 static int 1566 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd) 1567 { 1568 struct hifn_dma *dma = sc->sc_dma; 1569 bus_dmamap_t map = cmd->src_map; 1570 int idx, i; 1571 u_int32_t last = 0; 1572 1573 idx = dma->srci; 1574 for (i = 0; i < map->dm_nsegs; i++) { 1575 if (i == map->dm_nsegs - 1) 1576 last = HIFN_D_LAST; 1577 1578 dma->srcr[idx].p = htole32(map->dm_segs[i].ds_addr); 1579 dma->srcr[idx].l = htole32(map->dm_segs[i].ds_len | 1580 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last); 1581 HIFN_SRCR_SYNC(sc, idx, 1582 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1583 1584 if (++idx == HIFN_D_SRC_RSIZE) { 1585 dma->srcr[idx].l = htole32(HIFN_D_VALID | 1586 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1587 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE, 1588 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1589 idx = 0; 1590 } 1591 } 1592 dma->srci = idx; 1593 dma->srcu += map->dm_nsegs; 1594 return (idx); 1595 } 1596 1597 static int 1598 hifn_crypto(struct hifn_softc *sc, struct hifn_command *cmd, 1599 struct cryptop *crp, int hint) 1600 { 1601 struct hifn_dma *dma = sc->sc_dma; 1602 u_int32_t cmdlen; 1603 int cmdi, resi, s, err = 0; 1604 1605 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 1606 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) 1607 return (ENOMEM); 1608 1609 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1610 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 1611 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 1612 err = ENOMEM; 1613 goto err_srcmap1; 1614 } 1615 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1616 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 1617 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 1618 err = ENOMEM; 1619 goto err_srcmap1; 1620 } 1621 } else { 1622 err = EINVAL; 1623 goto err_srcmap1; 1624 } 1625 1626 if (hifn_dmamap_aligned(cmd->src_map)) { 1627 cmd->sloplen = cmd->src_map->dm_mapsize & 3; 1628 if (crp->crp_flags & CRYPTO_F_IOV) 1629 cmd->dstu.dst_io = cmd->srcu.src_io; 1630 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1631 cmd->dstu.dst_m = cmd->srcu.src_m; 1632 cmd->dst_map = cmd->src_map; 1633 } else { 1634 if (crp->crp_flags & CRYPTO_F_IOV) { 1635 err = EINVAL; 1636 goto err_srcmap; 1637 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1638 int totlen, len; 1639 struct mbuf *m, *m0, *mlast; 1640 1641 totlen = cmd->src_map->dm_mapsize; 1642 if (cmd->srcu.src_m->m_flags & M_PKTHDR) { 1643 len = MHLEN; 1644 MGETHDR(m0, M_DONTWAIT, MT_DATA); 1645 } else { 1646 len = MLEN; 1647 MGET(m0, M_DONTWAIT, MT_DATA); 1648 } 1649 if (m0 == NULL) { 1650 err = ENOMEM; 1651 goto err_srcmap; 1652 } 1653 if (len == MHLEN) 1654 M_DUP_PKTHDR(m0, cmd->srcu.src_m); 1655 if (totlen >= MINCLSIZE) { 1656 MCLGET(m0, M_DONTWAIT); 1657 if (m0->m_flags & M_EXT) 1658 len = MCLBYTES; 1659 } 1660 totlen -= len; 1661 m0->m_pkthdr.len = m0->m_len = len; 1662 mlast = m0; 1663 1664 while (totlen > 0) { 1665 MGET(m, M_DONTWAIT, MT_DATA); 1666 if (m == NULL) { 1667 err = ENOMEM; 1668 m_freem(m0); 1669 goto err_srcmap; 1670 } 1671 len = MLEN; 1672 if (totlen >= MINCLSIZE) { 1673 MCLGET(m, M_DONTWAIT); 1674 if (m->m_flags & M_EXT) 1675 len = MCLBYTES; 1676 } 1677 1678 m->m_len = len; 1679 if (m0->m_flags & M_PKTHDR) 1680 m0->m_pkthdr.len += len; 1681 totlen -= len; 1682 1683 mlast->m_next = m; 1684 mlast = m; 1685 } 1686 cmd->dstu.dst_m = m0; 1687 } 1688 } 1689 1690 if (cmd->dst_map == NULL) { 1691 if (bus_dmamap_create(sc->sc_dmat, 1692 HIFN_MAX_SEGLEN * MAX_SCATTER, MAX_SCATTER, 1693 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 1694 err = ENOMEM; 1695 goto err_srcmap; 1696 } 1697 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1698 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 1699 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 1700 err = ENOMEM; 1701 goto err_dstmap1; 1702 } 1703 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1704 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 1705 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 1706 err = ENOMEM; 1707 goto err_dstmap1; 1708 } 1709 } 1710 } 1711 1712 #ifdef HIFN_DEBUG 1713 if (hifn_debug) 1714 printf("%s: Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n", 1715 device_xname(&sc->sc_dv), 1716 READ_REG_1(sc, HIFN_1_DMA_CSR), 1717 READ_REG_1(sc, HIFN_1_DMA_IER), 1718 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1719 cmd->src_map->dm_nsegs, cmd->dst_map->dm_nsegs); 1720 #endif 1721 1722 if (cmd->src_map == cmd->dst_map) 1723 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1724 0, cmd->src_map->dm_mapsize, 1725 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 1726 else { 1727 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 1728 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1729 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 1730 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 1731 } 1732 1733 s = splnet(); 1734 1735 /* 1736 * need 1 cmd, and 1 res 1737 * need N src, and N dst 1738 */ 1739 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 1740 (dma->resu + 1) > HIFN_D_RES_RSIZE) { 1741 splx(s); 1742 err = ENOMEM; 1743 goto err_dstmap; 1744 } 1745 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 1746 (dma->dstu + cmd->dst_map->dm_nsegs + 1) > HIFN_D_DST_RSIZE) { 1747 splx(s); 1748 err = ENOMEM; 1749 goto err_dstmap; 1750 } 1751 1752 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 1753 dma->cmdi = 0; 1754 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 1755 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1756 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 1757 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1758 } 1759 cmdi = dma->cmdi++; 1760 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 1761 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 1762 1763 /* .p for command/result already set */ 1764 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 1765 HIFN_D_MASKDONEIRQ); 1766 HIFN_CMDR_SYNC(sc, cmdi, 1767 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1768 dma->cmdu++; 1769 if (sc->sc_c_busy == 0) { 1770 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1771 sc->sc_c_busy = 1; 1772 SET_LED(sc, HIFN_MIPSRST_LED0); 1773 } 1774 1775 /* 1776 * We don't worry about missing an interrupt (which a "command wait" 1777 * interrupt salvages us from), unless there is more than one command 1778 * in the queue. 1779 * 1780 * XXX We do seem to miss some interrupts. So we always enable 1781 * XXX command wait. From OpenBSD revision 1.149. 1782 * 1783 */ 1784 #if 0 1785 if (dma->cmdu > 1) { 1786 #endif 1787 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 1788 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1789 #if 0 1790 } 1791 #endif 1792 1793 hifnstats.hst_ipackets++; 1794 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 1795 1796 hifn_dmamap_load_src(sc, cmd); 1797 if (sc->sc_s_busy == 0) { 1798 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1799 sc->sc_s_busy = 1; 1800 SET_LED(sc, HIFN_MIPSRST_LED1); 1801 } 1802 1803 /* 1804 * Unlike other descriptors, we don't mask done interrupt from 1805 * result descriptor. 1806 */ 1807 #ifdef HIFN_DEBUG 1808 if (hifn_debug) 1809 printf("load res\n"); 1810 #endif 1811 if (dma->resi == HIFN_D_RES_RSIZE) { 1812 dma->resi = 0; 1813 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 1814 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 1815 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 1816 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1817 } 1818 resi = dma->resi++; 1819 dma->hifn_commands[resi] = cmd; 1820 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 1821 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 1822 HIFN_D_VALID | HIFN_D_LAST); 1823 HIFN_RESR_SYNC(sc, resi, 1824 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1825 dma->resu++; 1826 if (sc->sc_r_busy == 0) { 1827 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1828 sc->sc_r_busy = 1; 1829 SET_LED(sc, HIFN_MIPSRST_LED2); 1830 } 1831 1832 if (cmd->sloplen) 1833 cmd->slopidx = resi; 1834 1835 hifn_dmamap_load_dst(sc, cmd); 1836 1837 if (sc->sc_d_busy == 0) { 1838 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1839 sc->sc_d_busy = 1; 1840 } 1841 1842 #ifdef HIFN_DEBUG 1843 if (hifn_debug) 1844 printf("%s: command: stat %8x ier %8x\n", 1845 device_xname(&sc->sc_dv), 1846 READ_REG_1(sc, HIFN_1_DMA_CSR), READ_REG_1(sc, HIFN_1_DMA_IER)); 1847 #endif 1848 1849 sc->sc_active = 5; 1850 splx(s); 1851 return (err); /* success */ 1852 1853 err_dstmap: 1854 if (cmd->src_map != cmd->dst_map) 1855 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 1856 err_dstmap1: 1857 if (cmd->src_map != cmd->dst_map) 1858 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 1859 err_srcmap: 1860 if (crp->crp_flags & CRYPTO_F_IMBUF && 1861 cmd->srcu.src_m != cmd->dstu.dst_m) 1862 m_freem(cmd->dstu.dst_m); 1863 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 1864 err_srcmap1: 1865 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 1866 return (err); 1867 } 1868 1869 static void 1870 hifn_tick(void *vsc) 1871 { 1872 struct hifn_softc *sc = vsc; 1873 int s; 1874 1875 s = splnet(); 1876 if (sc->sc_active == 0) { 1877 struct hifn_dma *dma = sc->sc_dma; 1878 u_int32_t r = 0; 1879 1880 if (dma->cmdu == 0 && sc->sc_c_busy) { 1881 sc->sc_c_busy = 0; 1882 r |= HIFN_DMACSR_C_CTRL_DIS; 1883 CLR_LED(sc, HIFN_MIPSRST_LED0); 1884 } 1885 if (dma->srcu == 0 && sc->sc_s_busy) { 1886 sc->sc_s_busy = 0; 1887 r |= HIFN_DMACSR_S_CTRL_DIS; 1888 CLR_LED(sc, HIFN_MIPSRST_LED1); 1889 } 1890 if (dma->dstu == 0 && sc->sc_d_busy) { 1891 sc->sc_d_busy = 0; 1892 r |= HIFN_DMACSR_D_CTRL_DIS; 1893 } 1894 if (dma->resu == 0 && sc->sc_r_busy) { 1895 sc->sc_r_busy = 0; 1896 r |= HIFN_DMACSR_R_CTRL_DIS; 1897 CLR_LED(sc, HIFN_MIPSRST_LED2); 1898 } 1899 if (r) 1900 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r); 1901 } 1902 else 1903 sc->sc_active--; 1904 splx(s); 1905 #ifdef __OpenBSD__ 1906 timeout_add(&sc->sc_tickto, hz); 1907 #else 1908 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc); 1909 #endif 1910 } 1911 1912 static int 1913 hifn_intr(void *arg) 1914 { 1915 struct hifn_softc *sc = arg; 1916 struct hifn_dma *dma = sc->sc_dma; 1917 u_int32_t dmacsr, restart; 1918 int i, u; 1919 1920 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR); 1921 1922 #ifdef HIFN_DEBUG 1923 if (hifn_debug) 1924 printf("%s: irq: stat %08x ien %08x u %d/%d/%d/%d\n", 1925 device_xname(&sc->sc_dv), 1926 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), 1927 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1928 #endif 1929 1930 /* Nothing in the DMA unit interrupted */ 1931 if ((dmacsr & sc->sc_dmaier) == 0) 1932 return (0); 1933 1934 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier); 1935 1936 if (dmacsr & HIFN_DMACSR_ENGINE) 1937 WRITE_REG_0(sc, HIFN_0_PUISR, READ_REG_0(sc, HIFN_0_PUISR)); 1938 1939 if ((sc->sc_flags & HIFN_HAS_PUBLIC) && 1940 (dmacsr & HIFN_DMACSR_PUBDONE)) 1941 WRITE_REG_1(sc, HIFN_1_PUB_STATUS, 1942 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1943 1944 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1945 if (restart) 1946 printf("%s: overrun %x\n", device_xname(&sc->sc_dv), dmacsr); 1947 1948 if (sc->sc_flags & HIFN_IS_7811) { 1949 if (dmacsr & HIFN_DMACSR_ILLR) 1950 printf("%s: illegal read\n", device_xname(&sc->sc_dv)); 1951 if (dmacsr & HIFN_DMACSR_ILLW) 1952 printf("%s: illegal write\n", device_xname(&sc->sc_dv)); 1953 } 1954 1955 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1956 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1957 if (restart) { 1958 printf("%s: abort, resetting.\n", device_xname(&sc->sc_dv)); 1959 hifnstats.hst_abort++; 1960 hifn_abort(sc); 1961 return (1); 1962 } 1963 1964 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->resu == 0)) { 1965 /* 1966 * If no slots to process and we receive a "waiting on 1967 * command" interrupt, we disable the "waiting on command" 1968 * (by clearing it). 1969 */ 1970 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT; 1971 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 1972 } 1973 1974 /* clear the rings */ 1975 i = dma->resk; 1976 while (dma->resu != 0) { 1977 HIFN_RESR_SYNC(sc, i, 1978 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1979 if (dma->resr[i].l & htole32(HIFN_D_VALID)) { 1980 HIFN_RESR_SYNC(sc, i, 1981 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1982 break; 1983 } 1984 1985 if (i != HIFN_D_RES_RSIZE) { 1986 struct hifn_command *cmd; 1987 1988 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD); 1989 cmd = dma->hifn_commands[i]; 1990 KASSERT(cmd != NULL 1991 /*("hifn_intr: null command slot %u", i)*/); 1992 dma->hifn_commands[i] = NULL; 1993 1994 hifn_callback(sc, cmd, dma->result_bufs[i]); 1995 hifnstats.hst_opackets++; 1996 } 1997 1998 if (++i == (HIFN_D_RES_RSIZE + 1)) 1999 i = 0; 2000 else 2001 dma->resu--; 2002 } 2003 dma->resk = i; 2004 2005 i = dma->srck; u = dma->srcu; 2006 while (u != 0) { 2007 HIFN_SRCR_SYNC(sc, i, 2008 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2009 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) { 2010 HIFN_SRCR_SYNC(sc, i, 2011 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2012 break; 2013 } 2014 if (++i == (HIFN_D_SRC_RSIZE + 1)) 2015 i = 0; 2016 else 2017 u--; 2018 } 2019 dma->srck = i; dma->srcu = u; 2020 2021 i = dma->cmdk; u = dma->cmdu; 2022 while (u != 0) { 2023 HIFN_CMDR_SYNC(sc, i, 2024 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2025 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) { 2026 HIFN_CMDR_SYNC(sc, i, 2027 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2028 break; 2029 } 2030 if (i != HIFN_D_CMD_RSIZE) { 2031 u--; 2032 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE); 2033 } 2034 if (++i == (HIFN_D_CMD_RSIZE + 1)) 2035 i = 0; 2036 } 2037 dma->cmdk = i; dma->cmdu = u; 2038 2039 return (1); 2040 } 2041 2042 /* 2043 * Allocate a new 'session' and return an encoded session id. 'sidp' 2044 * contains our registration id, and should contain an encoded session 2045 * id on successful allocation. 2046 */ 2047 static int 2048 hifn_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 2049 { 2050 struct cryptoini *c; 2051 struct hifn_softc *sc = arg; 2052 int i, mac = 0, cry = 0, comp = 0; 2053 2054 KASSERT(sc != NULL /*, ("hifn_newsession: null softc")*/); 2055 if (sidp == NULL || cri == NULL || sc == NULL) 2056 return (EINVAL); 2057 2058 for (i = 0; i < sc->sc_maxses; i++) 2059 if (sc->sc_sessions[i].hs_state == HS_STATE_FREE) 2060 break; 2061 if (i == sc->sc_maxses) 2062 return (ENOMEM); 2063 2064 for (c = cri; c != NULL; c = c->cri_next) { 2065 switch (c->cri_alg) { 2066 case CRYPTO_MD5: 2067 case CRYPTO_SHA1: 2068 case CRYPTO_MD5_HMAC_96: 2069 case CRYPTO_SHA1_HMAC_96: 2070 if (mac) 2071 return (EINVAL); 2072 mac = 1; 2073 break; 2074 case CRYPTO_DES_CBC: 2075 case CRYPTO_3DES_CBC: 2076 case CRYPTO_AES_CBC: 2077 /* Note that this is an initialization 2078 vector, not a cipher key; any function 2079 giving sufficient Hamming distance 2080 between outputs is fine. Use of RC4 2081 to generate IVs has been FIPS140-2 2082 certified by several labs. */ 2083 #ifdef __NetBSD__ 2084 arc4randbytes(sc->sc_sessions[i].hs_iv, 2085 c->cri_alg == CRYPTO_AES_CBC ? 2086 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2087 #else /* FreeBSD and OpenBSD have get_random_bytes */ 2088 /* XXX this may read fewer, does it matter? */ 2089 get_random_bytes(sc->sc_sessions[i].hs_iv, 2090 c->cri_alg == CRYPTO_AES_CBC ? 2091 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2092 #endif 2093 /*FALLTHROUGH*/ 2094 case CRYPTO_ARC4: 2095 if (cry) 2096 return (EINVAL); 2097 cry = 1; 2098 break; 2099 #ifdef HAVE_CRYPTO_LZS 2100 case CRYPTO_LZS_COMP: 2101 if (comp) 2102 return (EINVAL); 2103 comp = 1; 2104 break; 2105 #endif 2106 default: 2107 return (EINVAL); 2108 } 2109 } 2110 if (mac == 0 && cry == 0 && comp == 0) 2111 return (EINVAL); 2112 2113 /* 2114 * XXX only want to support compression without chaining to 2115 * MAC/crypt engine right now 2116 */ 2117 if ((comp && mac) || (comp && cry)) 2118 return (EINVAL); 2119 2120 *sidp = HIFN_SID(device_unit(&sc->sc_dv), i); 2121 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2122 2123 return (0); 2124 } 2125 2126 /* 2127 * Deallocate a session. 2128 * XXX this routine should run a zero'd mac/encrypt key into context ram. 2129 * XXX to blow away any keys already stored there. 2130 */ 2131 static int 2132 hifn_freesession(void *arg, u_int64_t tid) 2133 { 2134 struct hifn_softc *sc = arg; 2135 int session; 2136 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 2137 2138 KASSERT(sc != NULL /*, ("hifn_freesession: null softc")*/); 2139 if (sc == NULL) 2140 return (EINVAL); 2141 2142 session = HIFN_SESSION(sid); 2143 if (session >= sc->sc_maxses) 2144 return (EINVAL); 2145 2146 memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session])); 2147 return (0); 2148 } 2149 2150 static int 2151 hifn_process(void *arg, struct cryptop *crp, int hint) 2152 { 2153 struct hifn_softc *sc = arg; 2154 struct hifn_command *cmd = NULL; 2155 int session, err, ivlen; 2156 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 2157 2158 if (crp == NULL || crp->crp_callback == NULL) { 2159 hifnstats.hst_invalid++; 2160 return (EINVAL); 2161 } 2162 session = HIFN_SESSION(crp->crp_sid); 2163 2164 if (sc == NULL || session >= sc->sc_maxses) { 2165 err = EINVAL; 2166 goto errout; 2167 } 2168 2169 cmd = (struct hifn_command *)malloc(sizeof(struct hifn_command), 2170 M_DEVBUF, M_NOWAIT|M_ZERO); 2171 if (cmd == NULL) { 2172 hifnstats.hst_nomem++; 2173 err = ENOMEM; 2174 goto errout; 2175 } 2176 2177 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2178 cmd->srcu.src_m = (struct mbuf *)crp->crp_buf; 2179 cmd->dstu.dst_m = (struct mbuf *)crp->crp_buf; 2180 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2181 cmd->srcu.src_io = (struct uio *)crp->crp_buf; 2182 cmd->dstu.dst_io = (struct uio *)crp->crp_buf; 2183 } else { 2184 err = EINVAL; 2185 goto errout; /* XXX we don't handle contiguous buffers! */ 2186 } 2187 2188 crd1 = crp->crp_desc; 2189 if (crd1 == NULL) { 2190 err = EINVAL; 2191 goto errout; 2192 } 2193 crd2 = crd1->crd_next; 2194 2195 if (crd2 == NULL) { 2196 if (crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2197 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2198 crd1->crd_alg == CRYPTO_SHA1 || 2199 crd1->crd_alg == CRYPTO_MD5) { 2200 maccrd = crd1; 2201 enccrd = NULL; 2202 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 2203 crd1->crd_alg == CRYPTO_3DES_CBC || 2204 crd1->crd_alg == CRYPTO_AES_CBC || 2205 crd1->crd_alg == CRYPTO_ARC4) { 2206 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0) 2207 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2208 maccrd = NULL; 2209 enccrd = crd1; 2210 #ifdef HAVE_CRYPTO_LZS 2211 } else if (crd1->crd_alg == CRYPTO_LZS_COMP) { 2212 return (hifn_compression(sc, crp, cmd)); 2213 #endif 2214 } else { 2215 err = EINVAL; 2216 goto errout; 2217 } 2218 } else { 2219 if ((crd1->crd_alg == CRYPTO_MD5_HMAC_96 || 2220 crd1->crd_alg == CRYPTO_SHA1_HMAC_96 || 2221 crd1->crd_alg == CRYPTO_MD5 || 2222 crd1->crd_alg == CRYPTO_SHA1) && 2223 (crd2->crd_alg == CRYPTO_DES_CBC || 2224 crd2->crd_alg == CRYPTO_3DES_CBC || 2225 crd2->crd_alg == CRYPTO_AES_CBC || 2226 crd2->crd_alg == CRYPTO_ARC4) && 2227 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 2228 cmd->base_masks = HIFN_BASE_CMD_DECODE; 2229 maccrd = crd1; 2230 enccrd = crd2; 2231 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 2232 crd1->crd_alg == CRYPTO_ARC4 || 2233 crd1->crd_alg == CRYPTO_3DES_CBC || 2234 crd1->crd_alg == CRYPTO_AES_CBC) && 2235 (crd2->crd_alg == CRYPTO_MD5_HMAC_96 || 2236 crd2->crd_alg == CRYPTO_SHA1_HMAC_96 || 2237 crd2->crd_alg == CRYPTO_MD5 || 2238 crd2->crd_alg == CRYPTO_SHA1) && 2239 (crd1->crd_flags & CRD_F_ENCRYPT)) { 2240 enccrd = crd1; 2241 maccrd = crd2; 2242 } else { 2243 /* 2244 * We cannot order the 7751 as requested 2245 */ 2246 err = EINVAL; 2247 goto errout; 2248 } 2249 } 2250 2251 if (enccrd) { 2252 cmd->enccrd = enccrd; 2253 cmd->base_masks |= HIFN_BASE_CMD_CRYPT; 2254 switch (enccrd->crd_alg) { 2255 case CRYPTO_ARC4: 2256 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4; 2257 if ((enccrd->crd_flags & CRD_F_ENCRYPT) 2258 != sc->sc_sessions[session].hs_prev_op) 2259 sc->sc_sessions[session].hs_state = 2260 HS_STATE_USED; 2261 break; 2262 case CRYPTO_DES_CBC: 2263 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES | 2264 HIFN_CRYPT_CMD_MODE_CBC | 2265 HIFN_CRYPT_CMD_NEW_IV; 2266 break; 2267 case CRYPTO_3DES_CBC: 2268 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES | 2269 HIFN_CRYPT_CMD_MODE_CBC | 2270 HIFN_CRYPT_CMD_NEW_IV; 2271 break; 2272 case CRYPTO_AES_CBC: 2273 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES | 2274 HIFN_CRYPT_CMD_MODE_CBC | 2275 HIFN_CRYPT_CMD_NEW_IV; 2276 break; 2277 default: 2278 err = EINVAL; 2279 goto errout; 2280 } 2281 if (enccrd->crd_alg != CRYPTO_ARC4) { 2282 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ? 2283 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2284 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 2285 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2286 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2287 else 2288 bcopy(sc->sc_sessions[session].hs_iv, 2289 cmd->iv, ivlen); 2290 2291 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) 2292 == 0) { 2293 if (crp->crp_flags & CRYPTO_F_IMBUF) 2294 m_copyback(cmd->srcu.src_m, 2295 enccrd->crd_inject, 2296 ivlen, cmd->iv); 2297 else if (crp->crp_flags & CRYPTO_F_IOV) 2298 cuio_copyback(cmd->srcu.src_io, 2299 enccrd->crd_inject, 2300 ivlen, cmd->iv); 2301 } 2302 } else { 2303 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 2304 memcpy(cmd->iv, enccrd->crd_iv, ivlen); 2305 else if (crp->crp_flags & CRYPTO_F_IMBUF) 2306 m_copydata(cmd->srcu.src_m, 2307 enccrd->crd_inject, ivlen, cmd->iv); 2308 else if (crp->crp_flags & CRYPTO_F_IOV) 2309 cuio_copydata(cmd->srcu.src_io, 2310 enccrd->crd_inject, ivlen, cmd->iv); 2311 } 2312 } 2313 2314 cmd->ck = enccrd->crd_key; 2315 cmd->cklen = enccrd->crd_klen >> 3; 2316 2317 /* 2318 * Need to specify the size for the AES key in the masks. 2319 */ 2320 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) == 2321 HIFN_CRYPT_CMD_ALG_AES) { 2322 switch (cmd->cklen) { 2323 case 16: 2324 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128; 2325 break; 2326 case 24: 2327 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192; 2328 break; 2329 case 32: 2330 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256; 2331 break; 2332 default: 2333 err = EINVAL; 2334 goto errout; 2335 } 2336 } 2337 2338 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2339 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY; 2340 } 2341 2342 if (maccrd) { 2343 cmd->maccrd = maccrd; 2344 cmd->base_masks |= HIFN_BASE_CMD_MAC; 2345 2346 switch (maccrd->crd_alg) { 2347 case CRYPTO_MD5: 2348 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2349 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2350 HIFN_MAC_CMD_POS_IPSEC; 2351 break; 2352 case CRYPTO_MD5_HMAC_96: 2353 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 | 2354 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2355 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2356 break; 2357 case CRYPTO_SHA1: 2358 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2359 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH | 2360 HIFN_MAC_CMD_POS_IPSEC; 2361 break; 2362 case CRYPTO_SHA1_HMAC_96: 2363 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 | 2364 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC | 2365 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC; 2366 break; 2367 } 2368 2369 if ((maccrd->crd_alg == CRYPTO_SHA1_HMAC_96 || 2370 maccrd->crd_alg == CRYPTO_MD5_HMAC_96) && 2371 sc->sc_sessions[session].hs_state == HS_STATE_USED) { 2372 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY; 2373 memcpy(cmd->mac, maccrd->crd_key, maccrd->crd_klen >> 3); 2374 memset(cmd->mac + (maccrd->crd_klen >> 3), 0, 2375 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3)); 2376 } 2377 } 2378 2379 cmd->crp = crp; 2380 cmd->session_num = session; 2381 cmd->softc = sc; 2382 2383 err = hifn_crypto(sc, cmd, crp, hint); 2384 if (err == 0) { 2385 if (enccrd) 2386 sc->sc_sessions[session].hs_prev_op = 2387 enccrd->crd_flags & CRD_F_ENCRYPT; 2388 if (sc->sc_sessions[session].hs_state == HS_STATE_USED) 2389 sc->sc_sessions[session].hs_state = HS_STATE_KEY; 2390 return 0; 2391 } else if (err == ERESTART) { 2392 /* 2393 * There weren't enough resources to dispatch the request 2394 * to the part. Notify the caller so they'll requeue this 2395 * request and resubmit it again soon. 2396 */ 2397 #ifdef HIFN_DEBUG 2398 if (hifn_debug) 2399 printf(device_xname(&sc->sc_dv), "requeue request\n"); 2400 #endif 2401 free(cmd, M_DEVBUF); 2402 sc->sc_needwakeup |= CRYPTO_SYMQ; 2403 return (err); 2404 } 2405 2406 errout: 2407 if (cmd != NULL) 2408 free(cmd, M_DEVBUF); 2409 if (err == EINVAL) 2410 hifnstats.hst_invalid++; 2411 else 2412 hifnstats.hst_nomem++; 2413 crp->crp_etype = err; 2414 crypto_done(crp); 2415 return (0); 2416 } 2417 2418 static void 2419 hifn_abort(struct hifn_softc *sc) 2420 { 2421 struct hifn_dma *dma = sc->sc_dma; 2422 struct hifn_command *cmd; 2423 struct cryptop *crp; 2424 int i, u; 2425 2426 i = dma->resk; u = dma->resu; 2427 while (u != 0) { 2428 cmd = dma->hifn_commands[i]; 2429 KASSERT(cmd != NULL /*, ("hifn_abort: null cmd slot %u", i)*/); 2430 dma->hifn_commands[i] = NULL; 2431 crp = cmd->crp; 2432 2433 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) { 2434 /* Salvage what we can. */ 2435 hifnstats.hst_opackets++; 2436 hifn_callback(sc, cmd, dma->result_bufs[i]); 2437 } else { 2438 if (cmd->src_map == cmd->dst_map) { 2439 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2440 0, cmd->src_map->dm_mapsize, 2441 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 2442 } else { 2443 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2444 0, cmd->src_map->dm_mapsize, 2445 BUS_DMASYNC_POSTWRITE); 2446 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2447 0, cmd->dst_map->dm_mapsize, 2448 BUS_DMASYNC_POSTREAD); 2449 } 2450 2451 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2452 m_freem(cmd->srcu.src_m); 2453 crp->crp_buf = (void *)cmd->dstu.dst_m; 2454 } 2455 2456 /* non-shared buffers cannot be restarted */ 2457 if (cmd->src_map != cmd->dst_map) { 2458 /* 2459 * XXX should be EAGAIN, delayed until 2460 * after the reset. 2461 */ 2462 crp->crp_etype = ENOMEM; 2463 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2464 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2465 } else 2466 crp->crp_etype = ENOMEM; 2467 2468 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2469 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2470 2471 free(cmd, M_DEVBUF); 2472 if (crp->crp_etype != EAGAIN) 2473 crypto_done(crp); 2474 } 2475 2476 if (++i == HIFN_D_RES_RSIZE) 2477 i = 0; 2478 u--; 2479 } 2480 dma->resk = i; dma->resu = u; 2481 2482 /* Force upload of key next time */ 2483 for (i = 0; i < sc->sc_maxses; i++) 2484 if (sc->sc_sessions[i].hs_state == HS_STATE_KEY) 2485 sc->sc_sessions[i].hs_state = HS_STATE_USED; 2486 2487 hifn_reset_board(sc, 1); 2488 hifn_init_dma(sc); 2489 hifn_init_pci_registers(sc); 2490 } 2491 2492 static void 2493 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *resbuf) 2494 { 2495 struct hifn_dma *dma = sc->sc_dma; 2496 struct cryptop *crp = cmd->crp; 2497 struct cryptodesc *crd; 2498 struct mbuf *m; 2499 int totlen, i, u, ivlen; 2500 2501 if (cmd->src_map == cmd->dst_map) 2502 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2503 0, cmd->src_map->dm_mapsize, 2504 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2505 else { 2506 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2507 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2508 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2509 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2510 } 2511 2512 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2513 if (cmd->srcu.src_m != cmd->dstu.dst_m) { 2514 crp->crp_buf = (void *)cmd->dstu.dst_m; 2515 totlen = cmd->src_map->dm_mapsize; 2516 for (m = cmd->dstu.dst_m; m != NULL; m = m->m_next) { 2517 if (totlen < m->m_len) { 2518 m->m_len = totlen; 2519 totlen = 0; 2520 } else 2521 totlen -= m->m_len; 2522 } 2523 cmd->dstu.dst_m->m_pkthdr.len = 2524 cmd->srcu.src_m->m_pkthdr.len; 2525 m_freem(cmd->srcu.src_m); 2526 } 2527 } 2528 2529 if (cmd->sloplen != 0) { 2530 if (crp->crp_flags & CRYPTO_F_IMBUF) 2531 m_copyback((struct mbuf *)crp->crp_buf, 2532 cmd->src_map->dm_mapsize - cmd->sloplen, 2533 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2534 else if (crp->crp_flags & CRYPTO_F_IOV) 2535 cuio_copyback((struct uio *)crp->crp_buf, 2536 cmd->src_map->dm_mapsize - cmd->sloplen, 2537 cmd->sloplen, (void *)&dma->slop[cmd->slopidx]); 2538 } 2539 2540 i = dma->dstk; u = dma->dstu; 2541 while (u != 0) { 2542 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2543 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2544 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2545 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2546 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2547 offsetof(struct hifn_dma, dstr[i]), 2548 sizeof(struct hifn_desc), 2549 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2550 break; 2551 } 2552 if (++i == (HIFN_D_DST_RSIZE + 1)) 2553 i = 0; 2554 else 2555 u--; 2556 } 2557 dma->dstk = i; dma->dstu = u; 2558 2559 hifnstats.hst_obytes += cmd->dst_map->dm_mapsize; 2560 2561 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) == 2562 HIFN_BASE_CMD_CRYPT) { 2563 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2564 if (crd->crd_alg != CRYPTO_DES_CBC && 2565 crd->crd_alg != CRYPTO_3DES_CBC && 2566 crd->crd_alg != CRYPTO_AES_CBC) 2567 continue; 2568 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ? 2569 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH); 2570 if (crp->crp_flags & CRYPTO_F_IMBUF) 2571 m_copydata((struct mbuf *)crp->crp_buf, 2572 crd->crd_skip + crd->crd_len - ivlen, 2573 ivlen, 2574 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2575 else if (crp->crp_flags & CRYPTO_F_IOV) { 2576 cuio_copydata((struct uio *)crp->crp_buf, 2577 crd->crd_skip + crd->crd_len - ivlen, 2578 ivlen, 2579 cmd->softc->sc_sessions[cmd->session_num].hs_iv); 2580 } 2581 /* XXX We do not handle contig data */ 2582 break; 2583 } 2584 } 2585 2586 if (cmd->base_masks & HIFN_BASE_CMD_MAC) { 2587 u_int8_t *macbuf; 2588 2589 macbuf = resbuf + sizeof(struct hifn_base_result); 2590 if (cmd->base_masks & HIFN_BASE_CMD_COMP) 2591 macbuf += sizeof(struct hifn_comp_result); 2592 macbuf += sizeof(struct hifn_mac_result); 2593 2594 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 2595 int len; 2596 2597 if (crd->crd_alg == CRYPTO_MD5) 2598 len = 16; 2599 else if (crd->crd_alg == CRYPTO_SHA1) 2600 len = 20; 2601 else if (crd->crd_alg == CRYPTO_MD5_HMAC_96 || 2602 crd->crd_alg == CRYPTO_SHA1_HMAC_96) 2603 len = 12; 2604 else 2605 continue; 2606 2607 if (crp->crp_flags & CRYPTO_F_IMBUF) 2608 m_copyback((struct mbuf *)crp->crp_buf, 2609 crd->crd_inject, len, macbuf); 2610 else if ((crp->crp_flags & CRYPTO_F_IOV) && crp->crp_mac) 2611 memcpy(crp->crp_mac, (void *)macbuf, len); 2612 break; 2613 } 2614 } 2615 2616 if (cmd->src_map != cmd->dst_map) { 2617 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2618 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2619 } 2620 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2621 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2622 free(cmd, M_DEVBUF); 2623 crypto_done(crp); 2624 } 2625 2626 #ifdef HAVE_CRYPTO_LZS 2627 2628 static int 2629 hifn_compression(struct hifn_softc *sc, struct cryptop *crp, 2630 struct hifn_command *cmd) 2631 { 2632 struct cryptodesc *crd = crp->crp_desc; 2633 int s, err = 0; 2634 2635 cmd->compcrd = crd; 2636 cmd->base_masks |= HIFN_BASE_CMD_COMP; 2637 2638 if ((crp->crp_flags & CRYPTO_F_IMBUF) == 0) { 2639 /* 2640 * XXX can only handle mbufs right now since we can 2641 * XXX dynamically resize them. 2642 */ 2643 err = EINVAL; 2644 return (ENOMEM); 2645 } 2646 2647 if ((crd->crd_flags & CRD_F_COMP) == 0) 2648 cmd->base_masks |= HIFN_BASE_CMD_DECODE; 2649 if (crd->crd_alg == CRYPTO_LZS_COMP) 2650 cmd->comp_masks |= HIFN_COMP_CMD_ALG_LZS | 2651 HIFN_COMP_CMD_CLEARHIST; 2652 2653 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2654 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->src_map)) { 2655 err = ENOMEM; 2656 goto fail; 2657 } 2658 2659 if (bus_dmamap_create(sc->sc_dmat, HIFN_MAX_DMALEN, MAX_SCATTER, 2660 HIFN_MAX_SEGLEN, 0, BUS_DMA_NOWAIT, &cmd->dst_map)) { 2661 err = ENOMEM; 2662 goto fail; 2663 } 2664 2665 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2666 int len; 2667 2668 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map, 2669 cmd->srcu.src_m, BUS_DMA_NOWAIT)) { 2670 err = ENOMEM; 2671 goto fail; 2672 } 2673 2674 len = cmd->src_map->dm_mapsize / MCLBYTES; 2675 if ((cmd->src_map->dm_mapsize % MCLBYTES) != 0) 2676 len++; 2677 len *= MCLBYTES; 2678 2679 if ((crd->crd_flags & CRD_F_COMP) == 0) 2680 len *= 4; 2681 2682 if (len > HIFN_MAX_DMALEN) 2683 len = HIFN_MAX_DMALEN; 2684 2685 cmd->dstu.dst_m = hifn_mkmbuf_chain(len, cmd->srcu.src_m); 2686 if (cmd->dstu.dst_m == NULL) { 2687 err = ENOMEM; 2688 goto fail; 2689 } 2690 2691 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2692 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2693 err = ENOMEM; 2694 goto fail; 2695 } 2696 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2697 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map, 2698 cmd->srcu.src_io, BUS_DMA_NOWAIT)) { 2699 err = ENOMEM; 2700 goto fail; 2701 } 2702 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map, 2703 cmd->dstu.dst_io, BUS_DMA_NOWAIT)) { 2704 err = ENOMEM; 2705 goto fail; 2706 } 2707 } 2708 2709 if (cmd->src_map == cmd->dst_map) 2710 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2711 0, cmd->src_map->dm_mapsize, 2712 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 2713 else { 2714 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2715 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2716 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2717 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2718 } 2719 2720 cmd->crp = crp; 2721 /* 2722 * Always use session 0. The modes of compression we use are 2723 * stateless and there is always at least one compression 2724 * context, zero. 2725 */ 2726 cmd->session_num = 0; 2727 cmd->softc = sc; 2728 2729 s = splnet(); 2730 err = hifn_compress_enter(sc, cmd); 2731 splx(s); 2732 2733 if (err != 0) 2734 goto fail; 2735 return (0); 2736 2737 fail: 2738 if (cmd->dst_map != NULL) { 2739 if (cmd->dst_map->dm_nsegs > 0) 2740 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2741 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2742 } 2743 if (cmd->src_map != NULL) { 2744 if (cmd->src_map->dm_nsegs > 0) 2745 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2746 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2747 } 2748 free(cmd, M_DEVBUF); 2749 if (err == EINVAL) 2750 hifnstats.hst_invalid++; 2751 else 2752 hifnstats.hst_nomem++; 2753 crp->crp_etype = err; 2754 crypto_done(crp); 2755 return (0); 2756 } 2757 2758 /* 2759 * must be called at splnet() 2760 */ 2761 static int 2762 hifn_compress_enter(struct hifn_softc *sc, struct hifn_command *cmd) 2763 { 2764 struct hifn_dma *dma = sc->sc_dma; 2765 int cmdi, resi; 2766 u_int32_t cmdlen; 2767 2768 if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE || 2769 (dma->resu + 1) > HIFN_D_CMD_RSIZE) 2770 return (ENOMEM); 2771 2772 if ((dma->srcu + cmd->src_map->dm_nsegs) > HIFN_D_SRC_RSIZE || 2773 (dma->dstu + cmd->dst_map->dm_nsegs) > HIFN_D_DST_RSIZE) 2774 return (ENOMEM); 2775 2776 if (dma->cmdi == HIFN_D_CMD_RSIZE) { 2777 dma->cmdi = 0; 2778 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID | 2779 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2780 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE, 2781 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2782 } 2783 cmdi = dma->cmdi++; 2784 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]); 2785 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE); 2786 2787 /* .p for command/result already set */ 2788 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST | 2789 HIFN_D_MASKDONEIRQ); 2790 HIFN_CMDR_SYNC(sc, cmdi, 2791 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2792 dma->cmdu++; 2793 if (sc->sc_c_busy == 0) { 2794 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 2795 sc->sc_c_busy = 1; 2796 SET_LED(sc, HIFN_MIPSRST_LED0); 2797 } 2798 2799 /* 2800 * We don't worry about missing an interrupt (which a "command wait" 2801 * interrupt salvages us from), unless there is more than one command 2802 * in the queue. 2803 */ 2804 if (dma->cmdu > 1) { 2805 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT; 2806 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier); 2807 } 2808 2809 hifnstats.hst_ipackets++; 2810 hifnstats.hst_ibytes += cmd->src_map->dm_mapsize; 2811 2812 hifn_dmamap_load_src(sc, cmd); 2813 if (sc->sc_s_busy == 0) { 2814 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 2815 sc->sc_s_busy = 1; 2816 SET_LED(sc, HIFN_MIPSRST_LED1); 2817 } 2818 2819 /* 2820 * Unlike other descriptors, we don't mask done interrupt from 2821 * result descriptor. 2822 */ 2823 if (dma->resi == HIFN_D_RES_RSIZE) { 2824 dma->resi = 0; 2825 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID | 2826 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ); 2827 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE, 2828 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2829 } 2830 resi = dma->resi++; 2831 dma->hifn_commands[resi] = cmd; 2832 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD); 2833 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | 2834 HIFN_D_VALID | HIFN_D_LAST); 2835 HIFN_RESR_SYNC(sc, resi, 2836 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2837 dma->resu++; 2838 if (sc->sc_r_busy == 0) { 2839 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 2840 sc->sc_r_busy = 1; 2841 SET_LED(sc, HIFN_MIPSRST_LED2); 2842 } 2843 2844 if (cmd->sloplen) 2845 cmd->slopidx = resi; 2846 2847 hifn_dmamap_load_dst(sc, cmd); 2848 2849 if (sc->sc_d_busy == 0) { 2850 WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 2851 sc->sc_d_busy = 1; 2852 } 2853 sc->sc_active = 5; 2854 cmd->cmd_callback = hifn_callback_comp; 2855 return (0); 2856 } 2857 2858 static void 2859 hifn_callback_comp(struct hifn_softc *sc, struct hifn_command *cmd, 2860 u_int8_t *resbuf) 2861 { 2862 struct hifn_base_result baseres; 2863 struct cryptop *crp = cmd->crp; 2864 struct hifn_dma *dma = sc->sc_dma; 2865 struct mbuf *m; 2866 int err = 0, i, u; 2867 u_int32_t olen; 2868 bus_size_t dstsize; 2869 2870 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2871 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2872 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2873 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2874 2875 dstsize = cmd->dst_map->dm_mapsize; 2876 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map); 2877 2878 memcpy(&baseres, resbuf, sizeof(struct hifn_base_result)); 2879 2880 i = dma->dstk; u = dma->dstu; 2881 while (u != 0) { 2882 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2883 offsetof(struct hifn_dma, dstr[i]), sizeof(struct hifn_desc), 2884 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2885 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) { 2886 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap, 2887 offsetof(struct hifn_dma, dstr[i]), 2888 sizeof(struct hifn_desc), 2889 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2890 break; 2891 } 2892 if (++i == (HIFN_D_DST_RSIZE + 1)) 2893 i = 0; 2894 else 2895 u--; 2896 } 2897 dma->dstk = i; dma->dstu = u; 2898 2899 if (baseres.flags & htole16(HIFN_BASE_RES_DSTOVERRUN)) { 2900 bus_size_t xlen; 2901 2902 xlen = dstsize; 2903 2904 m_freem(cmd->dstu.dst_m); 2905 2906 if (xlen == HIFN_MAX_DMALEN) { 2907 /* We've done all we can. */ 2908 err = E2BIG; 2909 goto out; 2910 } 2911 2912 xlen += MCLBYTES; 2913 2914 if (xlen > HIFN_MAX_DMALEN) 2915 xlen = HIFN_MAX_DMALEN; 2916 2917 cmd->dstu.dst_m = hifn_mkmbuf_chain(xlen, 2918 cmd->srcu.src_m); 2919 if (cmd->dstu.dst_m == NULL) { 2920 err = ENOMEM; 2921 goto out; 2922 } 2923 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map, 2924 cmd->dstu.dst_m, BUS_DMA_NOWAIT)) { 2925 err = ENOMEM; 2926 goto out; 2927 } 2928 2929 bus_dmamap_sync(sc->sc_dmat, cmd->src_map, 2930 0, cmd->src_map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2931 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map, 2932 0, cmd->dst_map->dm_mapsize, BUS_DMASYNC_PREREAD); 2933 2934 /* already at splnet... */ 2935 err = hifn_compress_enter(sc, cmd); 2936 if (err != 0) 2937 goto out; 2938 return; 2939 } 2940 2941 olen = dstsize - (letoh16(baseres.dst_cnt) | 2942 (((letoh16(baseres.session) & HIFN_BASE_RES_DSTLEN_M) >> 2943 HIFN_BASE_RES_DSTLEN_S) << 16)); 2944 2945 crp->crp_olen = olen - cmd->compcrd->crd_skip; 2946 2947 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2948 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2949 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2950 2951 m = cmd->dstu.dst_m; 2952 if (m->m_flags & M_PKTHDR) 2953 m->m_pkthdr.len = olen; 2954 crp->crp_buf = (void *)m; 2955 for (; m != NULL; m = m->m_next) { 2956 if (olen >= m->m_len) 2957 olen -= m->m_len; 2958 else { 2959 m->m_len = olen; 2960 olen = 0; 2961 } 2962 } 2963 2964 m_freem(cmd->srcu.src_m); 2965 free(cmd, M_DEVBUF); 2966 crp->crp_etype = 0; 2967 crypto_done(crp); 2968 return; 2969 2970 out: 2971 if (cmd->dst_map != NULL) { 2972 if (cmd->src_map->dm_nsegs != 0) 2973 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2974 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map); 2975 } 2976 if (cmd->src_map != NULL) { 2977 if (cmd->src_map->dm_nsegs != 0) 2978 bus_dmamap_unload(sc->sc_dmat, cmd->src_map); 2979 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map); 2980 } 2981 if (cmd->dstu.dst_m != NULL) 2982 m_freem(cmd->dstu.dst_m); 2983 free(cmd, M_DEVBUF); 2984 crp->crp_etype = err; 2985 crypto_done(crp); 2986 } 2987 2988 static struct mbuf * 2989 hifn_mkmbuf_chain(int totlen, struct mbuf *mtemplate) 2990 { 2991 int len; 2992 struct mbuf *m, *m0, *mlast; 2993 2994 if (mtemplate->m_flags & M_PKTHDR) { 2995 len = MHLEN; 2996 MGETHDR(m0, M_DONTWAIT, MT_DATA); 2997 } else { 2998 len = MLEN; 2999 MGET(m0, M_DONTWAIT, MT_DATA); 3000 } 3001 if (m0 == NULL) 3002 return (NULL); 3003 if (len == MHLEN) 3004 M_DUP_PKTHDR(m0, mtemplate); 3005 MCLGET(m0, M_DONTWAIT); 3006 if (!(m0->m_flags & M_EXT)) 3007 m_freem(m0); 3008 len = MCLBYTES; 3009 3010 totlen -= len; 3011 m0->m_pkthdr.len = m0->m_len = len; 3012 mlast = m0; 3013 3014 while (totlen > 0) { 3015 MGET(m, M_DONTWAIT, MT_DATA); 3016 if (m == NULL) { 3017 m_freem(m0); 3018 return (NULL); 3019 } 3020 MCLGET(m, M_DONTWAIT); 3021 if (!(m->m_flags & M_EXT)) { 3022 m_freem(m0); 3023 return (NULL); 3024 } 3025 len = MCLBYTES; 3026 m->m_len = len; 3027 if (m0->m_flags & M_PKTHDR) 3028 m0->m_pkthdr.len += len; 3029 totlen -= len; 3030 3031 mlast->m_next = m; 3032 mlast = m; 3033 } 3034 3035 return (m0); 3036 } 3037 #endif /* HAVE_CRYPTO_LZS */ 3038 3039 static void 3040 hifn_write_4(struct hifn_softc *sc, int reggrp, bus_size_t reg, u_int32_t val) 3041 { 3042 /* 3043 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0 3044 * and Group 1 registers; avoid conditions that could create 3045 * burst writes by doing a read in between the writes. 3046 */ 3047 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3048 if (sc->sc_waw_lastgroup == reggrp && 3049 sc->sc_waw_lastreg == reg - 4) { 3050 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID); 3051 } 3052 sc->sc_waw_lastgroup = reggrp; 3053 sc->sc_waw_lastreg = reg; 3054 } 3055 if (reggrp == 0) 3056 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val); 3057 else 3058 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val); 3059 3060 } 3061 3062 static u_int32_t 3063 hifn_read_4(struct hifn_softc *sc, int reggrp, bus_size_t reg) 3064 { 3065 if (sc->sc_flags & HIFN_NO_BURSTWRITE) { 3066 sc->sc_waw_lastgroup = -1; 3067 sc->sc_waw_lastreg = 1; 3068 } 3069 if (reggrp == 0) 3070 return (bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg)); 3071 return (bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg)); 3072 } 3073
Indexes created Thu Sep 25 15:09:46 GMT 2025