1 /* $NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * sun8i_crypto -- Allwinner Crypto Engine driver 34 * 35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37 * the moment, but in principle it could be wired up with opencrypto(9) 38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41 * 42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43 */ 44 45 #include <sys/cdefs.h> 46 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $"); 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/atomic.h> 51 #include <sys/bus.h> 52 #include <sys/callout.h> 53 #include <sys/conf.h> 54 #include <sys/cprng.h> 55 #include <sys/device.h> 56 #include <sys/kernel.h> 57 #include <sys/kmem.h> 58 #include <sys/mbuf.h> 59 #include <sys/mutex.h> 60 #include <sys/rndsource.h> 61 #include <sys/sdt.h> 62 #include <sys/sysctl.h> 63 #include <sys/workqueue.h> 64 65 #include <dev/fdt/fdtvar.h> 66 67 #include <opencrypto/cryptodev.h> 68 69 #include <arm/sunxi/sun8i_crypto.h> 70 71 #define SUN8I_CRYPTO_TIMEOUT hz 72 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 73 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 74 75 struct sun8i_crypto_config { 76 u_int mod_rate; /* module clock rate */ 77 }; 78 79 /* 80 * The module clock is set to 50 MHz on H3, 300 MHz otherwise. 81 * From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c: 82 * Module clock is lower on H3 than other SoC due to some DMA 83 * timeout occurring with high value. 84 */ 85 static const struct sun8i_crypto_config sun50i_a64_crypto_config = { 86 .mod_rate = 300*1000*1000, 87 }; 88 89 static const struct sun8i_crypto_config sun50i_h5_crypto_config = { 90 .mod_rate = 300*1000*1000, 91 }; 92 93 static const struct sun8i_crypto_config sun8i_h3_crypto_config = { 94 .mod_rate = 50*1000*1000, 95 }; 96 97 struct sun8i_crypto_task; 98 99 struct sun8i_crypto_buf { 100 bus_dma_segment_t cb_seg[1]; 101 int cb_nsegs; 102 void *cb_kva; 103 }; 104 105 struct sun8i_crypto_softc { 106 device_t sc_dev; 107 bus_space_tag_t sc_bst; 108 bus_space_handle_t sc_bsh; 109 bus_dma_tag_t sc_dmat; 110 struct pool_cache *sc_taskpool; 111 112 const struct sun8i_crypto_config *sc_cfg; 113 114 struct workqueue *sc_wq; 115 void *sc_ih; 116 bool sc_polling; 117 118 kmutex_t sc_lock; 119 struct sun8i_crypto_chan { 120 struct sun8i_crypto_task *cc_task; 121 unsigned cc_starttime; 122 } sc_chan[SUN8I_CRYPTO_NCHAN]; 123 struct callout sc_timeout; 124 125 kmutex_t sc_intr_lock; 126 uint32_t sc_done; 127 uint32_t sc_esr; 128 struct work sc_work; 129 bool sc_work_pending; 130 131 struct sun8i_crypto_rng { 132 struct sun8i_crypto_buf cr_buf; 133 struct sun8i_crypto_task *cr_task; 134 struct krndsource cr_rndsource; 135 bool cr_pending; 136 } sc_rng; 137 struct sun8i_crypto_selftest { 138 struct sun8i_crypto_buf cs_in; 139 struct sun8i_crypto_buf cs_key; 140 struct sun8i_crypto_buf cs_out; 141 struct sun8i_crypto_task *cs_task; 142 bool cs_pending; 143 bool cs_passed; 144 } sc_selftest; 145 struct sun8i_crypto_sysctl { 146 struct sysctllog *cy_log; 147 const struct sysctlnode *cy_root_node; 148 const struct sysctlnode *cy_trng_node; 149 } sc_sysctl; 150 struct sun8i_crypto_opencrypto { 151 uint32_t co_driverid; 152 } sc_opencrypto; 153 }; 154 155 struct sun8i_crypto_task { 156 struct sun8i_crypto_buf ct_descbuf; 157 struct sun8i_crypto_taskdesc *ct_desc; 158 struct sun8i_crypto_buf ct_ivbuf; 159 void *ct_iv; 160 struct sun8i_crypto_buf ct_ctrbuf; 161 void *ct_ctr; 162 bus_dmamap_t ct_descmap; 163 bus_dmamap_t ct_keymap; 164 bus_dmamap_t ct_ivmap; /* IV input */ 165 bus_dmamap_t ct_ctrmap; /* updated IV output */ 166 bus_dmamap_t ct_srcmap; 167 bus_dmamap_t ct_dstmap; 168 uint32_t ct_nbytes; 169 int ct_flags; 170 #define TASK_KEY __BIT(0) 171 #define TASK_IV __BIT(1) 172 #define TASK_CTR __BIT(2) 173 #define TASK_SRC __BIT(3) 174 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 175 void (*ct_callback)(struct sun8i_crypto_softc *, 176 struct sun8i_crypto_task *, void *, int); 177 void *ct_cookie; 178 }; 179 180 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 181 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 182 183 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 184 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 185 186 /* 187 * Forward declarations 188 */ 189 190 static int sun8i_crypto_match(device_t, cfdata_t, void *); 191 static void sun8i_crypto_attach(device_t, device_t, void *); 192 193 static int sun8i_crypto_task_ctor(void *, void *, int); 194 static void sun8i_crypto_task_dtor(void *, void *); 195 static struct sun8i_crypto_task * 196 sun8i_crypto_task_get(struct sun8i_crypto_softc *, 197 void (*)(struct sun8i_crypto_softc *, 198 struct sun8i_crypto_task *, void *, int), 199 void *, int); 200 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 201 struct sun8i_crypto_task *); 202 203 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 204 struct sun8i_crypto_task *, uint32_t, 205 uint32_t, uint32_t, uint32_t); 206 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 207 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 208 209 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 210 struct sun8i_crypto_task *, uint32_t); 211 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 212 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 213 214 static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 215 struct sun8i_crypto_task *); 216 217 static void sun8i_crypto_timeout(void *); 218 static int sun8i_crypto_intr(void *); 219 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 220 static void sun8i_crypto_worker(struct work *, void *); 221 222 static bool sun8i_crypto_poll(struct sun8i_crypto_softc *, uint32_t *, 223 uint32_t *); 224 static bool sun8i_crypto_done(struct sun8i_crypto_softc *, uint32_t, 225 uint32_t, unsigned); 226 static bool sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 227 int); 228 229 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 230 struct sun8i_crypto_buf *, int); 231 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 232 struct sun8i_crypto_buf *); 233 234 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 235 static void sun8i_crypto_rng_get(size_t, void *); 236 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 237 struct sun8i_crypto_task *, void *, int); 238 239 static bool sun8i_crypto_selftest(struct sun8i_crypto_softc *); 240 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 241 struct sun8i_crypto_task *, void *, int); 242 243 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 244 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 245 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 246 struct sun8i_crypto_task *, void *, int); 247 248 static void sun8i_crypto_register(struct sun8i_crypto_softc *); 249 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 250 static int sun8i_crypto_newsession(void *, uint32_t *, 251 struct cryptoini *); 252 static void sun8i_crypto_freesession(void *, uint64_t); 253 static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 254 static int sun8i_crypto_process(void *, struct cryptop *, int); 255 static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 256 struct sun8i_crypto_task *, void *, int); 257 258 /* 259 * Probes 260 */ 261 262 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 263 "bus_size_t"/*reg*/, 264 "uint32_t"/*value*/); 265 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 266 "bus_size_t"/*reg*/, 267 "uint32_t"/*write*/); 268 269 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 270 "struct sun8i_crypto_task *"/*task*/); 271 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 272 "int"/*error*/); 273 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 274 "struct sun8i_crypto_task *"/*task*/); 275 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 276 "struct sun8i_crypto_task *"/*task*/); 277 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 278 "struct sun8i_crypto_task *"/*task*/); 279 280 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 281 "struct sun8i_crypto_task *"/*task*/, 282 "uint32_t"/*tdqc*/, 283 "uint32_t"/*tdqs*/, 284 "uint32_t"/*tdqa*/, 285 "struct sun8i_crypto_taskdesc *"/*desc*/, 286 "int"/*error*/); 287 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 288 "struct sun8i_crypto_task *"/*task*/, 289 "bus_addr_t"/*ds_addr*/, 290 "bus_size_t"/*ds_len*/); 291 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 292 "struct sun8i_crypto_task *"/*task*/, 293 "int"/*error*/); 294 295 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 296 "struct sun8i_crypto_softc *"/*sc*/, 297 "struct sun8i_crypto_task *"/*task*/, 298 "int"/*error*/); 299 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 300 "struct sun8i_crypto_softc *"/*sc*/, 301 "struct sun8i_crypto_task *"/*task*/, 302 "unsigned"/*chan*/); 303 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 304 "struct sun8i_crypto_softc *"/*sc*/, 305 "uint32_t"/*isr*/, 306 "uint32_t"/*esr*/); 307 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 308 "struct sun8i_crypto_softc *"/*sc*/, 309 "unsigned"/*chan*/, 310 "int"/*error*/); 311 312 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 313 "struct sun8i_crypto_softc *"/*sc*/, 314 "struct cryptop *"/*crp*/, 315 "int"/*hint*/); 316 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 317 "struct sun8i_crypto_softc *"/*sc*/, 318 "struct cryptop *"/*crp*/, 319 "int"/*hint*/); 320 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 321 "struct sun8i_crypto_softc *"/*sc*/, 322 "struct cryptop *"/*crp*/, 323 "int"/*hint*/, 324 "struct sun8i_crypto_task *"/*task*/); 325 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 326 "struct sun8i_crypto_softc *"/*sc*/, 327 "struct cryptop *"/*crp*/, 328 "int"/*error*/); 329 330 /* 331 * Register access 332 */ 333 334 static uint32_t 335 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 336 { 337 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 338 339 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 340 return v; 341 } 342 343 static void 344 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 345 { 346 347 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 348 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 349 } 350 351 /* 352 * Autoconf goo 353 */ 354 355 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 356 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 357 358 static const struct device_compatible_entry compat_data[] = { 359 { .compat = "allwinner,sun50i-a64-crypto", 360 .data = &sun50i_a64_crypto_config }, 361 { .compat = "allwinner,sun50i-h5-crypto", 362 .data = &sun50i_h5_crypto_config }, 363 { .compat = "allwinner,sun8i-h3-crypto", 364 .data = &sun8i_h3_crypto_config }, 365 DEVICE_COMPAT_EOL 366 }; 367 368 static int 369 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 370 { 371 const struct fdt_attach_args *const faa = aux; 372 373 return of_compatible_match(faa->faa_phandle, compat_data); 374 } 375 376 static void 377 sun8i_crypto_attach(device_t parent, device_t self, void *aux) 378 { 379 struct sun8i_crypto_softc *const sc = device_private(self); 380 const struct fdt_attach_args *const faa = aux; 381 bus_addr_t addr; 382 bus_size_t size; 383 const int phandle = faa->faa_phandle; 384 char intrstr[128]; 385 struct clk *clk; 386 struct fdtbus_reset *rst; 387 u_int mod_rate; 388 389 sc->sc_dev = self; 390 sc->sc_dmat = faa->faa_dmat; 391 sc->sc_bst = faa->faa_bst; 392 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 393 0, 0, 0, "sun8icry", NULL, IPL_SOFTSERIAL, 394 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 395 sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data; 396 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); 397 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_VM); 398 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 399 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 400 if (workqueue_create(&sc->sc_wq, device_xname(self), 401 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 402 aprint_error(": couldn't create workqueue\n"); 403 return; 404 } 405 406 /* 407 * Prime the pool with enough tasks that each channel can be 408 * busy with a task as we prepare another task for when it's 409 * done. 410 */ 411 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 412 413 /* Get and map device registers. */ 414 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 415 aprint_error(": couldn't get registers\n"); 416 return; 417 } 418 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 419 aprint_error(": couldn't map registers\n"); 420 return; 421 } 422 423 /* Get an interrupt handle. */ 424 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 425 aprint_error(": failed to decode interrupt\n"); 426 return; 427 } 428 429 /* Enable the bus clock. */ 430 if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 431 aprint_error(": couldn't enable bus clock\n"); 432 return; 433 } 434 435 /* Get the module clock and set it. */ 436 mod_rate = sc->sc_cfg->mod_rate; 437 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 438 if (clk_enable(clk) != 0) { 439 aprint_error(": couldn't enable CE clock\n"); 440 return; 441 } 442 if (clk_set_rate(clk, mod_rate) != 0) { 443 aprint_error(": couldn't set CE clock to %d MHz\n", 444 mod_rate / (1000 * 1000)); 445 return; 446 } 447 } 448 449 /* Get a reset handle if we need and try to deassert it. */ 450 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 451 if (fdtbus_reset_deassert(rst) != 0) { 452 aprint_error(": couldn't de-assert reset\n"); 453 return; 454 } 455 } 456 457 aprint_naive("\n"); 458 aprint_normal(": Crypto Engine\n"); 459 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 460 461 /* 462 * Disable and clear interrupts. Start in polling mode for 463 * synchronous self-tests and the first RNG draw. 464 */ 465 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 466 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 467 sc->sc_polling = true; 468 469 /* Establish an interrupt handler. */ 470 sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM, 471 FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self)); 472 if (sc->sc_ih == NULL) { 473 aprint_error_dev(self, "failed to establish interrupt on %s\n", 474 intrstr); 475 return; 476 } 477 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 478 479 /* Perform self-tests. If they fail, stop here. */ 480 if (!sun8i_crypto_selftest(sc)) 481 return; 482 483 /* 484 * Set up the RNG. This will try to synchronously draw the 485 * first sample by polling, so do this before we establish 486 * the interrupt handler. 487 */ 488 sun8i_crypto_rng_attach(sc); 489 490 /* 491 * Self-test has passed and first RNG draw has finished. Use 492 * interrupts, not polling, for all subsequent tasks. Set this 493 * atomically in case the interrupt handler has fired -- can't 494 * be from us because we've kept ICR set to 0 to mask all 495 * interrupts, but in case the interrupt vector is shared. 496 */ 497 atomic_store_relaxed(&sc->sc_polling, false); 498 499 /* Attach the sysctl. */ 500 sun8i_crypto_sysctl_attach(sc); 501 502 /* Register opencrypto handlers. */ 503 sun8i_crypto_register(sc); 504 } 505 506 static int 507 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 508 { 509 struct sun8i_crypto_softc *sc = cookie; 510 struct sun8i_crypto_task *task = vtask; 511 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 512 int error; 513 514 /* Create a DMA buffer for the task descriptor. */ 515 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 516 &task->ct_descbuf, dmaflags); 517 if (error) 518 goto fail0; 519 task->ct_desc = task->ct_descbuf.cb_kva; 520 521 /* Create DMA buffers for the IV and CTR. */ 522 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 523 &task->ct_ivbuf, dmaflags); 524 if (error) 525 goto fail1; 526 task->ct_iv = task->ct_ivbuf.cb_kva; 527 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 528 &task->ct_ctrbuf, dmaflags); 529 if (error) 530 goto fail2; 531 task->ct_ctr = task->ct_ctrbuf.cb_kva; 532 533 /* Create a DMA map for the task descriptor and preload it. */ 534 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 535 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 536 if (error) 537 goto fail3; 538 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 539 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 540 if (error) 541 goto fail4; 542 543 /* Create DMA maps for the key, IV, and CTR. */ 544 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 545 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 546 if (error) 547 goto fail5; 548 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 549 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 550 if (error) 551 goto fail6; 552 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 553 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 554 if (error) 555 goto fail7; 556 557 /* Create DMA maps for the src and dst scatter/gather vectors. */ 558 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 559 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 560 &task->ct_srcmap); 561 if (error) 562 goto fail8; 563 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 564 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 565 &task->ct_dstmap); 566 if (error) 567 goto fail9; 568 569 /* Success! */ 570 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 571 return 0; 572 573 fail10: __unused 574 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 575 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 576 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 577 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 578 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 579 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 580 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 581 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 582 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 583 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 584 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 585 return error; 586 } 587 588 static void 589 sun8i_crypto_task_dtor(void *cookie, void *vtask) 590 { 591 struct sun8i_crypto_softc *sc = cookie; 592 struct sun8i_crypto_task *task = vtask; 593 594 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 595 596 /* XXX Zero the bounce buffers if there are any. */ 597 598 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 599 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 600 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 601 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 602 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 603 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 604 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 605 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 606 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 607 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 608 } 609 610 /* 611 * sun8i_crypto_task_get(sc, callback, cookie, pflags) 612 * 613 * Allocate a task that will call callback(sc, task, cookie, 614 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 615 * PR_NOWAIT, may fail and return NULL. No further allocation is 616 * needed to submit the task if this succeeds (although task 617 * submission may still fail if all channels are busy). 618 */ 619 static struct sun8i_crypto_task * 620 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 621 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 622 void *, int), 623 void *cookie, int pflags) 624 { 625 struct sun8i_crypto_task *task; 626 627 /* Allocate a task, or fail if we can't. */ 628 task = pool_cache_get(sc->sc_taskpool, pflags); 629 if (task == NULL) 630 goto out; 631 632 /* Set up flags and the callback. */ 633 task->ct_flags = 0; 634 task->ct_callback = callback; 635 task->ct_cookie = cookie; 636 637 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 638 return task; 639 } 640 641 /* 642 * sun8i_crypto_task_invalid(sc, task, cookie, error) 643 * 644 * Callback for a task not currently in use, to detect errors. 645 */ 646 static void 647 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 648 struct sun8i_crypto_task *task, void *cookie, int error) 649 { 650 void (*callback)(struct sun8i_crypto_softc *, 651 struct sun8i_crypto_task *, void *, int) = cookie; 652 653 panic("task for callback %p used after free", callback); 654 } 655 656 /* 657 * sun8i_crypto_task_put(sc, task) 658 * 659 * Free a task obtained with sun8i_crypto_task_get. 660 */ 661 static void 662 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 663 struct sun8i_crypto_task *task) 664 { 665 666 SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 667 668 task->ct_cookie = task->ct_callback; 669 task->ct_callback = &sun8i_crypto_task_invalid; 670 pool_cache_put(sc->sc_taskpool, task); 671 } 672 673 /* 674 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 675 * 676 * Set up the task descriptor after the relevant DMA maps have 677 * been loaded for a transfer of nbytes. bus_dmamap_sync matches 678 * sun8i_crypto_chan_done. May fail if input is inadequately 679 * aligned. 680 * 681 * XXX Teach this to support task chains. 682 */ 683 static int 684 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 685 struct sun8i_crypto_task *task, uint32_t nbytes, 686 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 687 { 688 struct sun8i_crypto_taskdesc *desc = task->ct_desc; 689 int error; 690 691 KASSERT(tdqs == 0 || tdqa == 0); 692 KASSERT(nbytes % 4 == 0); 693 694 memset(desc, 0, sizeof(*desc)); 695 696 /* Always enable interrupt for the task. */ 697 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 698 699 desc->td_tdqc = htole32(tdqc); 700 desc->td_tdqs = htole32(tdqs); 701 desc->td_tdqa = htole32(tdqa); 702 703 if (task->ct_flags & TASK_KEY) { 704 bus_dmamap_t keymap = task->ct_keymap; 705 KASSERT(keymap->dm_nsegs == 1); 706 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 707 bus_dmamap_sync(sc->sc_dmat, keymap, 0, 708 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 709 } 710 if (task->ct_flags & TASK_IV) { 711 bus_dmamap_t ivmap = task->ct_ivmap; 712 KASSERT(ivmap->dm_nsegs == 1); 713 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 714 bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 715 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 716 } 717 if (task->ct_flags & TASK_CTR) { 718 bus_dmamap_t ctrmap = task->ct_ctrmap; 719 KASSERT(ctrmap->dm_nsegs == 1); 720 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 721 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 722 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 723 } 724 725 if (task->ct_flags & TASK_BYTES) 726 desc->td_datalen = htole32(nbytes); 727 else 728 desc->td_datalen = htole32(nbytes/4); 729 730 if (task->ct_flags & TASK_SRC) { 731 bus_dmamap_t srcmap = task->ct_srcmap; 732 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 733 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 734 nbytes); 735 if (error) 736 return error; 737 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 738 BUS_DMASYNC_PREWRITE); 739 } 740 741 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 742 nbytes); 743 if (error) 744 goto out; 745 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 746 BUS_DMASYNC_PREREAD); 747 748 task->ct_nbytes = nbytes; 749 750 /* Success! */ 751 error = 0; 752 753 out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 754 task, tdqc, tdqs, tdqa, desc, error); 755 return error; 756 } 757 758 /* 759 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 760 * 761 * Set up a task's scatter/gather vector -- src or dst -- with the 762 * given DMA map for a transfer of nbytes. May fail if input is 763 * inadequately aligned. 764 */ 765 static int 766 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 767 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 768 uint32_t nbytes __diagused) 769 { 770 uint32_t total __diagused = 0; 771 unsigned i; 772 773 /* 774 * Verify that the alignment is correct and initialize the 775 * scatter/gather vector. 776 */ 777 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 778 for (i = 0; i < map->dm_nsegs; i++) { 779 if ((map->dm_segs[i].ds_addr % 4) | 780 (map->dm_segs[i].ds_len % 4)) { 781 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 782 task, 783 map->dm_segs[i].ds_addr, 784 map->dm_segs[i].ds_len); 785 return EINVAL; 786 } 787 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 788 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 789 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 790 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 791 total += map->dm_segs[i].ds_len; 792 } 793 794 /* Set the remainder to zero. */ 795 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 796 adrlen[i].adr = 0; 797 adrlen[i].len = 0; 798 } 799 800 /* Verify the total size matches the transfer length. */ 801 KASSERT(total == nbytes); 802 803 /* Success! */ 804 return 0; 805 } 806 807 /* 808 * sun8i_crypto_task_load_trng(task, nbytes) 809 * 810 * Set up the task descriptor for a transfer of nbytes from the 811 * TRNG. 812 */ 813 static int 814 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 815 struct sun8i_crypto_task *task, uint32_t nbytes) 816 { 817 uint32_t tdqc = 0; 818 819 /* Caller must provide dst only. */ 820 KASSERT((task->ct_flags & TASK_KEY) == 0); 821 KASSERT((task->ct_flags & TASK_IV) == 0); 822 KASSERT((task->ct_flags & TASK_CTR) == 0); 823 KASSERT((task->ct_flags & TASK_SRC) == 0); 824 825 /* Set up the task descriptor queue control words. */ 826 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 827 SUN8I_CRYPTO_TDQC_METHOD); 828 829 /* Fill in the descriptor. */ 830 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 831 } 832 833 static int 834 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 835 struct sun8i_crypto_task *task, 836 uint32_t nbytes, uint32_t keysize, uint32_t dir) 837 { 838 uint32_t tdqc = 0, tdqs = 0; 839 840 /* Caller must provide key, src, and dst only. */ 841 KASSERT(task->ct_flags & TASK_KEY); 842 KASSERT((task->ct_flags & TASK_IV) == 0); 843 KASSERT((task->ct_flags & TASK_CTR) == 0); 844 KASSERT(task->ct_flags & TASK_SRC); 845 846 /* Set up the task descriptor queue control word. */ 847 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 848 SUN8I_CRYPTO_TDQC_METHOD); 849 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 850 851 #ifdef DIAGNOSTIC 852 switch (keysize) { 853 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 854 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 855 break; 856 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 857 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 858 break; 859 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 860 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 861 break; 862 } 863 #endif 864 865 /* Set up the symmetric control word. */ 866 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 867 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 868 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 869 SUN8I_CRYPTO_TDQS_OP_MODE); 870 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 871 872 /* Fill in the descriptor. */ 873 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 874 } 875 876 /* 877 * sun8i_crypto_submit(sc, task) 878 * 879 * Submit a task to the crypto engine after it has been loaded 880 * with sun8i_crypto_task_load. On success, guarantees to 881 * eventually call the task's callback. 882 */ 883 static int 884 sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 885 struct sun8i_crypto_task *task) 886 { 887 unsigned i, retries = 0; 888 uint32_t icr; 889 int error = 0; 890 891 /* One at a time at the device registers, please. */ 892 mutex_enter(&sc->sc_lock); 893 894 /* Find a channel. */ 895 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 896 if (sc->sc_chan[i].cc_task == NULL) 897 break; 898 } 899 if (i == SUN8I_CRYPTO_NCHAN) { 900 device_printf(sc->sc_dev, "no free channels\n"); 901 error = ERESTART; 902 goto out; 903 } 904 905 /* 906 * Set the channel id. Caller is responsible for setting up 907 * all other parts of the descriptor. 908 */ 909 task->ct_desc->td_cid = htole32(i); 910 911 /* 912 * Prepare to send the descriptor to the device by DMA. 913 * Matches POSTWRITE in sun8i_crypto_chan_done. 914 */ 915 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 916 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 917 918 /* Confirm we're ready to go. */ 919 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 920 device_printf(sc->sc_dev, "TLR not clear\n"); 921 error = EIO; 922 goto out; 923 } 924 925 /* 926 * Enable interrupts for this channel, unless we're still 927 * polling. 928 */ 929 if (!sc->sc_polling) { 930 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 931 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 932 SUN8I_CRYPTO_ICR_INTR_EN); 933 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 934 } 935 936 /* Set the task descriptor queue address. */ 937 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 938 task->ct_descmap->dm_segs[0].ds_addr); 939 940 /* Notify the engine to load it, and wait for acknowledgement. */ 941 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 942 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 943 { 944 /* 945 * XXX Timeout pulled from arse. Is it even important 946 * to wait here? 947 */ 948 if (++retries == 1000) { 949 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 950 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 951 /* 952 * Hope it clears eventually; if not, we'll 953 * time out. 954 */ 955 break; 956 } 957 DELAY(1); 958 } 959 960 /* 961 * Loaded up and ready to go. Start a timer ticking if it's 962 * not already and we're not polling. 963 */ 964 sc->sc_chan[i].cc_task = task; 965 sc->sc_chan[i].cc_starttime = getticks(); 966 if (!sc->sc_polling && !callout_pending(&sc->sc_timeout)) 967 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 968 969 out: /* Done! */ 970 if (error) 971 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 972 sc, task, error); 973 else 974 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 975 sc, task, i); 976 mutex_exit(&sc->sc_lock); 977 return error; 978 } 979 980 /* 981 * sun8i_crypto_timeout(cookie) 982 * 983 * Timeout handler. Schedules work in a thread to cancel all 984 * pending tasks that were started long enough ago we're bored of 985 * waiting for them. 986 */ 987 static void 988 sun8i_crypto_timeout(void *cookie) 989 { 990 struct sun8i_crypto_softc *sc = cookie; 991 992 mutex_enter(&sc->sc_intr_lock); 993 sun8i_crypto_schedule_worker(sc); 994 mutex_exit(&sc->sc_intr_lock); 995 } 996 997 /* 998 * sun8i_crypto_intr(cookie) 999 * 1000 * Device interrupt handler. Find what channels have completed, 1001 * whether with success or with failure, and schedule work in 1002 * thread context to invoke the appropriate callbacks. 1003 */ 1004 static int 1005 sun8i_crypto_intr(void *cookie) 1006 { 1007 struct sun8i_crypto_softc *sc = cookie; 1008 uint32_t done, esr; 1009 1010 if (atomic_load_relaxed(&sc->sc_polling) || 1011 !sun8i_crypto_poll(sc, &done, &esr)) 1012 return 0; /* not ours */ 1013 1014 mutex_enter(&sc->sc_intr_lock); 1015 sun8i_crypto_schedule_worker(sc); 1016 sc->sc_done |= done; 1017 sc->sc_esr |= esr; 1018 mutex_exit(&sc->sc_intr_lock); 1019 1020 return 1; 1021 } 1022 1023 /* 1024 * sun8i_crypto_schedule_worker(sc) 1025 * 1026 * Ensure that crypto engine thread context work to invoke task 1027 * callbacks will run promptly. Idempotent. 1028 */ 1029 static void 1030 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 1031 { 1032 1033 KASSERT(mutex_owned(&sc->sc_intr_lock)); 1034 1035 /* Start the worker if necessary. */ 1036 if (!sc->sc_work_pending) { 1037 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1038 sc->sc_work_pending = true; 1039 } 1040 } 1041 1042 /* 1043 * sun8i_crypto_worker(wk, cookie) 1044 * 1045 * Thread-context worker: Invoke all task callbacks for which the 1046 * device has notified us of completion or for which we gave up 1047 * waiting. 1048 */ 1049 static void 1050 sun8i_crypto_worker(struct work *wk, void *cookie) 1051 { 1052 struct sun8i_crypto_softc *sc = cookie; 1053 uint32_t done, esr; 1054 1055 /* 1056 * Under the interrupt lock, acknowledge our work and claim the 1057 * done mask and error status. 1058 */ 1059 mutex_enter(&sc->sc_intr_lock); 1060 KASSERT(sc->sc_work_pending); 1061 sc->sc_work_pending = false; 1062 done = sc->sc_done; 1063 esr = sc->sc_esr; 1064 sc->sc_done = 0; 1065 sc->sc_esr = 0; 1066 mutex_exit(&sc->sc_intr_lock); 1067 1068 /* 1069 * If we cleared any channels, it is time to allow opencrypto 1070 * to issue new operations. Asymmetric operations (which we 1071 * don't support, at the moment, but we could) and symmetric 1072 * operations (which we do) use the same task channels, so we 1073 * unblock both kinds. 1074 */ 1075 if (sun8i_crypto_done(sc, done, esr, getticks())) { 1076 crypto_unblock(sc->sc_opencrypto.co_driverid, 1077 CRYPTO_SYMQ|CRYPTO_ASYMQ); 1078 } 1079 } 1080 1081 /* 1082 * sun8i_crypto_poll(sc, &done, &esr) 1083 * 1084 * Poll for completion. Sets done and esr to the mask of done 1085 * channels and error channels. Returns true if anything was 1086 * done or failed. 1087 */ 1088 static bool 1089 sun8i_crypto_poll(struct sun8i_crypto_softc *sc, 1090 uint32_t *donep, uint32_t *esrp) 1091 { 1092 uint32_t isr, esr; 1093 1094 /* 1095 * Get and acknowledge the interrupts and error status. 1096 * 1097 * XXX Data sheet says the error status register is read-only, 1098 * but then advises writing 1 to bit x1xx (keysram access error 1099 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 1100 */ 1101 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 1102 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 1103 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 1104 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 1105 1106 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 1107 1108 *donep = __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 1109 *esrp = esr; 1110 1111 return *donep || *esrp; 1112 } 1113 1114 /* 1115 * sun8i_crypto_done(sc, done, esr, now) 1116 * 1117 * Invoke all task callbacks for the channels in done or esr, or 1118 * for which we gave up waiting, according to the time `now'. 1119 * Returns true if any channels completed or timed out. 1120 */ 1121 static bool 1122 sun8i_crypto_done(struct sun8i_crypto_softc *sc, uint32_t done, uint32_t esr, 1123 unsigned now) 1124 { 1125 uint32_t esr_chan; 1126 unsigned i; 1127 bool anydone = false; 1128 bool schedtimeout = false; 1129 int error; 1130 1131 /* Under the lock, process the channels. */ 1132 mutex_enter(&sc->sc_lock); 1133 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 1134 /* Check whether the channel is done. */ 1135 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 1136 /* Nope. Do we have a task to time out? */ 1137 if (sc->sc_chan[i].cc_task != NULL) { 1138 if (now - sc->sc_chan[i].cc_starttime >= 1139 SUN8I_CRYPTO_TIMEOUT) { 1140 anydone |= sun8i_crypto_chan_done(sc, 1141 i, ETIMEDOUT); 1142 } else { 1143 schedtimeout = true; 1144 } 1145 } 1146 continue; 1147 } 1148 1149 /* Channel is done. Interpret the error if any. */ 1150 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1151 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1152 device_printf(sc->sc_dev, "channel %u:" 1153 " alg not supported\n", i); 1154 error = ENODEV; 1155 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1156 device_printf(sc->sc_dev, "channel %u:" 1157 " data length error\n", i); 1158 error = EIO; /* XXX */ 1159 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1160 device_printf(sc->sc_dev, "channel %u:" 1161 " key sram error\n", i); 1162 error = EIO; /* XXX */ 1163 } else if (esr_chan != 0) { 1164 error = EIO; /* generic I/O error */ 1165 } else { 1166 error = 0; 1167 } 1168 1169 /* 1170 * Notify the task of completion. May release the lock 1171 * to invoke a callback. 1172 */ 1173 anydone |= sun8i_crypto_chan_done(sc, i, error); 1174 } 1175 mutex_exit(&sc->sc_lock); 1176 1177 /* 1178 * If there are tasks still pending, make sure there's a 1179 * timeout scheduled for them. If the callout is already 1180 * pending, it will take another pass through here to time some 1181 * things out and schedule a new timeout. 1182 */ 1183 if (schedtimeout && !callout_pending(&sc->sc_timeout)) 1184 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 1185 1186 return anydone; 1187 } 1188 1189 /* 1190 * sun8i_crypto_chan_done(sc, i, error) 1191 * 1192 * Notify the callback for the task on channel i, if there is one, 1193 * of the specified error, or 0 for success. 1194 */ 1195 static bool 1196 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1197 { 1198 struct sun8i_crypto_task *task; 1199 uint32_t nbytes; 1200 uint32_t icr; 1201 1202 KASSERT(mutex_owned(&sc->sc_lock)); 1203 1204 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 1205 1206 /* Claim the task if there is one; bail if not. */ 1207 if ((task = sc->sc_chan[i].cc_task) == NULL) { 1208 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1209 i, error); 1210 /* We did not clear a channel. */ 1211 return false; 1212 } 1213 sc->sc_chan[i].cc_task = NULL; 1214 1215 /* Disable interrupts on this channel. */ 1216 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1217 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1218 SUN8I_CRYPTO_ICR_INTR_EN); 1219 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1220 1221 /* 1222 * Finished sending the descriptor to the device by DMA. 1223 * Matches PREWRITE in sun8i_crypto_task_submit. 1224 */ 1225 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1226 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1227 1228 /* 1229 * Finished with all the other bits of DMA too. Matches 1230 * sun8i_crypto_task_load. 1231 */ 1232 nbytes = task->ct_nbytes; 1233 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 1234 BUS_DMASYNC_POSTREAD); 1235 if (task->ct_flags & TASK_SRC) 1236 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 1237 BUS_DMASYNC_POSTWRITE); 1238 if (task->ct_flags & TASK_CTR) 1239 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 1240 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 1241 if (task->ct_flags & TASK_IV) 1242 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 1243 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1244 if (task->ct_flags & TASK_KEY) 1245 /* XXX Can we zero the bounce buffer if there is one? */ 1246 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 1247 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1248 1249 /* Temporarily release the lock to invoke the callback. */ 1250 mutex_exit(&sc->sc_lock); 1251 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1252 (*task->ct_callback)(sc, task, task->ct_cookie, error); 1253 mutex_enter(&sc->sc_lock); 1254 1255 /* We cleared a channel. */ 1256 return true; 1257 } 1258 1259 /* 1260 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 1261 * 1262 * Allocate a single-segment DMA-safe buffer and map it into KVA. 1263 * May fail if dmaflags is BUS_DMA_NOWAIT. 1264 */ 1265 static int 1266 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 1267 struct sun8i_crypto_buf *buf, int dmaflags) 1268 { 1269 int error; 1270 1271 /* Allocate a DMA-safe buffer. */ 1272 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 1273 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1274 if (error) 1275 goto fail0; 1276 1277 /* Map the buffer into kernel virtual address space. */ 1278 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 1279 size, &buf->cb_kva, dmaflags); 1280 if (error) 1281 goto fail1; 1282 1283 /* Success! */ 1284 return 0; 1285 1286 fail2: __unused 1287 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1288 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1289 fail0: return error; 1290 } 1291 1292 /* 1293 * sun8i_crypto_freebuf(sc, buf) 1294 * 1295 * Unmap buf and free it. 1296 */ 1297 static void 1298 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1299 struct sun8i_crypto_buf *buf) 1300 { 1301 1302 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1303 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1304 } 1305 1306 /* 1307 * sun8i_crypto_rng_attach(sc) 1308 * 1309 * Attach an rndsource for the crypto engine's TRNG. 1310 */ 1311 static void 1312 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1313 { 1314 device_t self = sc->sc_dev; 1315 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1316 struct sun8i_crypto_task *task; 1317 unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT); 1318 uint32_t done, esr; 1319 int error; 1320 1321 /* Preallocate a buffer to reuse. */ 1322 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 1323 BUS_DMA_WAITOK); 1324 if (error) { 1325 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 1326 error); 1327 goto fail0; 1328 } 1329 1330 /* Create a task to reuse. */ 1331 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 1332 rng, PR_WAITOK); 1333 if (rng->cr_task == NULL) { 1334 aprint_error_dev(self, "failed to allocate RNG task\n"); 1335 error = ENOMEM; 1336 goto fail1; 1337 } 1338 1339 /* Preload the destination map. */ 1340 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1341 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1342 if (error) { 1343 aprint_error_dev(self, "failed to load RNG buffer: %d\n", 1344 error); 1345 goto fail2; 1346 } 1347 1348 /* 1349 * Attach the rndsource. This will trigger an initial call to 1350 * it since we have RND_FLAG_HASCB. 1351 */ 1352 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1353 rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1354 RND_TYPE_RNG, 1355 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1356 1357 /* 1358 * Poll for the first call to the RNG to complete. If not done 1359 * after the timeout, force a timeout. 1360 */ 1361 for (; rng->cr_pending && timo --> 0; DELAY(1000)) { 1362 if (sun8i_crypto_poll(sc, &done, &esr)) 1363 (void)sun8i_crypto_done(sc, done, esr, getticks()); 1364 } 1365 if (rng->cr_pending) { 1366 (void)sun8i_crypto_done(sc, 0, 0, 1367 (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT); 1368 KASSERT(!rng->cr_pending); 1369 } 1370 1371 return; 1372 1373 fail3: __unused 1374 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1375 fail2: sun8i_crypto_task_put(sc, task); 1376 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 1377 fail0: return; 1378 } 1379 1380 /* 1381 * sun8i_crypto_rng_get(nbytes, cookie) 1382 * 1383 * On-demand rndsource callback: try to gather nbytes of entropy 1384 * and enter them into the pool ASAP. 1385 */ 1386 static void 1387 sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1388 { 1389 struct sun8i_crypto_softc *sc = cookie; 1390 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1391 struct sun8i_crypto_task *task = rng->cr_task; 1392 bool pending; 1393 int error; 1394 1395 /* 1396 * Test and set the RNG-pending flag. If it's already in 1397 * progress, nothing to do here. 1398 */ 1399 mutex_enter(&sc->sc_lock); 1400 pending = rng->cr_pending; 1401 rng->cr_pending = true; 1402 mutex_exit(&sc->sc_lock); 1403 if (pending) 1404 return; 1405 1406 /* Load the task descriptor. */ 1407 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1408 if (error) 1409 goto fail; 1410 1411 /* Submit! */ 1412 error = sun8i_crypto_submit(sc, task); 1413 if (error) 1414 goto fail; 1415 1416 /* All done! */ 1417 return; 1418 1419 fail: mutex_enter(&sc->sc_lock); 1420 rng->cr_pending = false; 1421 mutex_exit(&sc->sc_lock); 1422 } 1423 1424 static void 1425 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1426 struct sun8i_crypto_task *task, void *cookie, int error) 1427 { 1428 struct sun8i_crypto_rng *rng = cookie; 1429 uint8_t *buf = rng->cr_buf.cb_kva; 1430 uint32_t entropybits; 1431 1432 KASSERT(rng == &sc->sc_rng); 1433 1434 /* If anything went wrong, forget about it. */ 1435 if (error) 1436 goto out; 1437 1438 /* 1439 * This TRNG has quite low entropy at best. But if it fails a 1440 * repeated output test, then assume it's busted. 1441 */ 1442 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1443 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1444 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1445 SUN8I_CRYPTO_RNGBYTES/2)) { 1446 device_printf(sc->sc_dev, "failed repeated output test\n"); 1447 entropybits = 0; 1448 } 1449 1450 /* 1451 * Actually we don't believe in any of the entropy until this 1452 * device has had more scrutiny. 1453 */ 1454 entropybits = 0; 1455 1456 /* Success! Enter and erase the data. */ 1457 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1458 entropybits); 1459 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1460 1461 out: /* Done -- clear the RNG-pending flag. */ 1462 mutex_enter(&sc->sc_lock); 1463 rng->cr_pending = false; 1464 mutex_exit(&sc->sc_lock); 1465 } 1466 1467 /* 1468 * Self-test 1469 */ 1470 1471 static const uint8_t selftest_input[16]; 1472 static const uint8_t selftest_key[16]; 1473 static const uint8_t selftest_output[16] = { 1474 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1475 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1476 }; 1477 1478 static bool 1479 sun8i_crypto_selftest(struct sun8i_crypto_softc *sc) 1480 { 1481 const size_t keybytes = sizeof selftest_key; 1482 const size_t nbytes = sizeof selftest_input; 1483 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 1484 struct sun8i_crypto_task *task; 1485 unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT); 1486 uint32_t done, esr; 1487 int error; 1488 1489 CTASSERT(sizeof selftest_input == sizeof selftest_output); 1490 1491 selftest->cs_pending = true; 1492 1493 /* Allocate an input buffer. */ 1494 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 1495 BUS_DMA_WAITOK); 1496 if (error) 1497 goto fail0; 1498 1499 /* Allocate a key buffer. */ 1500 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 1501 BUS_DMA_WAITOK); 1502 if (error) 1503 goto fail1; 1504 1505 /* Allocate an output buffer. */ 1506 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 1507 BUS_DMA_WAITOK); 1508 if (error) 1509 goto fail2; 1510 1511 /* Allocate a task descriptor. */ 1512 task = selftest->cs_task = sun8i_crypto_task_get(sc, 1513 sun8i_crypto_selftest_done, selftest, PR_WAITOK); 1514 if (selftest->cs_task == NULL) { 1515 error = ENOMEM; 1516 goto fail3; 1517 } 1518 1519 /* Copy the input and key into their buffers. */ 1520 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 1521 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1522 1523 /* Load the key, src, and dst for DMA transfers. */ 1524 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 1525 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1526 if (error) 1527 goto fail4; 1528 task->ct_flags |= TASK_KEY; 1529 1530 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 1531 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1532 if (error) 1533 goto fail5; 1534 task->ct_flags |= TASK_SRC; 1535 1536 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1537 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1538 if (error) 1539 goto fail6; 1540 1541 /* Set up the task descriptor. */ 1542 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 1543 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 1544 if (error) 1545 goto fail7; 1546 1547 /* Submit! */ 1548 error = sun8i_crypto_submit(sc, task); 1549 if (error) 1550 goto fail7; 1551 device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1552 1553 /* 1554 * Poll for completion. If not done after the timeout, force a 1555 * timeout. 1556 */ 1557 for (; selftest->cs_pending && timo --> 0; DELAY(1000)) { 1558 if (sun8i_crypto_poll(sc, &done, &esr)) 1559 (void)sun8i_crypto_done(sc, done, esr, getticks()); 1560 } 1561 if (selftest->cs_pending) { 1562 (void)sun8i_crypto_done(sc, 0, 0, 1563 (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT); 1564 KASSERT(!selftest->cs_pending); 1565 } 1566 1567 return selftest->cs_passed; 1568 1569 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1570 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1571 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1572 fail4: sun8i_crypto_task_put(sc, task); 1573 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1574 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1575 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1576 fail0: aprint_error_dev(sc->sc_dev, "failed to run self-test, error=%d\n", 1577 error); 1578 selftest->cs_pending = false; 1579 return false; 1580 } 1581 1582 static bool 1583 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1584 size_t n, const void *expected, const void *actual) 1585 { 1586 const uint8_t *e = expected; 1587 const uint8_t *a = actual; 1588 size_t i; 1589 1590 if (memcmp(e, a, n) == 0) 1591 return true; 1592 1593 device_printf(sc->sc_dev, "self-test: %s\n", title); 1594 printf("expected: "); 1595 for (i = 0; i < n; i++) 1596 printf("%02hhx", e[i]); 1597 printf("\n"); 1598 printf("actual: "); 1599 for (i = 0; i < n; i++) 1600 printf("%02hhx", a[i]); 1601 printf("\n"); 1602 return false; 1603 } 1604 1605 static void 1606 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1607 struct sun8i_crypto_task *task, void *cookie, int error) 1608 { 1609 const size_t keybytes = sizeof selftest_key; 1610 const size_t nbytes = sizeof selftest_input; 1611 struct sun8i_crypto_selftest *selftest = cookie; 1612 bool ok = true; 1613 1614 KASSERT(selftest == &sc->sc_selftest); 1615 1616 /* If anything went wrong, fail now. */ 1617 if (error) { 1618 device_printf(sc->sc_dev, "self-test error=%d\n", error); 1619 goto out; 1620 } 1621 1622 /* 1623 * Verify the input and key weren't clobbered, and verify the 1624 * output matches what we expect. 1625 */ 1626 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 1627 selftest_input, selftest->cs_in.cb_kva); 1628 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 1629 selftest_key, selftest->cs_key.cb_kva); 1630 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 1631 selftest_output, selftest->cs_out.cb_kva); 1632 1633 selftest->cs_passed = ok; 1634 if (ok) 1635 device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1636 1637 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1638 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1639 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1640 sun8i_crypto_task_put(sc, task); 1641 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1642 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1643 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1644 selftest->cs_pending = false; 1645 } 1646 1647 /* 1648 * Sysctl for testing 1649 */ 1650 1651 struct sun8i_crypto_userreq { 1652 kmutex_t cu_lock; 1653 kcondvar_t cu_cv; 1654 size_t cu_size; 1655 struct sun8i_crypto_buf cu_buf; 1656 struct sun8i_crypto_task *cu_task; 1657 int cu_error; 1658 bool cu_done; 1659 bool cu_cancel; 1660 }; 1661 1662 static void 1663 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1664 { 1665 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1666 int error; 1667 1668 /* hw.sun8icryptoN (node) */ 1669 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1670 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1671 SYSCTL_DESCR("sun8i crypto engine knobs"), 1672 NULL, 0, NULL, 0, 1673 CTL_HW, CTL_CREATE, CTL_EOL); 1674 if (error) { 1675 aprint_error_dev(sc->sc_dev, 1676 "failed to set up sysctl hw.%s: %d\n", 1677 device_xname(sc->sc_dev), error); 1678 return; 1679 } 1680 1681 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1682 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1683 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1684 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1685 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1686 if (error) { 1687 aprint_error_dev(sc->sc_dev, 1688 "failed to set up sysctl hw.%s.rng: %d\n", 1689 device_xname(sc->sc_dev), error); 1690 return; 1691 } 1692 } 1693 1694 static int 1695 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1696 { 1697 struct sysctlnode node = *rnode; 1698 struct sun8i_crypto_softc *sc = node.sysctl_data; 1699 struct sun8i_crypto_userreq *req; 1700 struct sun8i_crypto_task *task; 1701 size_t size; 1702 int error; 1703 1704 /* If oldp == NULL, the caller wants to learn the size. */ 1705 if (oldp == NULL) { 1706 *oldlenp = 4096; 1707 return 0; 1708 } 1709 1710 /* Truncate to 4096 bytes. */ 1711 size = MIN(4096, *oldlenp); 1712 if (size == 0) 1713 return 0; /* nothing to do */ 1714 1715 /* Allocate a request context. */ 1716 req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1717 if (req == NULL) 1718 return ENOMEM; 1719 1720 /* Initialize the request context. */ 1721 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1722 cv_init(&req->cu_cv, "sun8isy"); 1723 req->cu_size = size; 1724 req->cu_error = EIO; 1725 req->cu_done = false; 1726 req->cu_cancel = false; 1727 1728 /* Allocate a buffer for the RNG output. */ 1729 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1730 if (error) 1731 goto out0; 1732 1733 /* Allocate a task. */ 1734 task = req->cu_task = sun8i_crypto_task_get(sc, 1735 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 1736 if (task == NULL) { 1737 error = ENOMEM; 1738 goto out1; 1739 } 1740 1741 /* Set the task up for TRNG to our buffer. */ 1742 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1743 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1744 if (error) 1745 goto out2; 1746 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1747 if (error) 1748 goto out3; 1749 1750 /* Submit! */ 1751 error = sun8i_crypto_submit(sc, task); 1752 if (error) { 1753 /* Make sure we don't restart the syscall -- just fail. */ 1754 if (error == ERESTART) 1755 error = EBUSY; 1756 goto out3; 1757 } 1758 1759 /* Wait for the request to complete. */ 1760 mutex_enter(&req->cu_lock); 1761 while (!req->cu_done) { 1762 error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1763 if (error) { 1764 /* 1765 * If we finished while waiting to acquire the 1766 * lock, ignore the error and just return now. 1767 * Otherwise, notify the callback that it has 1768 * to clean up after us. 1769 */ 1770 if (req->cu_done) 1771 error = 0; 1772 else 1773 req->cu_cancel = true; 1774 break; 1775 } 1776 } 1777 mutex_exit(&req->cu_lock); 1778 1779 /* 1780 * Return early on error from cv_wait_sig, which means 1781 * interruption; the callback will clean up instead. 1782 */ 1783 if (error) 1784 return error; 1785 1786 /* Check for error from the device. */ 1787 error = req->cu_error; 1788 if (error) 1789 goto out3; 1790 1791 /* Copy out the data. */ 1792 node.sysctl_data = req->cu_buf.cb_kva; 1793 node.sysctl_size = size; 1794 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1795 1796 /* Clear the buffer. */ 1797 explicit_memset(req->cu_buf.cb_kva, 0, size); 1798 1799 /* Clean up. */ 1800 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1801 out2: sun8i_crypto_task_put(sc, task); 1802 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1803 out0: cv_destroy(&req->cu_cv); 1804 mutex_destroy(&req->cu_lock); 1805 kmem_free(req, sizeof(*req)); 1806 return error; 1807 } 1808 1809 static void 1810 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1811 struct sun8i_crypto_task *task, void *cookie, int error) 1812 { 1813 struct sun8i_crypto_userreq *req = cookie; 1814 bool cancel; 1815 1816 /* 1817 * Notify the waiting thread of the error, and find out whether 1818 * that thread cancelled. 1819 */ 1820 mutex_enter(&req->cu_lock); 1821 cancel = req->cu_cancel; 1822 req->cu_error = error; 1823 req->cu_done = true; 1824 cv_broadcast(&req->cu_cv); 1825 mutex_exit(&req->cu_lock); 1826 1827 /* 1828 * If it wasn't cancelled, we're done -- the main thread will 1829 * clean up after itself. 1830 */ 1831 if (!cancel) 1832 return; 1833 1834 /* Clean up after the main thread cancelled. */ 1835 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1836 sun8i_crypto_task_put(sc, task); 1837 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1838 cv_destroy(&req->cu_cv); 1839 mutex_destroy(&req->cu_lock); 1840 kmem_free(req, sizeof(*req)); 1841 } 1842 1843 /* 1844 * sun8i_crypto_register(sc) 1845 * 1846 * Register opencrypto algorithms supported by the crypto engine. 1847 */ 1848 static void 1849 sun8i_crypto_register(struct sun8i_crypto_softc *sc) 1850 { 1851 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 1852 1853 co->co_driverid = crypto_get_driverid(0); 1854 if (co->co_driverid == (uint32_t)-1) { 1855 aprint_error_dev(sc->sc_dev, 1856 "failed to register crypto driver\n"); 1857 return; 1858 } 1859 1860 sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 1861 sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 1862 #ifdef CRYPTO_AES_ECB 1863 sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 1864 #endif 1865 #ifdef CRYPTO_AES_XTS 1866 sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 1867 #endif 1868 #ifdef CRYPTO_DES_CBC 1869 sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 1870 #endif 1871 #ifdef CRYPTO_DES_ECB 1872 sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 1873 #endif 1874 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 1875 #ifdef CRYPTO_3DES_ECB 1876 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 1877 #endif 1878 1879 sun8i_crypto_register1(sc, CRYPTO_MD5); 1880 sun8i_crypto_register1(sc, CRYPTO_SHA1); 1881 #ifdef CRYPTO_SHA224 1882 sun8i_crypto_register1(sc, CRYPTO_SHA224); 1883 #endif 1884 #ifdef CRYPTO_SHA256 1885 sun8i_crypto_register1(sc, CRYPTO_SHA256); 1886 #endif 1887 1888 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 1889 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 1890 1891 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 1892 } 1893 1894 /* 1895 * sun8i_crypto_register1(sc, alg) 1896 * 1897 * Register support for one algorithm alg using 1898 * sun8i_crypto_newsession/freesession/process. 1899 */ 1900 static void 1901 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 1902 { 1903 1904 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 1905 sun8i_crypto_newsession, 1906 sun8i_crypto_freesession, 1907 sun8i_crypto_process, 1908 sc); 1909 } 1910 1911 /* 1912 * sun8i_crypto_newsession(cookie, sidp, cri) 1913 * 1914 * Called by opencrypto to allocate a new session. We don't keep 1915 * track of sessions, since there are no persistent keys in the 1916 * hardware that we take advantage of, so this only validates the 1917 * crypto operations and returns a dummy session id of 1. 1918 */ 1919 static int 1920 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 1921 { 1922 1923 /* No composition of operations is supported here. */ 1924 if (cri->cri_next) 1925 return EINVAL; 1926 1927 /* 1928 * No variation of rounds is supported here. (XXX Unused and 1929 * unimplemented in opencrypto(9) altogether?) 1930 */ 1931 if (cri->cri_rnd) 1932 return EINVAL; 1933 1934 /* 1935 * Validate per-algorithm key length. 1936 * 1937 * XXX Does opencrypto(9) do this internally? 1938 */ 1939 switch (cri->cri_alg) { 1940 case CRYPTO_MD5: 1941 case CRYPTO_SHA1: 1942 #ifdef CRYPTO_SHA224 1943 case CRYPTO_SHA224: 1944 #endif 1945 #ifdef CRYPTO_SHA256 1946 case CRYPTO_SHA256: 1947 #endif 1948 if (cri->cri_klen) 1949 return EINVAL; 1950 break; 1951 case CRYPTO_AES_CBC: 1952 #ifdef CRYPTO_AES_ECB 1953 case CRYPTO_AES_ECB: 1954 #endif 1955 switch (cri->cri_klen) { 1956 case 128: 1957 case 192: 1958 case 256: 1959 break; 1960 default: 1961 return EINVAL; 1962 } 1963 break; 1964 case CRYPTO_AES_CTR: 1965 /* 1966 * opencrypto `AES-CTR' takes four bytes of the input 1967 * block as the last four bytes of the key, for reasons 1968 * that are not entirely clear. 1969 */ 1970 switch (cri->cri_klen) { 1971 case 128 + 32: 1972 case 192 + 32: 1973 case 256 + 32: 1974 break; 1975 default: 1976 return EINVAL; 1977 } 1978 break; 1979 #ifdef CRYPTO_AES_XTS 1980 case CRYPTO_AES_XTS: 1981 switch (cri->cri_klen) { 1982 case 256: 1983 case 384: 1984 case 512: 1985 break; 1986 default: 1987 return EINVAL; 1988 } 1989 break; 1990 #endif 1991 case CRYPTO_DES_CBC: 1992 #ifdef CRYPTO_DES_ECB 1993 case CRYPTO_DES_ECB: 1994 #endif 1995 switch (cri->cri_klen) { 1996 case 64: 1997 break; 1998 default: 1999 return EINVAL; 2000 } 2001 break; 2002 case CRYPTO_3DES_CBC: 2003 #ifdef CRYPTO_3DES_ECB 2004 case CRYPTO_3DES_ECB: 2005 #endif 2006 switch (cri->cri_klen) { 2007 case 192: 2008 break; 2009 default: 2010 return EINVAL; 2011 } 2012 break; 2013 case CRYPTO_SHA1_HMAC: 2014 /* 2015 * XXX Unclear what the length limit is, but since HMAC 2016 * behaves qualitatively different for a key of at 2017 * least the full block size -- and is generally best 2018 * to use with half the block size -- let's limit it to 2019 * one block. 2020 */ 2021 if (cri->cri_klen % 8) 2022 return EINVAL; 2023 if (cri->cri_klen > 512) 2024 return EINVAL; 2025 break; 2026 case CRYPTO_SHA2_256_HMAC: 2027 if (cri->cri_klen % 8) 2028 return EINVAL; 2029 if (cri->cri_klen > 512) 2030 return EINVAL; 2031 break; 2032 default: 2033 panic("unsupported algorithm %d", cri->cri_alg); 2034 } 2035 2036 KASSERT(cri->cri_klen % 8 == 0); 2037 2038 /* Success! */ 2039 *sidp = 1; 2040 return 0; 2041 } 2042 2043 /* 2044 * sun8i_crypto_freesession(cookie, dsid) 2045 * 2046 * Called by opencrypto to free a session. We don't keep track of 2047 * sessions, since there are no persistent keys in the hardware 2048 * that we take advantage of, so this is a no-op. 2049 * 2050 * Note: dsid is actually a 64-bit quantity containing both the 2051 * driver id in the high half and the session id in the low half. 2052 */ 2053 static void 2054 sun8i_crypto_freesession(void *cookie, uint64_t dsid) 2055 { 2056 2057 KASSERT((dsid & 0xffffffff) == 1); 2058 } 2059 2060 /* 2061 * sun8i_crypto_ivlen(crd) 2062 * 2063 * Return the crypto engine's notion of `IV length', in bytes, for 2064 * an opencrypto operation. 2065 */ 2066 static u_int 2067 sun8i_crypto_ivlen(const struct cryptodesc *crd) 2068 { 2069 2070 switch (crd->crd_alg) { 2071 case CRYPTO_AES_CBC: 2072 return 16; 2073 #ifdef CRYPTO_AES_XTS 2074 case CRYPTO_AES_XTS: 2075 return 16; 2076 #endif 2077 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 2078 return 8; 2079 #ifdef CRYPTO_DES_CBC 2080 case CRYPTO_DES_CBC: 2081 return 8; 2082 #endif 2083 case CRYPTO_3DES_CBC: 2084 return 8; 2085 case CRYPTO_MD5: 2086 return 16; 2087 #ifdef CRYPTO_SHA224 2088 case CRYPTO_SHA224: 2089 return 32; 2090 #endif 2091 #ifdef CRYPTO_SHA256 2092 case CRYPTO_SHA256: 2093 return 32; 2094 #endif 2095 case CRYPTO_SHA1_HMAC: 2096 return 20; 2097 case CRYPTO_SHA2_256_HMAC: 2098 return 32; 2099 default: 2100 return 0; 2101 } 2102 } 2103 2104 /* 2105 * sun8i_crypto_process(cookie, crp, hint) 2106 * 2107 * Main opencrypto processing dispatch. 2108 */ 2109 static int 2110 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 2111 { 2112 struct sun8i_crypto_softc *sc = cookie; 2113 struct sun8i_crypto_task *task; 2114 struct cryptodesc *crd = crp->crp_desc; 2115 unsigned klen, ivlen; 2116 uint32_t tdqc = 0, tdqs = 0; 2117 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 2118 const uint32_t tdqa = 0; 2119 int error; 2120 2121 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 2122 2123 /* Reject compositions -- we do not handle them. */ 2124 if (crd->crd_next != NULL) { 2125 error = EOPNOTSUPP; 2126 goto fail0; 2127 } 2128 2129 /* Reject transfers with nonsense skip. */ 2130 if (crd->crd_skip < 0) { 2131 error = EINVAL; 2132 goto fail0; 2133 } 2134 2135 /* 2136 * Actually just reject any nonzero skip, because it requires 2137 * DMA segment bookkeeping that we don't do yet. 2138 */ 2139 if (crd->crd_skip) { 2140 error = EOPNOTSUPP; 2141 goto fail0; 2142 } 2143 2144 /* Reject large transfers. */ 2145 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 2146 error = EFBIG; 2147 goto fail0; 2148 } 2149 2150 /* Reject nonsense, unaligned, or mismatched lengths. */ 2151 if (crd->crd_len < 0 || 2152 crd->crd_len % 4 || 2153 crd->crd_len != crp->crp_ilen) { 2154 error = EINVAL; 2155 goto fail0; 2156 } 2157 2158 /* Reject mismatched buffer lengths. */ 2159 /* XXX Handle crd_skip. */ 2160 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2161 struct mbuf *m = crp->crp_buf; 2162 uint32_t nbytes = 0; 2163 while (m != NULL) { 2164 KASSERT(m->m_len >= 0); 2165 if (m->m_len > crd->crd_len || 2166 nbytes > crd->crd_len - m->m_len) { 2167 error = EINVAL; 2168 goto fail0; 2169 } 2170 nbytes += m->m_len; 2171 m = m->m_next; 2172 } 2173 if (nbytes != crd->crd_len) { 2174 error = EINVAL; 2175 goto fail0; 2176 } 2177 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2178 struct uio *uio = crp->crp_buf; 2179 if (uio->uio_resid != crd->crd_len) { 2180 error = EINVAL; 2181 goto fail0; 2182 } 2183 } 2184 2185 /* Get a task, or fail with ERESTART if we can't. */ 2186 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 2187 PR_NOWAIT); 2188 if (task == NULL) { 2189 /* 2190 * Don't invoke crypto_done -- we are asking the 2191 * opencrypto(9) machinery to queue the request and get 2192 * back to us. 2193 */ 2194 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 2195 return ERESTART; 2196 } 2197 2198 /* Load key in, if relevant. */ 2199 klen = crd->crd_klen; 2200 if (klen) { 2201 if (crd->crd_alg == CRYPTO_AES_CTR) 2202 /* AES-CTR is special -- see IV processing below. */ 2203 klen -= 32; 2204 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 2205 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 2206 if (error) 2207 goto fail1; 2208 task->ct_flags |= TASK_KEY; 2209 } 2210 2211 /* Handle the IV, if relevant. */ 2212 ivlen = sun8i_crypto_ivlen(crd); 2213 if (ivlen) { 2214 void *iv; 2215 2216 /* 2217 * If there's an explicit IV, use it; otherwise 2218 * randomly generate one. 2219 */ 2220 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2221 iv = crd->crd_iv; 2222 } else { 2223 cprng_fast(task->ct_iv, ivlen); 2224 iv = task->ct_iv; 2225 } 2226 2227 /* 2228 * If the IV is not already present in the user's 2229 * buffer, copy it over. 2230 */ 2231 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 2232 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2233 m_copyback(crp->crp_buf, crd->crd_inject, 2234 ivlen, iv); 2235 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2236 cuio_copyback(crp->crp_buf, crd->crd_inject, 2237 ivlen, iv); 2238 } else { 2239 panic("invalid buffer type %x", 2240 crp->crp_flags); 2241 } 2242 } 2243 2244 /* 2245 * opencrypto's idea of `AES-CTR' is special. 2246 * 2247 * - The low 4 bytes of the input block are drawn from 2248 * an extra 4 bytes at the end of the key. 2249 * 2250 * - The next 8 bytes of the input block are drawn from 2251 * the opencrypto iv. 2252 * 2253 * - The high 4 bytes are the big-endian block counter, 2254 * which starts at 1 because why not. 2255 */ 2256 if (crd->crd_alg == CRYPTO_AES_CTR) { 2257 uint8_t block[16]; 2258 uint32_t blkno = 1; 2259 2260 /* Format the initial input block. */ 2261 memcpy(block, crd->crd_key + klen/8, 4); 2262 memcpy(block + 4, iv, 8); 2263 be32enc(block + 12, blkno); 2264 2265 /* Copy it into the DMA buffer. */ 2266 memcpy(task->ct_iv, block, 16); 2267 iv = task->ct_iv; 2268 ivlen = 16; 2269 } 2270 2271 /* Load the IV. */ 2272 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 2273 NULL, BUS_DMA_NOWAIT); 2274 if (error) 2275 goto fail1; 2276 task->ct_flags |= TASK_IV; 2277 } 2278 2279 /* Load the src and dst. */ 2280 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2281 struct mbuf *m = crp->crp_buf; 2282 2283 /* XXX Handle crd_skip. */ 2284 KASSERT(crd->crd_skip == 0); 2285 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 2286 BUS_DMA_NOWAIT); 2287 if (error) 2288 goto fail1; 2289 task->ct_flags |= TASK_SRC; 2290 2291 /* XXX Handle crd_skip. */ 2292 KASSERT(crd->crd_skip == 0); 2293 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 2294 BUS_DMA_NOWAIT); 2295 if (error) 2296 goto fail1; 2297 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2298 struct uio *uio = crp->crp_buf; 2299 2300 /* XXX Handle crd_skip. */ 2301 KASSERT(crd->crd_skip == 0); 2302 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 2303 BUS_DMA_NOWAIT); 2304 if (error) 2305 goto fail1; 2306 task->ct_flags |= TASK_SRC; 2307 2308 /* XXX Handle crd_skip. */ 2309 KASSERT(crd->crd_skip == 0); 2310 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 2311 BUS_DMA_NOWAIT); 2312 if (error) 2313 goto fail1; 2314 } else { 2315 panic("invalid buffer type %x", crp->crp_flags); 2316 } 2317 2318 /* Set the encryption direction. */ 2319 if (crd->crd_flags & CRD_F_ENCRYPT) 2320 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 2321 else 2322 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 2323 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 2324 2325 /* Set the method. */ 2326 switch (crd->crd_alg) { 2327 case CRYPTO_AES_CBC: 2328 case CRYPTO_AES_CTR: 2329 #ifdef CRYPTO_AES_ECB 2330 case CRYPTO_AES_ECB: 2331 #endif 2332 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2333 break; 2334 #ifdef CRYPTO_AES_XTS 2335 case CRYPTO_AES_XTS: 2336 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2337 break; 2338 #endif 2339 case CRYPTO_DES_CBC: 2340 #ifdef CRYPTO_DES_ECB 2341 case CRYPTO_DES_ECB: 2342 #endif 2343 method = SUN8I_CRYPTO_TDQC_METHOD_DES; 2344 break; 2345 case CRYPTO_3DES_CBC: 2346 #ifdef CRYPTO_3DES_ECB 2347 case CRYPTO_3DES_ECB: 2348 #endif 2349 method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 2350 break; 2351 case CRYPTO_MD5: 2352 method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 2353 break; 2354 case CRYPTO_SHA1: 2355 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 2356 break; 2357 #ifdef CRYPTO_SHA224 2358 case CRYPTO_SHA224: 2359 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 2360 break; 2361 #endif 2362 #ifdef CRYPTO_SHA256 2363 case CRYPTO_SHA256: 2364 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 2365 break; 2366 #endif 2367 case CRYPTO_SHA1_HMAC: 2368 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 2369 break; 2370 case CRYPTO_SHA2_256_HMAC: 2371 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 2372 break; 2373 default: 2374 panic("unknown algorithm %d", crd->crd_alg); 2375 } 2376 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 2377 2378 /* Set the key selector. No idea how to use the internal keys. */ 2379 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 2380 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 2381 2382 /* XXX Deal with AES_CTS_Last_Block_Flag. */ 2383 2384 /* Set the mode. */ 2385 switch (crd->crd_alg) { 2386 #ifdef CRYPTO_AES_ECB 2387 case CRYPTO_AES_ECB: 2388 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2389 break; 2390 #endif 2391 #ifdef CRYPTO_DES_ECB 2392 case CRYPTO_DES_ECB: 2393 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2394 break; 2395 #endif 2396 #ifdef CRYPTO_3DES_ECB 2397 case CRYPTO_3DES_ECB: 2398 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2399 break; 2400 #endif 2401 case CRYPTO_AES_CBC: 2402 case CRYPTO_DES_CBC: 2403 case CRYPTO_3DES_CBC: 2404 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 2405 break; 2406 case CRYPTO_AES_CTR: 2407 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 2408 break; 2409 #ifdef CRYPTO_AES_XTS 2410 case CRYPTO_AES_XTS: 2411 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 2412 break; 2413 #endif 2414 default: 2415 panic("unknown algorithm %d", crd->crd_alg); 2416 } 2417 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 2418 2419 /* Set the CTR width. */ 2420 switch (crd->crd_alg) { 2421 case CRYPTO_AES_CTR: 2422 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 2423 break; 2424 } 2425 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 2426 2427 /* Set the AES key size. */ 2428 switch (crd->crd_alg) { 2429 case CRYPTO_AES_CBC: 2430 #ifdef CRYPTO_AES_ECB 2431 case CRYPTO_AES_ECB: 2432 #endif 2433 switch (crd->crd_klen) { 2434 case 128: 2435 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2436 break; 2437 case 192: 2438 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2439 break; 2440 case 256: 2441 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2442 break; 2443 default: 2444 panic("invalid AES key size in bits: %u", 2445 crd->crd_klen); 2446 } 2447 break; 2448 case CRYPTO_AES_CTR: 2449 switch (crd->crd_klen) { 2450 case 128 + 32: 2451 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2452 break; 2453 case 192 + 32: 2454 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2455 break; 2456 case 256 + 32: 2457 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2458 break; 2459 default: 2460 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 2461 crd->crd_klen); 2462 } 2463 break; 2464 #ifdef CRYPTO_AES_XTS 2465 case CRYPTO_AES_XTS: 2466 switch (crd->crd_klen) { 2467 case 256: 2468 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2469 break; 2470 case 384: 2471 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2472 break; 2473 case 512: 2474 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2475 break; 2476 default: 2477 panic("invalid AES-XTS key size in bits: %u", 2478 crd->crd_klen); 2479 } 2480 break; 2481 #endif 2482 } 2483 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 2484 2485 /* Set up the task descriptor. */ 2486 error = sun8i_crypto_task_load(sc, task, crd->crd_len, 2487 tdqc, tdqs, tdqa); 2488 if (error) 2489 goto fail2; 2490 2491 /* Submit! */ 2492 error = sun8i_crypto_submit(sc, task); 2493 if (error) 2494 goto fail2; 2495 2496 /* Success! */ 2497 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 2498 return 0; 2499 2500 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2501 fail1: if (task->ct_flags & TASK_SRC) 2502 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2503 if (task->ct_flags & TASK_CTR) 2504 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2505 if (task->ct_flags & TASK_IV) 2506 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2507 if (task->ct_flags & TASK_KEY) 2508 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2509 sun8i_crypto_task_put(sc, task); 2510 fail0: KASSERT(error); 2511 KASSERT(error != ERESTART); 2512 crp->crp_etype = error; 2513 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2514 crypto_done(crp); 2515 return 0; 2516 } 2517 2518 /* 2519 * sun8i_crypto_callback(sc, task, cookie, error) 2520 * 2521 * Completion callback for a task submitted via opencrypto. 2522 * Release the task and pass the error on to opencrypto with 2523 * crypto_done. 2524 */ 2525 static void 2526 sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 2527 struct sun8i_crypto_task *task, void *cookie, int error) 2528 { 2529 struct cryptop *crp = cookie; 2530 struct cryptodesc *crd __diagused = crp->crp_desc; 2531 2532 KASSERT(error != ERESTART); 2533 KASSERT(crd != NULL); 2534 KASSERT(crd->crd_next == NULL); 2535 2536 /* Return the number of bytes processed. */ 2537 crp->crp_olen = error ? 0 : crp->crp_ilen; 2538 2539 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2540 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2541 if (task->ct_flags & TASK_CTR) 2542 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2543 if (task->ct_flags & TASK_IV) 2544 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2545 if (task->ct_flags & TASK_KEY) 2546 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2547 sun8i_crypto_task_put(sc, task); 2548 KASSERT(error != ERESTART); 2549 crp->crp_etype = error; 2550 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2551 crypto_done(crp); 2552 } 2553
Indexes created Sat Sep 20 22:09:52 GMT 2025