1 /* $NetBSD: sun8i_crypto.c,v 1.33 2026/10/27 15:32:24 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * sun8i_crypto -- Allwinner Crypto Engine driver 34 * 35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37 * the moment, but in principle it could be wired up with opencrypto(9) 38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41 * 42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43 * https://web.archive.org/web/20260825112212/https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 44 */ 45 46 #include <sys/cdefs.h> 47 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.33 2026/10/27 15:32:24 riastradh Exp $"); 48 49 #include <sys/types.h> 50 #include <sys/param.h> 51 #include <sys/atomic.h> 52 #include <sys/bus.h> 53 #include <sys/callout.h> 54 #include <sys/conf.h> 55 #include <sys/cprng.h> 56 #include <sys/device.h> 57 #include <sys/kernel.h> 58 #include <sys/kmem.h> 59 #include <sys/mbuf.h> 60 #include <sys/mutex.h> 61 #include <sys/rndsource.h> 62 #include <sys/sdt.h> 63 #include <sys/sysctl.h> 64 #include <sys/workqueue.h> 65 66 #include <dev/fdt/fdtvar.h> 67 68 #include <opencrypto/cryptodev.h> 69 70 #include <arm/sunxi/sun8i_crypto.h> 71 72 #define SUN8I_CRYPTO_TIMEOUT hz 73 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 74 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 75 76 struct sun8i_crypto_config { 77 u_int mod_rate; /* module clock rate */ 78 }; 79 80 /* 81 * The module clock is set to 50 MHz on H3, 300 MHz otherwise. 82 * From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c: 83 * Module clock is lower on H3 than other SoC due to some DMA 84 * timeout occurring with high value. 85 */ 86 static const struct sun8i_crypto_config sun50i_a64_crypto_config = { 87 .mod_rate = 300*1000*1000, 88 }; 89 90 static const struct sun8i_crypto_config sun50i_h5_crypto_config = { 91 .mod_rate = 300*1000*1000, 92 }; 93 94 static const struct sun8i_crypto_config sun8i_h3_crypto_config = { 95 .mod_rate = 50*1000*1000, 96 }; 97 98 struct sun8i_crypto_task; 99 100 struct sun8i_crypto_buf { 101 bus_dma_segment_t cb_seg[1]; 102 int cb_nsegs; 103 void *cb_kva; 104 }; 105 106 struct sun8i_crypto_softc { 107 device_t sc_dev; 108 bus_space_tag_t sc_bst; 109 bus_space_handle_t sc_bsh; 110 bus_dma_tag_t sc_dmat; 111 struct pool_cache *sc_taskpool; 112 113 const struct sun8i_crypto_config *sc_cfg; 114 115 struct workqueue *sc_wq; 116 void *sc_ih; 117 bool sc_polling; 118 119 kmutex_t sc_lock; 120 struct sun8i_crypto_chan { 121 struct sun8i_crypto_task *cc_task; 122 unsigned cc_starttime; 123 } sc_chan[SUN8I_CRYPTO_NCHAN]; 124 struct callout sc_timeout; 125 126 kmutex_t sc_intr_lock; 127 uint32_t sc_done; 128 uint32_t sc_esr; 129 struct work sc_work; 130 bool sc_work_pending; 131 132 struct sun8i_crypto_rng { 133 struct sun8i_crypto_buf cr_buf; 134 struct sun8i_crypto_task *cr_task; 135 struct krndsource cr_rndsource; 136 bool cr_pending; 137 } sc_rng; 138 struct sun8i_crypto_selftest { 139 struct sun8i_crypto_buf cs_in; 140 struct sun8i_crypto_buf cs_key; 141 struct sun8i_crypto_buf cs_out; 142 struct sun8i_crypto_task *cs_task; 143 bool cs_pending; 144 bool cs_passed; 145 } sc_selftest; 146 struct sun8i_crypto_sysctl { 147 struct sysctllog *cy_log; 148 const struct sysctlnode *cy_root_node; 149 const struct sysctlnode *cy_trng_node; 150 } sc_sysctl; 151 struct sun8i_crypto_opencrypto { 152 uint32_t co_driverid; 153 } sc_opencrypto; 154 }; 155 156 struct sun8i_crypto_task { 157 struct sun8i_crypto_buf ct_descbuf; 158 struct sun8i_crypto_taskdesc *ct_desc; 159 struct sun8i_crypto_buf ct_ivbuf; 160 void *ct_iv; 161 struct sun8i_crypto_buf ct_ctrbuf; 162 void *ct_ctr; 163 bus_dmamap_t ct_descmap; 164 bus_dmamap_t ct_keymap; 165 bus_dmamap_t ct_ivmap; /* IV input */ 166 bus_dmamap_t ct_ctrmap; /* updated IV output */ 167 bus_dmamap_t ct_srcmap; 168 bus_dmamap_t ct_dstmap; 169 uint32_t ct_nbytes; 170 int ct_flags; 171 #define TASK_KEY __BIT(0) 172 #define TASK_IV __BIT(1) 173 #define TASK_CTR __BIT(2) 174 #define TASK_SRC __BIT(3) 175 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 176 void (*ct_callback)(struct sun8i_crypto_softc *, 177 struct sun8i_crypto_task *, void *, int); 178 void *ct_cookie; 179 }; 180 181 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 182 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 183 184 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 185 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 186 187 /* 188 * Forward declarations 189 */ 190 191 static int sun8i_crypto_match(device_t, cfdata_t, void *); 192 static void sun8i_crypto_attach(device_t, device_t, void *); 193 194 static int sun8i_crypto_task_ctor(void *, void *, int); 195 static void sun8i_crypto_task_dtor(void *, void *); 196 static struct sun8i_crypto_task * 197 sun8i_crypto_task_get(struct sun8i_crypto_softc *, 198 void (*)(struct sun8i_crypto_softc *, 199 struct sun8i_crypto_task *, void *, int), 200 void *, int); 201 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 202 struct sun8i_crypto_task *); 203 204 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 205 struct sun8i_crypto_task *, uint32_t, 206 uint32_t, uint32_t, uint32_t); 207 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 208 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 209 210 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 211 struct sun8i_crypto_task *, uint32_t); 212 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 213 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 214 215 static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 216 struct sun8i_crypto_task *); 217 218 static void sun8i_crypto_timeout(void *); 219 static int sun8i_crypto_intr(void *); 220 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 221 static void sun8i_crypto_worker(struct work *, void *); 222 223 static bool sun8i_crypto_poll(struct sun8i_crypto_softc *, uint32_t *, 224 uint32_t *); 225 static bool sun8i_crypto_done(struct sun8i_crypto_softc *, uint32_t, 226 uint32_t, unsigned); 227 static bool sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 228 int); 229 230 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 231 struct sun8i_crypto_buf *, int); 232 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 233 struct sun8i_crypto_buf *); 234 235 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 236 static void sun8i_crypto_rng_get(size_t, void *); 237 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 238 struct sun8i_crypto_task *, void *, int); 239 240 static bool sun8i_crypto_selftest(struct sun8i_crypto_softc *); 241 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 242 struct sun8i_crypto_task *, void *, int); 243 244 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 245 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 246 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 247 struct sun8i_crypto_task *, void *, int); 248 249 static void sun8i_crypto_register(struct sun8i_crypto_softc *); 250 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 251 static int sun8i_crypto_newsession(void *, uint32_t *, 252 struct cryptoini *); 253 static void sun8i_crypto_freesession(void *, uint64_t); 254 static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 255 static int sun8i_crypto_process(void *, struct cryptop *, int); 256 static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 257 struct sun8i_crypto_task *, void *, int); 258 259 /* 260 * Probes 261 */ 262 263 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 264 "bus_size_t"/*reg*/, 265 "uint32_t"/*value*/); 266 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 267 "bus_size_t"/*reg*/, 268 "uint32_t"/*write*/); 269 270 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 271 "struct sun8i_crypto_task *"/*task*/); 272 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 273 "int"/*error*/); 274 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 275 "struct sun8i_crypto_task *"/*task*/); 276 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 277 "struct sun8i_crypto_task *"/*task*/); 278 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 279 "struct sun8i_crypto_task *"/*task*/); 280 281 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 282 "struct sun8i_crypto_task *"/*task*/, 283 "uint32_t"/*tdqc*/, 284 "uint32_t"/*tdqs*/, 285 "uint32_t"/*tdqa*/, 286 "struct sun8i_crypto_taskdesc *"/*desc*/, 287 "int"/*error*/); 288 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 289 "struct sun8i_crypto_task *"/*task*/, 290 "bus_addr_t"/*ds_addr*/, 291 "bus_size_t"/*ds_len*/); 292 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 293 "struct sun8i_crypto_task *"/*task*/, 294 "int"/*error*/); 295 296 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 297 "struct sun8i_crypto_softc *"/*sc*/, 298 "struct sun8i_crypto_task *"/*task*/, 299 "int"/*error*/); 300 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 301 "struct sun8i_crypto_softc *"/*sc*/, 302 "struct sun8i_crypto_task *"/*task*/, 303 "unsigned"/*chan*/); 304 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 305 "struct sun8i_crypto_softc *"/*sc*/, 306 "uint32_t"/*isr*/, 307 "uint32_t"/*esr*/); 308 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 309 "struct sun8i_crypto_softc *"/*sc*/, 310 "unsigned"/*chan*/, 311 "int"/*error*/); 312 313 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 314 "struct sun8i_crypto_softc *"/*sc*/, 315 "struct cryptop *"/*crp*/, 316 "int"/*hint*/); 317 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 318 "struct sun8i_crypto_softc *"/*sc*/, 319 "struct cryptop *"/*crp*/, 320 "int"/*hint*/); 321 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 322 "struct sun8i_crypto_softc *"/*sc*/, 323 "struct cryptop *"/*crp*/, 324 "int"/*hint*/, 325 "struct sun8i_crypto_task *"/*task*/); 326 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 327 "struct sun8i_crypto_softc *"/*sc*/, 328 "struct cryptop *"/*crp*/, 329 "int"/*error*/); 330 331 /* 332 * Register access 333 */ 334 335 static uint32_t 336 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 337 { 338 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 339 340 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 341 return v; 342 } 343 344 static void 345 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 346 { 347 348 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 349 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 350 } 351 352 /* 353 * Autoconf goo 354 */ 355 356 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 357 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 358 359 static const struct device_compatible_entry compat_data[] = { 360 { .compat = "allwinner,sun50i-a64-crypto", 361 .data = &sun50i_a64_crypto_config }, 362 { .compat = "allwinner,sun50i-h5-crypto", 363 .data = &sun50i_h5_crypto_config }, 364 { .compat = "allwinner,sun8i-h3-crypto", 365 .data = &sun8i_h3_crypto_config }, 366 DEVICE_COMPAT_EOL 367 }; 368 369 static int 370 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 371 { 372 const struct fdt_attach_args *const faa = aux; 373 374 return of_compatible_match(faa->faa_phandle, compat_data); 375 } 376 377 static void 378 sun8i_crypto_attach(device_t parent, device_t self, void *aux) 379 { 380 struct sun8i_crypto_softc *const sc = device_private(self); 381 const struct fdt_attach_args *const faa = aux; 382 bus_addr_t addr; 383 bus_size_t size; 384 const int phandle = faa->faa_phandle; 385 char intrstr[128]; 386 struct clk *clk; 387 struct fdtbus_reset *rst; 388 u_int mod_rate; 389 390 sc->sc_dev = self; 391 sc->sc_dmat = faa->faa_dmat; 392 sc->sc_bst = faa->faa_bst; 393 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 394 0, 0, 0, "sun8icry", NULL, IPL_SOFTSERIAL, 395 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 396 sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data; 397 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); 398 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_VM); 399 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 400 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 401 if (workqueue_create(&sc->sc_wq, device_xname(self), 402 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 403 aprint_error(": couldn't create workqueue\n"); 404 return; 405 } 406 407 /* 408 * Prime the pool with enough tasks that each channel can be 409 * busy with a task as we prepare another task for when it's 410 * done. 411 */ 412 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 413 414 /* Get and map device registers. */ 415 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 416 aprint_error(": couldn't get registers\n"); 417 return; 418 } 419 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 420 aprint_error(": couldn't map registers\n"); 421 return; 422 } 423 424 /* Get an interrupt handle. */ 425 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 426 aprint_error(": failed to decode interrupt\n"); 427 return; 428 } 429 430 /* Enable the bus clock. */ 431 if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 432 aprint_error(": couldn't enable bus clock\n"); 433 return; 434 } 435 436 /* Get the module clock and set it. */ 437 mod_rate = sc->sc_cfg->mod_rate; 438 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 439 if (clk_enable(clk) != 0) { 440 aprint_error(": couldn't enable CE clock\n"); 441 return; 442 } 443 if (clk_set_rate(clk, mod_rate) != 0) { 444 aprint_error(": couldn't set CE clock to %d MHz\n", 445 mod_rate / (1000 * 1000)); 446 return; 447 } 448 } 449 450 /* Get a reset handle if we need and try to deassert it. */ 451 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 452 if (fdtbus_reset_deassert(rst) != 0) { 453 aprint_error(": couldn't de-assert reset\n"); 454 return; 455 } 456 } 457 458 aprint_naive("\n"); 459 aprint_normal(": Crypto Engine\n"); 460 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 461 462 /* 463 * Disable and clear interrupts. Start in polling mode for 464 * synchronous self-tests and the first RNG draw. 465 */ 466 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 467 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 468 sc->sc_polling = true; 469 470 /* Establish an interrupt handler. */ 471 sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM, 472 FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self)); 473 if (sc->sc_ih == NULL) { 474 aprint_error_dev(self, "failed to establish interrupt on %s\n", 475 intrstr); 476 return; 477 } 478 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 479 480 /* Perform self-tests. If they fail, stop here. */ 481 if (!sun8i_crypto_selftest(sc)) 482 return; 483 484 /* 485 * Set up the RNG. This will try to synchronously draw the 486 * first sample by polling, so do this before we establish 487 * the interrupt handler. 488 */ 489 sun8i_crypto_rng_attach(sc); 490 491 /* 492 * Self-test has passed and first RNG draw has finished. Use 493 * interrupts, not polling, for all subsequent tasks. Set this 494 * atomically in case the interrupt handler has fired -- can't 495 * be from us because we've kept ICR set to 0 to mask all 496 * interrupts, but in case the interrupt vector is shared. 497 */ 498 atomic_store_relaxed(&sc->sc_polling, false); 499 500 /* Attach the sysctl. */ 501 sun8i_crypto_sysctl_attach(sc); 502 503 /* Register opencrypto handlers. */ 504 sun8i_crypto_register(sc); 505 } 506 507 static int 508 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 509 { 510 struct sun8i_crypto_softc *sc = cookie; 511 struct sun8i_crypto_task *task = vtask; 512 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 513 int error; 514 515 /* Create a DMA buffer for the task descriptor. */ 516 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 517 &task->ct_descbuf, dmaflags); 518 if (error) 519 goto fail0; 520 task->ct_desc = task->ct_descbuf.cb_kva; 521 522 /* Create DMA buffers for the IV and CTR. */ 523 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 524 &task->ct_ivbuf, dmaflags); 525 if (error) 526 goto fail1; 527 task->ct_iv = task->ct_ivbuf.cb_kva; 528 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 529 &task->ct_ctrbuf, dmaflags); 530 if (error) 531 goto fail2; 532 task->ct_ctr = task->ct_ctrbuf.cb_kva; 533 534 /* Create a DMA map for the task descriptor and preload it. */ 535 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 536 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 537 if (error) 538 goto fail3; 539 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 540 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 541 if (error) 542 goto fail4; 543 544 /* Create DMA maps for the key, IV, and CTR. */ 545 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 546 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 547 if (error) 548 goto fail5; 549 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 550 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 551 if (error) 552 goto fail6; 553 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 554 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 555 if (error) 556 goto fail7; 557 558 /* Create DMA maps for the src and dst scatter/gather vectors. */ 559 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 560 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 561 &task->ct_srcmap); 562 if (error) 563 goto fail8; 564 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 565 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 566 &task->ct_dstmap); 567 if (error) 568 goto fail9; 569 570 /* Success! */ 571 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 572 return 0; 573 574 fail10: __unused 575 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 576 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 577 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 578 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 579 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 580 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 581 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 582 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 583 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 584 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 585 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 586 return error; 587 } 588 589 static void 590 sun8i_crypto_task_dtor(void *cookie, void *vtask) 591 { 592 struct sun8i_crypto_softc *sc = cookie; 593 struct sun8i_crypto_task *task = vtask; 594 595 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 596 597 /* XXX Zero the bounce buffers if there are any. */ 598 599 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 600 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 601 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 602 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 603 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 604 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 605 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 606 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 607 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 608 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 609 } 610 611 /* 612 * sun8i_crypto_task_get(sc, callback, cookie, pflags) 613 * 614 * Allocate a task that will call callback(sc, task, cookie, 615 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 616 * PR_NOWAIT, may fail and return NULL. No further allocation is 617 * needed to submit the task if this succeeds (although task 618 * submission may still fail if all channels are busy). 619 */ 620 static struct sun8i_crypto_task * 621 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 622 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 623 void *, int), 624 void *cookie, int pflags) 625 { 626 struct sun8i_crypto_task *task; 627 628 /* Allocate a task, or fail if we can't. */ 629 task = pool_cache_get(sc->sc_taskpool, pflags); 630 if (task == NULL) 631 goto out; 632 633 /* Set up flags and the callback. */ 634 task->ct_flags = 0; 635 task->ct_callback = callback; 636 task->ct_cookie = cookie; 637 638 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 639 return task; 640 } 641 642 /* 643 * sun8i_crypto_task_invalid(sc, task, cookie, error) 644 * 645 * Callback for a task not currently in use, to detect errors. 646 */ 647 static void 648 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 649 struct sun8i_crypto_task *task, void *cookie, int error) 650 { 651 void (*callback)(struct sun8i_crypto_softc *, 652 struct sun8i_crypto_task *, void *, int) = cookie; 653 654 panic("task for callback %p used after free", callback); 655 } 656 657 /* 658 * sun8i_crypto_task_put(sc, task) 659 * 660 * Free a task obtained with sun8i_crypto_task_get. 661 */ 662 static void 663 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 664 struct sun8i_crypto_task *task) 665 { 666 667 SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 668 669 task->ct_cookie = task->ct_callback; 670 task->ct_callback = &sun8i_crypto_task_invalid; 671 pool_cache_put(sc->sc_taskpool, task); 672 } 673 674 /* 675 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 676 * 677 * Set up the task descriptor after the relevant DMA maps have 678 * been loaded for a transfer of nbytes. bus_dmamap_sync matches 679 * sun8i_crypto_chan_done. May fail if input is inadequately 680 * aligned. 681 * 682 * XXX Teach this to support task chains. 683 */ 684 static int 685 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 686 struct sun8i_crypto_task *task, uint32_t nbytes, 687 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 688 { 689 struct sun8i_crypto_taskdesc *desc = task->ct_desc; 690 int error; 691 692 KASSERT(tdqs == 0 || tdqa == 0); 693 KASSERT(nbytes % 4 == 0); 694 695 memset(desc, 0, sizeof(*desc)); 696 697 /* Always enable interrupt for the task. */ 698 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 699 700 desc->td_tdqc = htole32(tdqc); 701 desc->td_tdqs = htole32(tdqs); 702 desc->td_tdqa = htole32(tdqa); 703 704 if (task->ct_flags & TASK_KEY) { 705 bus_dmamap_t keymap = task->ct_keymap; 706 KASSERT(keymap->dm_nsegs == 1); 707 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 708 bus_dmamap_sync(sc->sc_dmat, keymap, 0, 709 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 710 } 711 if (task->ct_flags & TASK_IV) { 712 bus_dmamap_t ivmap = task->ct_ivmap; 713 KASSERT(ivmap->dm_nsegs == 1); 714 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 715 bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 716 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 717 } 718 if (task->ct_flags & TASK_CTR) { 719 bus_dmamap_t ctrmap = task->ct_ctrmap; 720 KASSERT(ctrmap->dm_nsegs == 1); 721 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 722 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 723 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 724 } 725 726 if (task->ct_flags & TASK_BYTES) 727 desc->td_datalen = htole32(nbytes); 728 else 729 desc->td_datalen = htole32(nbytes/4); 730 731 if (task->ct_flags & TASK_SRC) { 732 bus_dmamap_t srcmap = task->ct_srcmap; 733 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 734 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 735 nbytes); 736 if (error) 737 return error; 738 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 739 BUS_DMASYNC_PREWRITE); 740 } 741 742 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 743 nbytes); 744 if (error) 745 goto out; 746 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 747 BUS_DMASYNC_PREREAD); 748 749 task->ct_nbytes = nbytes; 750 751 /* Success! */ 752 error = 0; 753 754 out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 755 task, tdqc, tdqs, tdqa, desc, error); 756 return error; 757 } 758 759 /* 760 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 761 * 762 * Set up a task's scatter/gather vector -- src or dst -- with the 763 * given DMA map for a transfer of nbytes. May fail if input is 764 * inadequately aligned. 765 */ 766 static int 767 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 768 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 769 uint32_t nbytes __diagused) 770 { 771 uint32_t total __diagused = 0; 772 unsigned i; 773 774 /* 775 * Verify that the alignment is correct and initialize the 776 * scatter/gather vector. 777 */ 778 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 779 for (i = 0; i < map->dm_nsegs; i++) { 780 if ((map->dm_segs[i].ds_addr % 4) | 781 (map->dm_segs[i].ds_len % 4)) { 782 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 783 task, 784 map->dm_segs[i].ds_addr, 785 map->dm_segs[i].ds_len); 786 return EINVAL; 787 } 788 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 789 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 790 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 791 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 792 total += map->dm_segs[i].ds_len; 793 } 794 795 /* Set the remainder to zero. */ 796 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 797 adrlen[i].adr = 0; 798 adrlen[i].len = 0; 799 } 800 801 /* Verify the total size matches the transfer length. */ 802 KASSERT(total == nbytes); 803 804 /* Success! */ 805 return 0; 806 } 807 808 /* 809 * sun8i_crypto_task_load_trng(task, nbytes) 810 * 811 * Set up the task descriptor for a transfer of nbytes from the 812 * TRNG. 813 */ 814 static int 815 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 816 struct sun8i_crypto_task *task, uint32_t nbytes) 817 { 818 uint32_t tdqc = 0; 819 820 /* Caller must provide dst only. */ 821 KASSERT((task->ct_flags & TASK_KEY) == 0); 822 KASSERT((task->ct_flags & TASK_IV) == 0); 823 KASSERT((task->ct_flags & TASK_CTR) == 0); 824 KASSERT((task->ct_flags & TASK_SRC) == 0); 825 826 /* Set up the task descriptor queue control words. */ 827 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 828 SUN8I_CRYPTO_TDQC_METHOD); 829 830 /* Fill in the descriptor. */ 831 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 832 } 833 834 static int 835 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 836 struct sun8i_crypto_task *task, 837 uint32_t nbytes, uint32_t keysize, uint32_t dir) 838 { 839 uint32_t tdqc = 0, tdqs = 0; 840 841 /* Caller must provide key, src, and dst only. */ 842 KASSERT(task->ct_flags & TASK_KEY); 843 KASSERT((task->ct_flags & TASK_IV) == 0); 844 KASSERT((task->ct_flags & TASK_CTR) == 0); 845 KASSERT(task->ct_flags & TASK_SRC); 846 847 /* Set up the task descriptor queue control word. */ 848 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 849 SUN8I_CRYPTO_TDQC_METHOD); 850 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 851 852 #ifdef DIAGNOSTIC 853 switch (keysize) { 854 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 855 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 856 break; 857 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 858 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 859 break; 860 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 861 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 862 break; 863 } 864 #endif 865 866 /* Set up the symmetric control word. */ 867 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 868 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 869 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 870 SUN8I_CRYPTO_TDQS_OP_MODE); 871 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 872 873 /* Fill in the descriptor. */ 874 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 875 } 876 877 /* 878 * sun8i_crypto_submit(sc, task) 879 * 880 * Submit a task to the crypto engine after it has been loaded 881 * with sun8i_crypto_task_load. On success, guarantees to 882 * eventually call the task's callback. 883 */ 884 static int 885 sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 886 struct sun8i_crypto_task *task) 887 { 888 unsigned i, retries = 0; 889 uint32_t icr; 890 int error = 0; 891 892 /* One at a time at the device registers, please. */ 893 mutex_enter(&sc->sc_lock); 894 895 /* Find a channel. */ 896 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 897 if (sc->sc_chan[i].cc_task == NULL) 898 break; 899 } 900 if (i == SUN8I_CRYPTO_NCHAN) { 901 device_printf(sc->sc_dev, "no free channels\n"); 902 error = ERESTART; 903 goto out; 904 } 905 906 /* 907 * Set the channel id. Caller is responsible for setting up 908 * all other parts of the descriptor. 909 */ 910 task->ct_desc->td_cid = htole32(i); 911 912 /* 913 * Prepare to send the descriptor to the device by DMA. 914 * Matches POSTWRITE in sun8i_crypto_chan_done. 915 */ 916 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 917 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 918 919 /* Confirm we're ready to go. */ 920 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 921 device_printf(sc->sc_dev, "TLR not clear\n"); 922 error = EIO; 923 goto out; 924 } 925 926 /* 927 * Enable interrupts for this channel, unless we're still 928 * polling. 929 */ 930 if (!sc->sc_polling) { 931 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 932 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 933 SUN8I_CRYPTO_ICR_INTR_EN); 934 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 935 } 936 937 /* Set the task descriptor queue address. */ 938 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 939 task->ct_descmap->dm_segs[0].ds_addr); 940 941 /* Notify the engine to load it, and wait for acknowledgement. */ 942 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 943 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 944 { 945 /* 946 * XXX Timeout pulled from arse. Is it even important 947 * to wait here? 948 */ 949 if (++retries == 1000) { 950 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 951 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 952 /* 953 * Hope it clears eventually; if not, we'll 954 * time out. 955 */ 956 break; 957 } 958 DELAY(1); 959 } 960 961 /* 962 * Loaded up and ready to go. Start a timer ticking if it's 963 * not already and we're not polling. 964 */ 965 sc->sc_chan[i].cc_task = task; 966 sc->sc_chan[i].cc_starttime = getticks(); 967 if (!sc->sc_polling && !callout_pending(&sc->sc_timeout)) 968 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 969 970 out: /* Done! */ 971 if (error) 972 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 973 sc, task, error); 974 else 975 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 976 sc, task, i); 977 mutex_exit(&sc->sc_lock); 978 return error; 979 } 980 981 /* 982 * sun8i_crypto_timeout(cookie) 983 * 984 * Timeout handler. Schedules work in a thread to cancel all 985 * pending tasks that were started long enough ago we're bored of 986 * waiting for them. 987 */ 988 static void 989 sun8i_crypto_timeout(void *cookie) 990 { 991 struct sun8i_crypto_softc *sc = cookie; 992 993 mutex_enter(&sc->sc_intr_lock); 994 sun8i_crypto_schedule_worker(sc); 995 mutex_exit(&sc->sc_intr_lock); 996 } 997 998 /* 999 * sun8i_crypto_intr(cookie) 1000 * 1001 * Device interrupt handler. Find what channels have completed, 1002 * whether with success or with failure, and schedule work in 1003 * thread context to invoke the appropriate callbacks. 1004 */ 1005 static int 1006 sun8i_crypto_intr(void *cookie) 1007 { 1008 struct sun8i_crypto_softc *sc = cookie; 1009 uint32_t done, esr; 1010 1011 if (atomic_load_relaxed(&sc->sc_polling) || 1012 !sun8i_crypto_poll(sc, &done, &esr)) 1013 return 0; /* not ours */ 1014 1015 mutex_enter(&sc->sc_intr_lock); 1016 sun8i_crypto_schedule_worker(sc); 1017 sc->sc_done |= done; 1018 sc->sc_esr |= esr; 1019 mutex_exit(&sc->sc_intr_lock); 1020 1021 return 1; 1022 } 1023 1024 /* 1025 * sun8i_crypto_schedule_worker(sc) 1026 * 1027 * Ensure that crypto engine thread context work to invoke task 1028 * callbacks will run promptly. Idempotent. 1029 */ 1030 static void 1031 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 1032 { 1033 1034 KASSERT(mutex_owned(&sc->sc_intr_lock)); 1035 1036 /* Start the worker if necessary. */ 1037 if (!sc->sc_work_pending) { 1038 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1039 sc->sc_work_pending = true; 1040 } 1041 } 1042 1043 /* 1044 * sun8i_crypto_worker(wk, cookie) 1045 * 1046 * Thread-context worker: Invoke all task callbacks for which the 1047 * device has notified us of completion or for which we gave up 1048 * waiting. 1049 */ 1050 static void 1051 sun8i_crypto_worker(struct work *wk, void *cookie) 1052 { 1053 struct sun8i_crypto_softc *sc = cookie; 1054 uint32_t done, esr; 1055 1056 /* 1057 * Under the interrupt lock, acknowledge our work and claim the 1058 * done mask and error status. 1059 */ 1060 mutex_enter(&sc->sc_intr_lock); 1061 KASSERT(sc->sc_work_pending); 1062 sc->sc_work_pending = false; 1063 done = sc->sc_done; 1064 esr = sc->sc_esr; 1065 sc->sc_done = 0; 1066 sc->sc_esr = 0; 1067 mutex_exit(&sc->sc_intr_lock); 1068 1069 /* 1070 * If we cleared any channels, it is time to allow opencrypto 1071 * to issue new operations. Asymmetric operations (which we 1072 * don't support, at the moment, but we could) and symmetric 1073 * operations (which we do) use the same task channels, so we 1074 * unblock both kinds. 1075 */ 1076 if (sun8i_crypto_done(sc, done, esr, getticks())) { 1077 crypto_unblock(sc->sc_opencrypto.co_driverid, 1078 CRYPTO_SYMQ|CRYPTO_ASYMQ); 1079 } 1080 } 1081 1082 /* 1083 * sun8i_crypto_poll(sc, &done, &esr) 1084 * 1085 * Poll for completion. Sets done and esr to the mask of done 1086 * channels and error channels. Returns true if anything was 1087 * done or failed. 1088 */ 1089 static bool 1090 sun8i_crypto_poll(struct sun8i_crypto_softc *sc, 1091 uint32_t *donep, uint32_t *esrp) 1092 { 1093 uint32_t isr, esr; 1094 1095 /* 1096 * Get and acknowledge the interrupts and error status. 1097 * 1098 * XXX Data sheet says the error status register is read-only, 1099 * but then advises writing 1 to bit x1xx (keysram access error 1100 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 1101 */ 1102 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 1103 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 1104 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 1105 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 1106 1107 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 1108 1109 *donep = __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 1110 *esrp = esr; 1111 1112 return *donep || *esrp; 1113 } 1114 1115 /* 1116 * sun8i_crypto_done(sc, done, esr, now) 1117 * 1118 * Invoke all task callbacks for the channels in done or esr, or 1119 * for which we gave up waiting, according to the time `now'. 1120 * Returns true if any channels completed or timed out. 1121 */ 1122 static bool 1123 sun8i_crypto_done(struct sun8i_crypto_softc *sc, uint32_t done, uint32_t esr, 1124 unsigned now) 1125 { 1126 uint32_t esr_chan; 1127 unsigned i; 1128 bool anydone = false; 1129 bool schedtimeout = false; 1130 int error; 1131 1132 /* Under the lock, process the channels. */ 1133 mutex_enter(&sc->sc_lock); 1134 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 1135 /* Check whether the channel is done. */ 1136 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 1137 /* Nope. Do we have a task to time out? */ 1138 if (sc->sc_chan[i].cc_task != NULL) { 1139 if (now - sc->sc_chan[i].cc_starttime >= 1140 SUN8I_CRYPTO_TIMEOUT) { 1141 anydone |= sun8i_crypto_chan_done(sc, 1142 i, ETIMEDOUT); 1143 } else { 1144 schedtimeout = true; 1145 } 1146 } 1147 continue; 1148 } 1149 1150 /* Channel is done. Interpret the error if any. */ 1151 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1152 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1153 device_printf(sc->sc_dev, "channel %u:" 1154 " alg not supported\n", i); 1155 error = ENODEV; 1156 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1157 device_printf(sc->sc_dev, "channel %u:" 1158 " data length error\n", i); 1159 error = EIO; /* XXX */ 1160 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1161 device_printf(sc->sc_dev, "channel %u:" 1162 " key sram error\n", i); 1163 error = EIO; /* XXX */ 1164 } else if (esr_chan != 0) { 1165 error = EIO; /* generic I/O error */ 1166 } else { 1167 error = 0; 1168 } 1169 1170 /* 1171 * Notify the task of completion. May release the lock 1172 * to invoke a callback. 1173 */ 1174 anydone |= sun8i_crypto_chan_done(sc, i, error); 1175 } 1176 mutex_exit(&sc->sc_lock); 1177 1178 /* 1179 * If there are tasks still pending, make sure there's a 1180 * timeout scheduled for them. If the callout is already 1181 * pending, it will take another pass through here to time some 1182 * things out and schedule a new timeout. 1183 */ 1184 if (schedtimeout && !callout_pending(&sc->sc_timeout)) 1185 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 1186 1187 return anydone; 1188 } 1189 1190 /* 1191 * sun8i_crypto_chan_done(sc, i, error) 1192 * 1193 * Notify the callback for the task on channel i, if there is one, 1194 * of the specified error, or 0 for success. 1195 */ 1196 static bool 1197 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1198 { 1199 struct sun8i_crypto_task *task; 1200 uint32_t nbytes; 1201 uint32_t icr; 1202 1203 KASSERT(mutex_owned(&sc->sc_lock)); 1204 1205 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 1206 1207 /* Claim the task if there is one; bail if not. */ 1208 if ((task = sc->sc_chan[i].cc_task) == NULL) { 1209 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1210 i, error); 1211 /* We did not clear a channel. */ 1212 return false; 1213 } 1214 sc->sc_chan[i].cc_task = NULL; 1215 1216 /* Disable interrupts on this channel. */ 1217 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1218 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1219 SUN8I_CRYPTO_ICR_INTR_EN); 1220 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1221 1222 /* 1223 * Finished sending the descriptor to the device by DMA. 1224 * Matches PREWRITE in sun8i_crypto_task_submit. 1225 */ 1226 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1227 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1228 1229 /* 1230 * Finished with all the other bits of DMA too. Matches 1231 * sun8i_crypto_task_load. 1232 */ 1233 nbytes = task->ct_nbytes; 1234 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 1235 BUS_DMASYNC_POSTREAD); 1236 if (task->ct_flags & TASK_SRC) 1237 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 1238 BUS_DMASYNC_POSTWRITE); 1239 if (task->ct_flags & TASK_CTR) 1240 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 1241 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 1242 if (task->ct_flags & TASK_IV) 1243 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 1244 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1245 if (task->ct_flags & TASK_KEY) 1246 /* XXX Can we zero the bounce buffer if there is one? */ 1247 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 1248 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1249 1250 /* Temporarily release the lock to invoke the callback. */ 1251 mutex_exit(&sc->sc_lock); 1252 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1253 (*task->ct_callback)(sc, task, task->ct_cookie, error); 1254 mutex_enter(&sc->sc_lock); 1255 1256 /* We cleared a channel. */ 1257 return true; 1258 } 1259 1260 /* 1261 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 1262 * 1263 * Allocate a single-segment DMA-safe buffer and map it into KVA. 1264 * May fail if dmaflags is BUS_DMA_NOWAIT. 1265 */ 1266 static int 1267 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 1268 struct sun8i_crypto_buf *buf, int dmaflags) 1269 { 1270 int error; 1271 1272 /* Allocate a DMA-safe buffer. */ 1273 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 1274 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1275 if (error) 1276 goto fail0; 1277 1278 /* Map the buffer into kernel virtual address space. */ 1279 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 1280 size, &buf->cb_kva, dmaflags); 1281 if (error) 1282 goto fail1; 1283 1284 /* Success! */ 1285 return 0; 1286 1287 fail2: __unused 1288 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1289 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1290 fail0: return error; 1291 } 1292 1293 /* 1294 * sun8i_crypto_freebuf(sc, buf) 1295 * 1296 * Unmap buf and free it. 1297 */ 1298 static void 1299 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1300 struct sun8i_crypto_buf *buf) 1301 { 1302 1303 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1304 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1305 } 1306 1307 /* 1308 * sun8i_crypto_rng_attach(sc) 1309 * 1310 * Attach an rndsource for the crypto engine's TRNG. 1311 */ 1312 static void 1313 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1314 { 1315 device_t self = sc->sc_dev; 1316 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1317 struct sun8i_crypto_task *task; 1318 unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT); 1319 uint32_t done, esr; 1320 int error; 1321 1322 /* Preallocate a buffer to reuse. */ 1323 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 1324 BUS_DMA_WAITOK); 1325 if (error) { 1326 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 1327 error); 1328 goto fail0; 1329 } 1330 1331 /* Create a task to reuse. */ 1332 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 1333 rng, PR_WAITOK); 1334 if (rng->cr_task == NULL) { 1335 aprint_error_dev(self, "failed to allocate RNG task\n"); 1336 error = ENOMEM; 1337 goto fail1; 1338 } 1339 1340 /* Preload the destination map. */ 1341 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1342 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1343 if (error) { 1344 aprint_error_dev(self, "failed to load RNG buffer: %d\n", 1345 error); 1346 goto fail2; 1347 } 1348 1349 /* 1350 * Attach the rndsource. This will trigger an initial call to 1351 * it since we have RND_FLAG_HASCB. 1352 */ 1353 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1354 rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1355 RND_TYPE_RNG, 1356 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1357 1358 /* 1359 * Poll for the first call to the RNG to complete. If not done 1360 * after the timeout, force a timeout. 1361 */ 1362 for (; rng->cr_pending && timo --> 0; DELAY(1000)) { 1363 if (sun8i_crypto_poll(sc, &done, &esr)) 1364 (void)sun8i_crypto_done(sc, done, esr, getticks()); 1365 } 1366 if (rng->cr_pending) { 1367 (void)sun8i_crypto_done(sc, 0, 0, 1368 (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT); 1369 KASSERT(!rng->cr_pending); 1370 } 1371 1372 return; 1373 1374 fail3: __unused 1375 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1376 fail2: sun8i_crypto_task_put(sc, task); 1377 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 1378 fail0: return; 1379 } 1380 1381 /* 1382 * sun8i_crypto_rng_get(nbytes, cookie) 1383 * 1384 * On-demand rndsource callback: try to gather nbytes of entropy 1385 * and enter them into the pool ASAP. 1386 */ 1387 static void 1388 sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1389 { 1390 struct sun8i_crypto_softc *sc = cookie; 1391 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1392 struct sun8i_crypto_task *task = rng->cr_task; 1393 bool pending; 1394 int error; 1395 1396 /* 1397 * Test and set the RNG-pending flag. If it's already in 1398 * progress, nothing to do here. 1399 */ 1400 mutex_enter(&sc->sc_lock); 1401 pending = rng->cr_pending; 1402 rng->cr_pending = true; 1403 mutex_exit(&sc->sc_lock); 1404 if (pending) 1405 return; 1406 1407 /* Load the task descriptor. */ 1408 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1409 if (error) 1410 goto fail; 1411 1412 /* Submit! */ 1413 error = sun8i_crypto_submit(sc, task); 1414 if (error) 1415 goto fail; 1416 1417 /* All done! */ 1418 return; 1419 1420 fail: mutex_enter(&sc->sc_lock); 1421 rng->cr_pending = false; 1422 mutex_exit(&sc->sc_lock); 1423 } 1424 1425 static void 1426 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1427 struct sun8i_crypto_task *task, void *cookie, int error) 1428 { 1429 struct sun8i_crypto_rng *rng = cookie; 1430 uint8_t *buf = rng->cr_buf.cb_kva; 1431 uint32_t entropybits; 1432 1433 KASSERT(rng == &sc->sc_rng); 1434 1435 /* If anything went wrong, forget about it. */ 1436 if (error) 1437 goto out; 1438 1439 /* 1440 * This TRNG has quite low entropy at best. But if it fails a 1441 * repeated output test, then assume it's busted. 1442 */ 1443 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1444 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1445 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1446 SUN8I_CRYPTO_RNGBYTES/2)) { 1447 device_printf(sc->sc_dev, "failed repeated output test\n"); 1448 entropybits = 0; 1449 } 1450 1451 /* 1452 * Actually we don't believe in any of the entropy until this 1453 * device has had more scrutiny. 1454 */ 1455 entropybits = 0; 1456 1457 /* Success! Enter and erase the data. */ 1458 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1459 entropybits); 1460 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1461 1462 out: /* Done -- clear the RNG-pending flag. */ 1463 mutex_enter(&sc->sc_lock); 1464 rng->cr_pending = false; 1465 mutex_exit(&sc->sc_lock); 1466 } 1467 1468 /* 1469 * Self-test 1470 */ 1471 1472 static const uint8_t selftest_input[16]; 1473 static const uint8_t selftest_key[16]; 1474 static const uint8_t selftest_output[16] = { 1475 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1476 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1477 }; 1478 1479 static bool 1480 sun8i_crypto_selftest(struct sun8i_crypto_softc *sc) 1481 { 1482 const size_t keybytes = sizeof selftest_key; 1483 const size_t nbytes = sizeof selftest_input; 1484 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 1485 struct sun8i_crypto_task *task; 1486 unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT); 1487 uint32_t done, esr; 1488 int error; 1489 1490 CTASSERT(sizeof selftest_input == sizeof selftest_output); 1491 1492 selftest->cs_pending = true; 1493 1494 /* Allocate an input buffer. */ 1495 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 1496 BUS_DMA_WAITOK); 1497 if (error) 1498 goto fail0; 1499 1500 /* Allocate a key buffer. */ 1501 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 1502 BUS_DMA_WAITOK); 1503 if (error) 1504 goto fail1; 1505 1506 /* Allocate an output buffer. */ 1507 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 1508 BUS_DMA_WAITOK); 1509 if (error) 1510 goto fail2; 1511 1512 /* Allocate a task descriptor. */ 1513 task = selftest->cs_task = sun8i_crypto_task_get(sc, 1514 sun8i_crypto_selftest_done, selftest, PR_WAITOK); 1515 if (selftest->cs_task == NULL) { 1516 error = ENOMEM; 1517 goto fail3; 1518 } 1519 1520 /* Copy the input and key into their buffers. */ 1521 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 1522 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1523 1524 /* Load the key, src, and dst for DMA transfers. */ 1525 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 1526 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1527 if (error) 1528 goto fail4; 1529 task->ct_flags |= TASK_KEY; 1530 1531 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 1532 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1533 if (error) 1534 goto fail5; 1535 task->ct_flags |= TASK_SRC; 1536 1537 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1538 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1539 if (error) 1540 goto fail6; 1541 1542 /* Set up the task descriptor. */ 1543 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 1544 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 1545 if (error) 1546 goto fail7; 1547 1548 /* Submit! */ 1549 error = sun8i_crypto_submit(sc, task); 1550 if (error) 1551 goto fail7; 1552 device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1553 1554 /* 1555 * Poll for completion. If not done after the timeout, force a 1556 * timeout. 1557 */ 1558 for (; selftest->cs_pending && timo --> 0; DELAY(1000)) { 1559 if (sun8i_crypto_poll(sc, &done, &esr)) 1560 (void)sun8i_crypto_done(sc, done, esr, getticks()); 1561 } 1562 if (selftest->cs_pending) { 1563 (void)sun8i_crypto_done(sc, 0, 0, 1564 (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT); 1565 KASSERT(!selftest->cs_pending); 1566 } 1567 1568 return selftest->cs_passed; 1569 1570 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1571 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1572 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1573 fail4: sun8i_crypto_task_put(sc, task); 1574 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1575 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1576 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1577 fail0: aprint_error_dev(sc->sc_dev, "failed to run self-test, error=%d\n", 1578 error); 1579 selftest->cs_pending = false; 1580 return false; 1581 } 1582 1583 static bool 1584 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1585 size_t n, const void *expected, const void *actual) 1586 { 1587 const uint8_t *e = expected; 1588 const uint8_t *a = actual; 1589 size_t i; 1590 1591 if (memcmp(e, a, n) == 0) 1592 return true; 1593 1594 device_printf(sc->sc_dev, "self-test: %s\n", title); 1595 printf("expected: "); 1596 for (i = 0; i < n; i++) 1597 printf("%02hhx", e[i]); 1598 printf("\n"); 1599 printf("actual: "); 1600 for (i = 0; i < n; i++) 1601 printf("%02hhx", a[i]); 1602 printf("\n"); 1603 return false; 1604 } 1605 1606 static void 1607 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1608 struct sun8i_crypto_task *task, void *cookie, int error) 1609 { 1610 const size_t keybytes = sizeof selftest_key; 1611 const size_t nbytes = sizeof selftest_input; 1612 struct sun8i_crypto_selftest *selftest = cookie; 1613 bool ok = true; 1614 1615 KASSERT(selftest == &sc->sc_selftest); 1616 1617 /* If anything went wrong, fail now. */ 1618 if (error) { 1619 device_printf(sc->sc_dev, "self-test error=%d\n", error); 1620 goto out; 1621 } 1622 1623 /* 1624 * Verify the input and key weren't clobbered, and verify the 1625 * output matches what we expect. 1626 */ 1627 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 1628 selftest_input, selftest->cs_in.cb_kva); 1629 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 1630 selftest_key, selftest->cs_key.cb_kva); 1631 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 1632 selftest_output, selftest->cs_out.cb_kva); 1633 1634 selftest->cs_passed = ok; 1635 if (ok) 1636 device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1637 1638 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1639 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1640 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1641 sun8i_crypto_task_put(sc, task); 1642 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1643 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1644 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1645 selftest->cs_pending = false; 1646 } 1647 1648 /* 1649 * Sysctl for testing 1650 */ 1651 1652 struct sun8i_crypto_userreq { 1653 kmutex_t cu_lock; 1654 kcondvar_t cu_cv; 1655 size_t cu_size; 1656 struct sun8i_crypto_buf cu_buf; 1657 struct sun8i_crypto_task *cu_task; 1658 int cu_error; 1659 bool cu_done; 1660 bool cu_cancel; 1661 }; 1662 1663 static void 1664 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1665 { 1666 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1667 int error; 1668 1669 /* hw.sun8icryptoN (node) */ 1670 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1671 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1672 SYSCTL_DESCR("sun8i crypto engine knobs"), 1673 NULL, 0, NULL, 0, 1674 CTL_HW, CTL_CREATE, CTL_EOL); 1675 if (error) { 1676 aprint_error_dev(sc->sc_dev, 1677 "failed to set up sysctl hw.%s: %d\n", 1678 device_xname(sc->sc_dev), error); 1679 return; 1680 } 1681 1682 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1683 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1684 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1685 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1686 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1687 if (error) { 1688 aprint_error_dev(sc->sc_dev, 1689 "failed to set up sysctl hw.%s.rng: %d\n", 1690 device_xname(sc->sc_dev), error); 1691 return; 1692 } 1693 } 1694 1695 static int 1696 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1697 { 1698 struct sysctlnode node = *rnode; 1699 struct sun8i_crypto_softc *sc = node.sysctl_data; 1700 struct sun8i_crypto_userreq *req; 1701 struct sun8i_crypto_task *task; 1702 size_t size; 1703 int error; 1704 1705 /* If oldp == NULL, the caller wants to learn the size. */ 1706 if (oldp == NULL) { 1707 *oldlenp = 4096; 1708 return 0; 1709 } 1710 1711 /* Truncate to 4096 bytes. */ 1712 size = MIN(4096, *oldlenp); 1713 if (size == 0) 1714 return 0; /* nothing to do */ 1715 1716 /* Allocate a request context. */ 1717 req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1718 if (req == NULL) 1719 return ENOMEM; 1720 1721 /* Initialize the request context. */ 1722 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1723 cv_init(&req->cu_cv, "sun8isy"); 1724 req->cu_size = size; 1725 req->cu_error = EIO; 1726 req->cu_done = false; 1727 req->cu_cancel = false; 1728 1729 /* Allocate a buffer for the RNG output. */ 1730 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1731 if (error) 1732 goto out0; 1733 1734 /* Allocate a task. */ 1735 task = req->cu_task = sun8i_crypto_task_get(sc, 1736 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 1737 if (task == NULL) { 1738 error = ENOMEM; 1739 goto out1; 1740 } 1741 1742 /* Set the task up for TRNG to our buffer. */ 1743 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1744 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1745 if (error) 1746 goto out2; 1747 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1748 if (error) 1749 goto out3; 1750 1751 /* Submit! */ 1752 error = sun8i_crypto_submit(sc, task); 1753 if (error) { 1754 /* Make sure we don't restart the syscall -- just fail. */ 1755 if (error == ERESTART) 1756 error = EBUSY; 1757 goto out3; 1758 } 1759 1760 /* Wait for the request to complete. */ 1761 mutex_enter(&req->cu_lock); 1762 while (!req->cu_done) { 1763 error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1764 if (error) { 1765 /* 1766 * If we finished while waiting to acquire the 1767 * lock, ignore the error and just return now. 1768 * Otherwise, notify the callback that it has 1769 * to clean up after us. 1770 */ 1771 if (req->cu_done) 1772 error = 0; 1773 else 1774 req->cu_cancel = true; 1775 break; 1776 } 1777 } 1778 mutex_exit(&req->cu_lock); 1779 1780 /* 1781 * Return early on error from cv_wait_sig, which means 1782 * interruption; the callback will clean up instead. 1783 */ 1784 if (error) 1785 return error; 1786 1787 /* Check for error from the device. */ 1788 error = req->cu_error; 1789 if (error) 1790 goto out3; 1791 1792 /* Copy out the data. */ 1793 node.sysctl_data = req->cu_buf.cb_kva; 1794 node.sysctl_size = size; 1795 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1796 1797 /* Clear the buffer. */ 1798 explicit_memset(req->cu_buf.cb_kva, 0, size); 1799 1800 /* Clean up. */ 1801 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1802 out2: sun8i_crypto_task_put(sc, task); 1803 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1804 out0: cv_destroy(&req->cu_cv); 1805 mutex_destroy(&req->cu_lock); 1806 kmem_free(req, sizeof(*req)); 1807 return error; 1808 } 1809 1810 static void 1811 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1812 struct sun8i_crypto_task *task, void *cookie, int error) 1813 { 1814 struct sun8i_crypto_userreq *req = cookie; 1815 bool cancel; 1816 1817 /* 1818 * Notify the waiting thread of the error, and find out whether 1819 * that thread cancelled. 1820 */ 1821 mutex_enter(&req->cu_lock); 1822 cancel = req->cu_cancel; 1823 req->cu_error = error; 1824 req->cu_done = true; 1825 cv_broadcast(&req->cu_cv); 1826 mutex_exit(&req->cu_lock); 1827 1828 /* 1829 * If it wasn't cancelled, we're done -- the main thread will 1830 * clean up after itself. 1831 */ 1832 if (!cancel) 1833 return; 1834 1835 /* Clean up after the main thread cancelled. */ 1836 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1837 sun8i_crypto_task_put(sc, task); 1838 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1839 cv_destroy(&req->cu_cv); 1840 mutex_destroy(&req->cu_lock); 1841 kmem_free(req, sizeof(*req)); 1842 } 1843 1844 /* 1845 * sun8i_crypto_register(sc) 1846 * 1847 * Register opencrypto algorithms supported by the crypto engine. 1848 */ 1849 static void 1850 sun8i_crypto_register(struct sun8i_crypto_softc *sc) 1851 { 1852 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 1853 1854 co->co_driverid = crypto_get_driverid(0); 1855 if (co->co_driverid == (uint32_t)-1) { 1856 aprint_error_dev(sc->sc_dev, 1857 "failed to register crypto driver\n"); 1858 return; 1859 } 1860 1861 sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 1862 sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 1863 #ifdef CRYPTO_AES_ECB 1864 sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 1865 #endif 1866 #ifdef CRYPTO_AES_XTS 1867 sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 1868 #endif 1869 #ifdef CRYPTO_DES_CBC 1870 sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 1871 #endif 1872 #ifdef CRYPTO_DES_ECB 1873 sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 1874 #endif 1875 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 1876 #ifdef CRYPTO_3DES_ECB 1877 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 1878 #endif 1879 1880 sun8i_crypto_register1(sc, CRYPTO_MD5); 1881 sun8i_crypto_register1(sc, CRYPTO_SHA1); 1882 #ifdef CRYPTO_SHA224 1883 sun8i_crypto_register1(sc, CRYPTO_SHA224); 1884 #endif 1885 #ifdef CRYPTO_SHA256 1886 sun8i_crypto_register1(sc, CRYPTO_SHA256); 1887 #endif 1888 1889 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 1890 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 1891 1892 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 1893 } 1894 1895 /* 1896 * sun8i_crypto_register1(sc, alg) 1897 * 1898 * Register support for one algorithm alg using 1899 * sun8i_crypto_newsession/freesession/process. 1900 */ 1901 static void 1902 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 1903 { 1904 1905 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 1906 sun8i_crypto_newsession, 1907 sun8i_crypto_freesession, 1908 sun8i_crypto_process, 1909 sc); 1910 } 1911 1912 /* 1913 * sun8i_crypto_newsession(cookie, sidp, cri) 1914 * 1915 * Called by opencrypto to allocate a new session. We don't keep 1916 * track of sessions, since there are no persistent keys in the 1917 * hardware that we take advantage of, so this only validates the 1918 * crypto operations and returns a dummy session id of 1. 1919 */ 1920 static int 1921 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 1922 { 1923 1924 /* No composition of operations is supported here. */ 1925 if (cri->cri_next) 1926 return EINVAL; 1927 1928 /* 1929 * No variation of rounds is supported here. (XXX Unused and 1930 * unimplemented in opencrypto(9) altogether?) 1931 */ 1932 if (cri->cri_rnd) 1933 return EINVAL; 1934 1935 /* 1936 * Validate per-algorithm key length. 1937 * 1938 * XXX Does opencrypto(9) do this internally? 1939 */ 1940 switch (cri->cri_alg) { 1941 case CRYPTO_MD5: 1942 case CRYPTO_SHA1: 1943 #ifdef CRYPTO_SHA224 1944 case CRYPTO_SHA224: 1945 #endif 1946 #ifdef CRYPTO_SHA256 1947 case CRYPTO_SHA256: 1948 #endif 1949 if (cri->cri_klen) 1950 return EINVAL; 1951 break; 1952 case CRYPTO_AES_CBC: 1953 #ifdef CRYPTO_AES_ECB 1954 case CRYPTO_AES_ECB: 1955 #endif 1956 switch (cri->cri_klen) { 1957 case 128: 1958 case 192: 1959 case 256: 1960 break; 1961 default: 1962 return EINVAL; 1963 } 1964 break; 1965 case CRYPTO_AES_CTR: 1966 /* 1967 * opencrypto `AES-CTR' takes four bytes of the input 1968 * block as the last four bytes of the key, for reasons 1969 * that are not entirely clear. 1970 */ 1971 switch (cri->cri_klen) { 1972 case 128 + 32: 1973 case 192 + 32: 1974 case 256 + 32: 1975 break; 1976 default: 1977 return EINVAL; 1978 } 1979 break; 1980 #ifdef CRYPTO_AES_XTS 1981 case CRYPTO_AES_XTS: 1982 switch (cri->cri_klen) { 1983 case 256: 1984 case 384: 1985 case 512: 1986 break; 1987 default: 1988 return EINVAL; 1989 } 1990 break; 1991 #endif 1992 case CRYPTO_DES_CBC: 1993 #ifdef CRYPTO_DES_ECB 1994 case CRYPTO_DES_ECB: 1995 #endif 1996 switch (cri->cri_klen) { 1997 case 64: 1998 break; 1999 default: 2000 return EINVAL; 2001 } 2002 break; 2003 case CRYPTO_3DES_CBC: 2004 #ifdef CRYPTO_3DES_ECB 2005 case CRYPTO_3DES_ECB: 2006 #endif 2007 switch (cri->cri_klen) { 2008 case 192: 2009 break; 2010 default: 2011 return EINVAL; 2012 } 2013 break; 2014 case CRYPTO_SHA1_HMAC: 2015 /* 2016 * XXX Unclear what the length limit is, but since HMAC 2017 * behaves qualitatively different for a key of at 2018 * least the full block size -- and is generally best 2019 * to use with half the block size -- let's limit it to 2020 * one block. 2021 */ 2022 if (cri->cri_klen % 8) 2023 return EINVAL; 2024 if (cri->cri_klen > 512) 2025 return EINVAL; 2026 break; 2027 case CRYPTO_SHA2_256_HMAC: 2028 if (cri->cri_klen % 8) 2029 return EINVAL; 2030 if (cri->cri_klen > 512) 2031 return EINVAL; 2032 break; 2033 default: 2034 panic("unsupported algorithm %d", cri->cri_alg); 2035 } 2036 2037 KASSERT(cri->cri_klen % 8 == 0); 2038 2039 /* Success! */ 2040 *sidp = 1; 2041 return 0; 2042 } 2043 2044 /* 2045 * sun8i_crypto_freesession(cookie, dsid) 2046 * 2047 * Called by opencrypto to free a session. We don't keep track of 2048 * sessions, since there are no persistent keys in the hardware 2049 * that we take advantage of, so this is a no-op. 2050 * 2051 * Note: dsid is actually a 64-bit quantity containing both the 2052 * driver id in the high half and the session id in the low half. 2053 */ 2054 static void 2055 sun8i_crypto_freesession(void *cookie, uint64_t dsid) 2056 { 2057 2058 KASSERT((dsid & 0xffffffff) == 1); 2059 } 2060 2061 /* 2062 * sun8i_crypto_ivlen(crd) 2063 * 2064 * Return the crypto engine's notion of `IV length', in bytes, for 2065 * an opencrypto operation. 2066 */ 2067 static u_int 2068 sun8i_crypto_ivlen(const struct cryptodesc *crd) 2069 { 2070 2071 switch (crd->crd_alg) { 2072 case CRYPTO_AES_CBC: 2073 return 16; 2074 #ifdef CRYPTO_AES_XTS 2075 case CRYPTO_AES_XTS: 2076 return 16; 2077 #endif 2078 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 2079 return 8; 2080 #ifdef CRYPTO_DES_CBC 2081 case CRYPTO_DES_CBC: 2082 return 8; 2083 #endif 2084 case CRYPTO_3DES_CBC: 2085 return 8; 2086 case CRYPTO_MD5: 2087 return 16; 2088 #ifdef CRYPTO_SHA224 2089 case CRYPTO_SHA224: 2090 return 32; 2091 #endif 2092 #ifdef CRYPTO_SHA256 2093 case CRYPTO_SHA256: 2094 return 32; 2095 #endif 2096 case CRYPTO_SHA1_HMAC: 2097 return 20; 2098 case CRYPTO_SHA2_256_HMAC: 2099 return 32; 2100 default: 2101 return 0; 2102 } 2103 } 2104 2105 /* 2106 * sun8i_crypto_process(cookie, crp, hint) 2107 * 2108 * Main opencrypto processing dispatch. 2109 */ 2110 static int 2111 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 2112 { 2113 struct sun8i_crypto_softc *sc = cookie; 2114 struct sun8i_crypto_task *task; 2115 struct cryptodesc *crd = crp->crp_desc; 2116 unsigned klen, ivlen; 2117 uint32_t tdqc = 0, tdqs = 0; 2118 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 2119 const uint32_t tdqa = 0; 2120 int error; 2121 2122 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 2123 2124 /* Reject compositions -- we do not handle them. */ 2125 if (crd->crd_next != NULL) { 2126 error = EOPNOTSUPP; 2127 goto fail0; 2128 } 2129 2130 /* Reject transfers with nonsense skip. */ 2131 if (crd->crd_skip < 0) { 2132 error = EINVAL; 2133 goto fail0; 2134 } 2135 2136 /* 2137 * Actually just reject any nonzero skip, because it requires 2138 * DMA segment bookkeeping that we don't do yet. 2139 */ 2140 if (crd->crd_skip) { 2141 error = EOPNOTSUPP; 2142 goto fail0; 2143 } 2144 2145 /* Reject large transfers. */ 2146 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 2147 error = EFBIG; 2148 goto fail0; 2149 } 2150 2151 /* Reject nonsense, unaligned, or mismatched lengths. */ 2152 if (crd->crd_len < 0 || 2153 crd->crd_len % 4 || 2154 crd->crd_len != crp->crp_ilen) { 2155 error = EINVAL; 2156 goto fail0; 2157 } 2158 2159 /* Reject mismatched buffer lengths. */ 2160 /* XXX Handle crd_skip. */ 2161 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2162 struct mbuf *m = crp->crp_buf; 2163 uint32_t nbytes = 0; 2164 while (m != NULL) { 2165 KASSERT(m->m_len >= 0); 2166 if (m->m_len > crd->crd_len || 2167 nbytes > crd->crd_len - m->m_len) { 2168 error = EINVAL; 2169 goto fail0; 2170 } 2171 nbytes += m->m_len; 2172 m = m->m_next; 2173 } 2174 if (nbytes != crd->crd_len) { 2175 error = EINVAL; 2176 goto fail0; 2177 } 2178 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2179 struct uio *uio = crp->crp_buf; 2180 if (uio->uio_resid != crd->crd_len) { 2181 error = EINVAL; 2182 goto fail0; 2183 } 2184 } 2185 2186 /* Get a task, or fail with ERESTART if we can't. */ 2187 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 2188 PR_NOWAIT); 2189 if (task == NULL) { 2190 /* 2191 * Don't invoke crypto_done -- we are asking the 2192 * opencrypto(9) machinery to queue the request and get 2193 * back to us. 2194 */ 2195 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 2196 return ERESTART; 2197 } 2198 2199 /* Load key in, if relevant. */ 2200 klen = crd->crd_klen; 2201 if (klen) { 2202 if (crd->crd_alg == CRYPTO_AES_CTR) 2203 /* AES-CTR is special -- see IV processing below. */ 2204 klen -= 32; 2205 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 2206 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 2207 if (error) 2208 goto fail1; 2209 task->ct_flags |= TASK_KEY; 2210 } 2211 2212 /* Handle the IV, if relevant. */ 2213 ivlen = sun8i_crypto_ivlen(crd); 2214 if (ivlen) { 2215 void *iv; 2216 2217 /* 2218 * If there's an explicit IV, use it; otherwise 2219 * randomly generate one. 2220 */ 2221 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2222 iv = crd->crd_iv; 2223 } else { 2224 cprng_fast(task->ct_iv, ivlen); 2225 iv = task->ct_iv; 2226 } 2227 2228 /* 2229 * If the IV is not already present in the user's 2230 * buffer, copy it over. 2231 */ 2232 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 2233 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2234 m_copyback(crp->crp_buf, crd->crd_inject, 2235 ivlen, iv); 2236 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2237 cuio_copyback(crp->crp_buf, crd->crd_inject, 2238 ivlen, iv); 2239 } else { 2240 panic("invalid buffer type %x", 2241 crp->crp_flags); 2242 } 2243 } 2244 2245 /* 2246 * opencrypto's idea of `AES-CTR' is special. 2247 * 2248 * - The low 4 bytes of the input block are drawn from 2249 * an extra 4 bytes at the end of the key. 2250 * 2251 * - The next 8 bytes of the input block are drawn from 2252 * the opencrypto iv. 2253 * 2254 * - The high 4 bytes are the big-endian block counter, 2255 * which starts at 1 because why not. 2256 */ 2257 if (crd->crd_alg == CRYPTO_AES_CTR) { 2258 uint8_t block[16]; 2259 uint32_t blkno = 1; 2260 2261 /* Format the initial input block. */ 2262 memcpy(block, crd->crd_key + klen/8, 4); 2263 memcpy(block + 4, iv, 8); 2264 be32enc(block + 12, blkno); 2265 2266 /* Copy it into the DMA buffer. */ 2267 memcpy(task->ct_iv, block, 16); 2268 iv = task->ct_iv; 2269 ivlen = 16; 2270 } 2271 2272 /* Load the IV. */ 2273 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 2274 NULL, BUS_DMA_NOWAIT); 2275 if (error) 2276 goto fail1; 2277 task->ct_flags |= TASK_IV; 2278 } 2279 2280 /* Load the src and dst. */ 2281 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2282 struct mbuf *m = crp->crp_buf; 2283 2284 /* XXX Handle crd_skip. */ 2285 KASSERT(crd->crd_skip == 0); 2286 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 2287 BUS_DMA_NOWAIT); 2288 if (error) 2289 goto fail1; 2290 task->ct_flags |= TASK_SRC; 2291 2292 /* XXX Handle crd_skip. */ 2293 KASSERT(crd->crd_skip == 0); 2294 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 2295 BUS_DMA_NOWAIT); 2296 if (error) 2297 goto fail1; 2298 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2299 struct uio *uio = crp->crp_buf; 2300 2301 /* XXX Handle crd_skip. */ 2302 KASSERT(crd->crd_skip == 0); 2303 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 2304 BUS_DMA_NOWAIT); 2305 if (error) 2306 goto fail1; 2307 task->ct_flags |= TASK_SRC; 2308 2309 /* XXX Handle crd_skip. */ 2310 KASSERT(crd->crd_skip == 0); 2311 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 2312 BUS_DMA_NOWAIT); 2313 if (error) 2314 goto fail1; 2315 } else { 2316 panic("invalid buffer type %x", crp->crp_flags); 2317 } 2318 2319 /* Set the encryption direction. */ 2320 if (crd->crd_flags & CRD_F_ENCRYPT) 2321 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 2322 else 2323 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 2324 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 2325 2326 /* Set the method. */ 2327 switch (crd->crd_alg) { 2328 case CRYPTO_AES_CBC: 2329 case CRYPTO_AES_CTR: 2330 #ifdef CRYPTO_AES_ECB 2331 case CRYPTO_AES_ECB: 2332 #endif 2333 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2334 break; 2335 #ifdef CRYPTO_AES_XTS 2336 case CRYPTO_AES_XTS: 2337 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2338 break; 2339 #endif 2340 case CRYPTO_DES_CBC: 2341 #ifdef CRYPTO_DES_ECB 2342 case CRYPTO_DES_ECB: 2343 #endif 2344 method = SUN8I_CRYPTO_TDQC_METHOD_DES; 2345 break; 2346 case CRYPTO_3DES_CBC: 2347 #ifdef CRYPTO_3DES_ECB 2348 case CRYPTO_3DES_ECB: 2349 #endif 2350 method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 2351 break; 2352 case CRYPTO_MD5: 2353 method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 2354 break; 2355 case CRYPTO_SHA1: 2356 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 2357 break; 2358 #ifdef CRYPTO_SHA224 2359 case CRYPTO_SHA224: 2360 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 2361 break; 2362 #endif 2363 #ifdef CRYPTO_SHA256 2364 case CRYPTO_SHA256: 2365 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 2366 break; 2367 #endif 2368 case CRYPTO_SHA1_HMAC: 2369 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 2370 break; 2371 case CRYPTO_SHA2_256_HMAC: 2372 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 2373 break; 2374 default: 2375 panic("unknown algorithm %d", crd->crd_alg); 2376 } 2377 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 2378 2379 /* Set the key selector. No idea how to use the internal keys. */ 2380 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 2381 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 2382 2383 /* XXX Deal with AES_CTS_Last_Block_Flag. */ 2384 2385 /* Set the mode. */ 2386 switch (crd->crd_alg) { 2387 #ifdef CRYPTO_AES_ECB 2388 case CRYPTO_AES_ECB: 2389 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2390 break; 2391 #endif 2392 #ifdef CRYPTO_DES_ECB 2393 case CRYPTO_DES_ECB: 2394 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2395 break; 2396 #endif 2397 #ifdef CRYPTO_3DES_ECB 2398 case CRYPTO_3DES_ECB: 2399 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2400 break; 2401 #endif 2402 case CRYPTO_AES_CBC: 2403 case CRYPTO_DES_CBC: 2404 case CRYPTO_3DES_CBC: 2405 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 2406 break; 2407 case CRYPTO_AES_CTR: 2408 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 2409 break; 2410 #ifdef CRYPTO_AES_XTS 2411 case CRYPTO_AES_XTS: 2412 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 2413 break; 2414 #endif 2415 default: 2416 panic("unknown algorithm %d", crd->crd_alg); 2417 } 2418 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 2419 2420 /* Set the CTR width. */ 2421 switch (crd->crd_alg) { 2422 case CRYPTO_AES_CTR: 2423 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 2424 break; 2425 } 2426 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 2427 2428 /* Set the AES key size. */ 2429 switch (crd->crd_alg) { 2430 case CRYPTO_AES_CBC: 2431 #ifdef CRYPTO_AES_ECB 2432 case CRYPTO_AES_ECB: 2433 #endif 2434 switch (crd->crd_klen) { 2435 case 128: 2436 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2437 break; 2438 case 192: 2439 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2440 break; 2441 case 256: 2442 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2443 break; 2444 default: 2445 panic("invalid AES key size in bits: %u", 2446 crd->crd_klen); 2447 } 2448 break; 2449 case CRYPTO_AES_CTR: 2450 switch (crd->crd_klen) { 2451 case 128 + 32: 2452 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2453 break; 2454 case 192 + 32: 2455 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2456 break; 2457 case 256 + 32: 2458 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2459 break; 2460 default: 2461 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 2462 crd->crd_klen); 2463 } 2464 break; 2465 #ifdef CRYPTO_AES_XTS 2466 case CRYPTO_AES_XTS: 2467 switch (crd->crd_klen) { 2468 case 256: 2469 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2470 break; 2471 case 384: 2472 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2473 break; 2474 case 512: 2475 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2476 break; 2477 default: 2478 panic("invalid AES-XTS key size in bits: %u", 2479 crd->crd_klen); 2480 } 2481 break; 2482 #endif 2483 } 2484 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 2485 2486 /* Set up the task descriptor. */ 2487 error = sun8i_crypto_task_load(sc, task, crd->crd_len, 2488 tdqc, tdqs, tdqa); 2489 if (error) 2490 goto fail2; 2491 2492 /* Submit! */ 2493 error = sun8i_crypto_submit(sc, task); 2494 if (error) 2495 goto fail2; 2496 2497 /* Success! */ 2498 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 2499 return 0; 2500 2501 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2502 fail1: if (task->ct_flags & TASK_SRC) 2503 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2504 if (task->ct_flags & TASK_CTR) 2505 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2506 if (task->ct_flags & TASK_IV) 2507 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2508 if (task->ct_flags & TASK_KEY) 2509 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2510 sun8i_crypto_task_put(sc, task); 2511 fail0: KASSERT(error); 2512 KASSERT(error != ERESTART); 2513 crp->crp_etype = error; 2514 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2515 crypto_done(crp); 2516 return 0; 2517 } 2518 2519 /* 2520 * sun8i_crypto_callback(sc, task, cookie, error) 2521 * 2522 * Completion callback for a task submitted via opencrypto. 2523 * Release the task and pass the error on to opencrypto with 2524 * crypto_done. 2525 */ 2526 static void 2527 sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 2528 struct sun8i_crypto_task *task, void *cookie, int error) 2529 { 2530 struct cryptop *crp = cookie; 2531 struct cryptodesc *crd __diagused = crp->crp_desc; 2532 2533 KASSERT(error != ERESTART); 2534 KASSERT(crd != NULL); 2535 KASSERT(crd->crd_next == NULL); 2536 2537 /* Return the number of bytes processed. */ 2538 crp->crp_olen = error ? 0 : crp->crp_ilen; 2539 2540 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2541 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2542 if (task->ct_flags & TASK_CTR) 2543 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2544 if (task->ct_flags & TASK_IV) 2545 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2546 if (task->ct_flags & TASK_KEY) 2547 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2548 sun8i_crypto_task_put(sc, task); 2549 KASSERT(error != ERESTART); 2550 crp->crp_etype = error; 2551 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2552 crypto_done(crp); 2553 } 2554
Indexes created Sun Nov 01 00:25:34 UTC 2026