sun8i_crypto.c revision 1.25
1 /* $NetBSD: sun8i_crypto.c,v 1.25 2021/04/28 16:57:05 bad Exp $ */ 2 3 /*- 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * sun8i_crypto -- Allwinner Crypto Engine driver 34 * 35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37 * the moment, but in principle it could be wired up with opencrypto(9) 38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41 * 42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43 */ 44 45 #include <sys/cdefs.h> 46 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.25 2021/04/28 16:57:05 bad Exp $"); 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/atomic.h> 51 #include <sys/bus.h> 52 #include <sys/callout.h> 53 #include <sys/conf.h> 54 #include <sys/cprng.h> 55 #include <sys/device.h> 56 #include <sys/kernel.h> 57 #include <sys/kmem.h> 58 #include <sys/mbuf.h> 59 #include <sys/mutex.h> 60 #include <sys/rndsource.h> 61 #include <sys/sdt.h> 62 #include <sys/sysctl.h> 63 #include <sys/workqueue.h> 64 65 #include <dev/fdt/fdtvar.h> 66 67 #include <opencrypto/cryptodev.h> 68 69 #include <arm/sunxi/sun8i_crypto.h> 70 71 #define SUN8I_CRYPTO_TIMEOUT hz 72 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 73 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 74 75 struct sun8i_crypto_config { 76 u_int mod_rate; /* module clock rate */ 77 }; 78 79 /* 80 * The module clock is set to 50 MHz on H3, 300 MHz otherwise. 81 * From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c: 82 * Module clock is lower on H3 than other SoC due to some DMA 83 * timeout occurring with high value. 84 */ 85 static const struct sun8i_crypto_config sun50i_a64_crypto_config = { 86 .mod_rate = 300*1000*1000, 87 }; 88 89 static const struct sun8i_crypto_config sun50i_h5_crypto_config = { 90 .mod_rate = 300*1000*1000, 91 }; 92 93 static const struct sun8i_crypto_config sun8i_h3_crypto_config = { 94 .mod_rate = 50*1000*1000, 95 }; 96 97 struct sun8i_crypto_task; 98 99 struct sun8i_crypto_buf { 100 bus_dma_segment_t cb_seg[1]; 101 int cb_nsegs; 102 void *cb_kva; 103 }; 104 105 struct sun8i_crypto_softc { 106 device_t sc_dev; 107 bus_space_tag_t sc_bst; 108 bus_space_handle_t sc_bsh; 109 bus_dma_tag_t sc_dmat; 110 struct pool_cache *sc_taskpool; 111 112 const struct sun8i_crypto_config *sc_cfg; 113 114 kmutex_t sc_lock; 115 struct sun8i_crypto_chan { 116 struct sun8i_crypto_task *cc_task; 117 unsigned cc_starttime; 118 } sc_chan[SUN8I_CRYPTO_NCHAN]; 119 struct callout sc_timeout; 120 struct workqueue *sc_wq; 121 struct work sc_work; 122 void *sc_ih; 123 uint32_t sc_done; 124 uint32_t sc_esr; 125 bool sc_work_pending; 126 struct sun8i_crypto_rng { 127 struct sun8i_crypto_buf cr_buf; 128 struct sun8i_crypto_task *cr_task; 129 struct krndsource cr_rndsource; 130 bool cr_pending; 131 } sc_rng; 132 struct sun8i_crypto_selftest { 133 struct sun8i_crypto_buf cs_in; 134 struct sun8i_crypto_buf cs_key; 135 struct sun8i_crypto_buf cs_out; 136 struct sun8i_crypto_task *cs_task; 137 } sc_selftest; 138 struct sun8i_crypto_sysctl { 139 struct sysctllog *cy_log; 140 const struct sysctlnode *cy_root_node; 141 const struct sysctlnode *cy_trng_node; 142 } sc_sysctl; 143 struct sun8i_crypto_opencrypto { 144 uint32_t co_driverid; 145 } sc_opencrypto; 146 }; 147 148 struct sun8i_crypto_task { 149 struct sun8i_crypto_buf ct_descbuf; 150 struct sun8i_crypto_taskdesc *ct_desc; 151 struct sun8i_crypto_buf ct_ivbuf; 152 void *ct_iv; 153 struct sun8i_crypto_buf ct_ctrbuf; 154 void *ct_ctr; 155 bus_dmamap_t ct_descmap; 156 bus_dmamap_t ct_keymap; 157 bus_dmamap_t ct_ivmap; /* IV input */ 158 bus_dmamap_t ct_ctrmap; /* updated IV output */ 159 bus_dmamap_t ct_srcmap; 160 bus_dmamap_t ct_dstmap; 161 uint32_t ct_nbytes; 162 int ct_flags; 163 #define TASK_KEY __BIT(0) 164 #define TASK_IV __BIT(1) 165 #define TASK_CTR __BIT(2) 166 #define TASK_SRC __BIT(3) 167 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 168 void (*ct_callback)(struct sun8i_crypto_softc *, 169 struct sun8i_crypto_task *, void *, int); 170 void *ct_cookie; 171 }; 172 173 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 174 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 175 176 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 177 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 178 179 /* 180 * Forward declarations 181 */ 182 183 static int sun8i_crypto_match(device_t, cfdata_t, void *); 184 static void sun8i_crypto_attach(device_t, device_t, void *); 185 186 static int sun8i_crypto_task_ctor(void *, void *, int); 187 static void sun8i_crypto_task_dtor(void *, void *); 188 static struct sun8i_crypto_task * 189 sun8i_crypto_task_get(struct sun8i_crypto_softc *, 190 void (*)(struct sun8i_crypto_softc *, 191 struct sun8i_crypto_task *, void *, int), 192 void *, int); 193 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 194 struct sun8i_crypto_task *); 195 196 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 197 struct sun8i_crypto_task *, uint32_t, 198 uint32_t, uint32_t, uint32_t); 199 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 200 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 201 202 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 203 struct sun8i_crypto_task *, uint32_t); 204 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 205 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 206 207 static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 208 struct sun8i_crypto_task *); 209 210 static void sun8i_crypto_timeout(void *); 211 static int sun8i_crypto_intr(void *); 212 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 213 static void sun8i_crypto_worker(struct work *, void *); 214 static void sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 215 int); 216 217 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 218 struct sun8i_crypto_buf *, int); 219 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 220 struct sun8i_crypto_buf *); 221 222 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 223 static void sun8i_crypto_rng_get(size_t, void *); 224 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 225 struct sun8i_crypto_task *, void *, int); 226 227 static void sun8i_crypto_selftest(device_t); 228 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 229 struct sun8i_crypto_task *, void *, int); 230 231 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 232 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 233 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 234 struct sun8i_crypto_task *, void *, int); 235 236 static void sun8i_crypto_register(struct sun8i_crypto_softc *); 237 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 238 static int sun8i_crypto_newsession(void *, uint32_t *, 239 struct cryptoini *); 240 static int sun8i_crypto_freesession(void *, uint64_t); 241 static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 242 static int sun8i_crypto_process(void *, struct cryptop *, int); 243 static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 244 struct sun8i_crypto_task *, void *, int); 245 246 /* 247 * Probes 248 */ 249 250 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 251 "bus_size_t"/*reg*/, 252 "uint32_t"/*value*/); 253 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 254 "bus_size_t"/*reg*/, 255 "uint32_t"/*write*/); 256 257 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 258 "struct sun8i_crypto_task *"/*task*/); 259 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 260 "int"/*error*/); 261 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 262 "struct sun8i_crypto_task *"/*task*/); 263 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 264 "struct sun8i_crypto_task *"/*task*/); 265 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 266 "struct sun8i_crypto_task *"/*task*/); 267 268 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 269 "struct sun8i_crypto_task *"/*task*/, 270 "uint32_t"/*tdqc*/, 271 "uint32_t"/*tdqs*/, 272 "uint32_t"/*tdqa*/, 273 "struct sun8i_crypto_taskdesc *"/*desc*/, 274 "int"/*error*/); 275 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 276 "struct sun8i_crypto_task *"/*task*/, 277 "bus_addr_t"/*ds_addr*/, 278 "bus_size_t"/*ds_len*/); 279 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 280 "struct sun8i_crypto_task *"/*task*/, 281 "int"/*error*/); 282 283 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 284 "struct sun8i_crypto_softc *"/*sc*/, 285 "struct sun8i_crypto_task *"/*task*/, 286 "int"/*error*/); 287 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 288 "struct sun8i_crypto_softc *"/*sc*/, 289 "struct sun8i_crypto_task *"/*task*/, 290 "unsigned"/*chan*/); 291 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 292 "struct sun8i_crypto_softc *"/*sc*/, 293 "uint32_t"/*isr*/, 294 "uint32_t"/*esr*/); 295 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 296 "struct sun8i_crypto_softc *"/*sc*/, 297 "unsigned"/*chan*/, 298 "int"/*error*/); 299 300 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 301 "struct sun8i_crypto_softc *"/*sc*/, 302 "struct cryptop *"/*crp*/, 303 "int"/*hint*/); 304 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 305 "struct sun8i_crypto_softc *"/*sc*/, 306 "struct cryptop *"/*crp*/, 307 "int"/*hint*/); 308 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 309 "struct sun8i_crypto_softc *"/*sc*/, 310 "struct cryptop *"/*crp*/, 311 "int"/*hint*/, 312 "struct sun8i_crypto_task *"/*task*/); 313 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 314 "struct sun8i_crypto_softc *"/*sc*/, 315 "struct cryptop *"/*crp*/, 316 "int"/*error*/); 317 318 /* 319 * Register access 320 */ 321 322 static uint32_t 323 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 324 { 325 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 326 327 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 328 return v; 329 } 330 331 static void 332 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 333 { 334 335 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 336 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 337 } 338 339 /* 340 * Autoconf goo 341 */ 342 343 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 344 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 345 346 static const struct device_compatible_entry compat_data[] = { 347 { .compat = "allwinner,sun50i-a64-crypto", 348 .data = &sun50i_a64_crypto_config }, 349 { .compat = "allwinner,sun50i-h5-crypto", 350 .data = &sun50i_h5_crypto_config }, 351 { .compat = "allwinner,sun8i-h3-crypto", 352 .data = &sun8i_h3_crypto_config }, 353 DEVICE_COMPAT_EOL 354 }; 355 356 static int 357 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 358 { 359 const struct fdt_attach_args *const faa = aux; 360 361 return of_compatible_match(faa->faa_phandle, compat_data); 362 } 363 364 static void 365 sun8i_crypto_attach(device_t parent, device_t self, void *aux) 366 { 367 struct sun8i_crypto_softc *const sc = device_private(self); 368 const struct fdt_attach_args *const faa = aux; 369 bus_addr_t addr; 370 bus_size_t size; 371 const int phandle = faa->faa_phandle; 372 char intrstr[128]; 373 struct clk *clk; 374 struct fdtbus_reset *rst; 375 u_int mod_rate; 376 377 sc->sc_dev = self; 378 sc->sc_dmat = faa->faa_dmat; 379 sc->sc_bst = faa->faa_bst; 380 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 381 0, 0, 0, "sun8icry", NULL, IPL_VM, 382 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 383 sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data; 384 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM); 385 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 386 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 387 if (workqueue_create(&sc->sc_wq, device_xname(self), 388 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 389 aprint_error(": couldn't create workqueue\n"); 390 return; 391 } 392 393 /* 394 * Prime the pool with enough tasks that each channel can be 395 * busy with a task as we prepare another task for when it's 396 * done. 397 */ 398 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 399 400 /* Get and map device registers. */ 401 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 402 aprint_error(": couldn't get registers\n"); 403 return; 404 } 405 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 406 aprint_error(": couldn't map registers\n"); 407 return; 408 } 409 410 /* Get an interrupt handle. */ 411 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 412 aprint_error(": failed to decode interrupt\n"); 413 return; 414 } 415 416 /* Enable the bus clock. */ 417 if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 418 aprint_error(": couldn't enable bus clock\n"); 419 return; 420 } 421 422 /* Get the module clock and set it. */ 423 mod_rate = sc->sc_cfg->mod_rate; 424 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 425 if (clk_enable(clk) != 0) { 426 aprint_error(": couldn't enable CE clock\n"); 427 return; 428 } 429 if (clk_set_rate(clk, mod_rate) != 0) { 430 aprint_error(": couldn't set CE clock to %d MHz\n", 431 mod_rate / (1000 * 1000)); 432 return; 433 } 434 } 435 436 /* Get a reset handle if we need and try to deassert it. */ 437 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 438 if (fdtbus_reset_deassert(rst) != 0) { 439 aprint_error(": couldn't de-assert reset\n"); 440 return; 441 } 442 } 443 444 aprint_naive("\n"); 445 aprint_normal(": Crypto Engine\n"); 446 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 447 448 /* Disable and clear interrupts. */ 449 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 450 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 451 452 /* Establish an interrupt handler. */ 453 sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM, 454 FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self)); 455 if (sc->sc_ih == NULL) { 456 aprint_error_dev(self, "failed to establish interrupt on %s\n", 457 intrstr); 458 return; 459 } 460 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 461 462 /* Set up the RNG. */ 463 sun8i_crypto_rng_attach(sc); 464 465 /* Attach the sysctl. */ 466 sun8i_crypto_sysctl_attach(sc); 467 468 /* Perform self-tests. */ 469 config_interrupts(self, sun8i_crypto_selftest); 470 471 /* Register opencrypto handlers. */ 472 sun8i_crypto_register(sc); 473 } 474 475 static int 476 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 477 { 478 struct sun8i_crypto_softc *sc = cookie; 479 struct sun8i_crypto_task *task = vtask; 480 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 481 int error; 482 483 /* Create a DMA buffer for the task descriptor. */ 484 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 485 &task->ct_descbuf, dmaflags); 486 if (error) 487 goto fail0; 488 task->ct_desc = task->ct_descbuf.cb_kva; 489 490 /* Create DMA buffers for the IV and CTR. */ 491 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 492 &task->ct_ivbuf, dmaflags); 493 if (error) 494 goto fail1; 495 task->ct_iv = task->ct_ivbuf.cb_kva; 496 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 497 &task->ct_ctrbuf, dmaflags); 498 if (error) 499 goto fail2; 500 task->ct_ctr = task->ct_ctrbuf.cb_kva; 501 502 /* Create a DMA map for the task descriptor and preload it. */ 503 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 504 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 505 if (error) 506 goto fail3; 507 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 508 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 509 if (error) 510 goto fail4; 511 512 /* Create DMA maps for the key, IV, and CTR. */ 513 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 514 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 515 if (error) 516 goto fail5; 517 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 518 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 519 if (error) 520 goto fail6; 521 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 522 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 523 if (error) 524 goto fail7; 525 526 /* Create DMA maps for the src and dst scatter/gather vectors. */ 527 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 528 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 529 &task->ct_srcmap); 530 if (error) 531 goto fail8; 532 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 533 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 534 &task->ct_dstmap); 535 if (error) 536 goto fail9; 537 538 /* Success! */ 539 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 540 return 0; 541 542 fail10: __unused 543 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 544 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 545 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 546 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 547 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 548 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 549 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 550 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 551 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 552 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 553 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 554 return error; 555 } 556 557 static void 558 sun8i_crypto_task_dtor(void *cookie, void *vtask) 559 { 560 struct sun8i_crypto_softc *sc = cookie; 561 struct sun8i_crypto_task *task = vtask; 562 563 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 564 565 /* XXX Zero the bounce buffers if there are any. */ 566 567 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 568 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 569 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 570 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 571 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 572 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 573 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 574 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 575 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 576 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 577 } 578 579 /* 580 * sun8i_crypto_task_get(sc, callback, cookie, pflags) 581 * 582 * Allocate a task that will call callback(sc, task, cookie, 583 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 584 * PR_NOWAIT, may fail and return NULL. No further allocation is 585 * needed to submit the task if this succeeds (although task 586 * submission may still fail if all channels are busy). 587 */ 588 static struct sun8i_crypto_task * 589 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 590 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 591 void *, int), 592 void *cookie, int pflags) 593 { 594 struct sun8i_crypto_task *task; 595 596 /* Allocate a task, or fail if we can't. */ 597 task = pool_cache_get(sc->sc_taskpool, pflags); 598 if (task == NULL) 599 goto out; 600 601 /* Set up flags and the callback. */ 602 task->ct_flags = 0; 603 task->ct_callback = callback; 604 task->ct_cookie = cookie; 605 606 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 607 return task; 608 } 609 610 /* 611 * sun8i_crypto_task_invalid(sc, task, cookie, error) 612 * 613 * Callback for a task not currently in use, to detect errors. 614 */ 615 static void 616 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 617 struct sun8i_crypto_task *task, void *cookie, int error) 618 { 619 void (*callback)(struct sun8i_crypto_softc *, 620 struct sun8i_crypto_task *, void *, int) = cookie; 621 622 panic("task for callback %p used after free", callback); 623 } 624 625 /* 626 * sun8i_crypto_task_put(sc, task) 627 * 628 * Free a task obtained with sun8i_crypto_task_get. 629 */ 630 static void 631 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 632 struct sun8i_crypto_task *task) 633 { 634 635 SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 636 637 task->ct_cookie = task->ct_callback; 638 task->ct_callback = &sun8i_crypto_task_invalid; 639 pool_cache_put(sc->sc_taskpool, task); 640 } 641 642 /* 643 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 644 * 645 * Set up the task descriptor after the relevant DMA maps have 646 * been loaded for a transfer of nbytes. bus_dmamap_sync matches 647 * sun8i_crypto_chan_done. May fail if input is inadequately 648 * aligned. 649 * 650 * XXX Teach this to support task chains. 651 */ 652 static int 653 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 654 struct sun8i_crypto_task *task, uint32_t nbytes, 655 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 656 { 657 struct sun8i_crypto_taskdesc *desc = task->ct_desc; 658 int error; 659 660 KASSERT(tdqs == 0 || tdqa == 0); 661 KASSERT(nbytes % 4 == 0); 662 663 memset(desc, 0, sizeof(*desc)); 664 665 /* Always enable interrupt for the task. */ 666 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 667 668 desc->td_tdqc = htole32(tdqc); 669 desc->td_tdqs = htole32(tdqs); 670 desc->td_tdqa = htole32(tdqa); 671 672 if (task->ct_flags & TASK_KEY) { 673 bus_dmamap_t keymap = task->ct_keymap; 674 KASSERT(keymap->dm_nsegs == 1); 675 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 676 bus_dmamap_sync(sc->sc_dmat, keymap, 0, 677 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 678 } 679 if (task->ct_flags & TASK_IV) { 680 bus_dmamap_t ivmap = task->ct_ivmap; 681 KASSERT(ivmap->dm_nsegs == 1); 682 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 683 bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 684 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 685 } 686 if (task->ct_flags & TASK_CTR) { 687 bus_dmamap_t ctrmap = task->ct_ctrmap; 688 KASSERT(ctrmap->dm_nsegs == 1); 689 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 690 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 691 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 692 } 693 694 if (task->ct_flags & TASK_BYTES) 695 desc->td_datalen = htole32(nbytes); 696 else 697 desc->td_datalen = htole32(nbytes/4); 698 699 if (task->ct_flags & TASK_SRC) { 700 bus_dmamap_t srcmap = task->ct_srcmap; 701 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 702 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 703 nbytes); 704 if (error) 705 return error; 706 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 707 BUS_DMASYNC_PREWRITE); 708 } 709 710 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 711 nbytes); 712 if (error) 713 goto out; 714 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 715 BUS_DMASYNC_PREREAD); 716 717 task->ct_nbytes = nbytes; 718 719 /* Success! */ 720 error = 0; 721 722 out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 723 task, tdqc, tdqs, tdqa, desc, error); 724 return error; 725 } 726 727 /* 728 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 729 * 730 * Set up a task's scatter/gather vector -- src or dst -- with the 731 * given DMA map for a transfer of nbytes. May fail if input is 732 * inadequately aligned. 733 */ 734 static int 735 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 736 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 737 uint32_t nbytes __diagused) 738 { 739 uint32_t total __diagused = 0; 740 unsigned i; 741 742 /* 743 * Verify that the alignment is correct and initialize the 744 * scatter/gather vector. 745 */ 746 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 747 for (i = 0; i < map->dm_nsegs; i++) { 748 if ((map->dm_segs[i].ds_addr % 4) | 749 (map->dm_segs[i].ds_len % 4)) { 750 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 751 task, 752 map->dm_segs[i].ds_addr, 753 map->dm_segs[i].ds_len); 754 return EINVAL; 755 } 756 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 757 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 758 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 759 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 760 total += map->dm_segs[i].ds_len; 761 } 762 763 /* Set the remainder to zero. */ 764 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 765 adrlen[i].adr = 0; 766 adrlen[i].len = 0; 767 } 768 769 /* Verify the total size matches the transfer length. */ 770 KASSERT(total == nbytes); 771 772 /* Success! */ 773 return 0; 774 } 775 776 /* 777 * sun8i_crypto_task_load_trng(task, nbytes) 778 * 779 * Set up the task descriptor for a transfer of nbytes from the 780 * TRNG. 781 */ 782 static int 783 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 784 struct sun8i_crypto_task *task, uint32_t nbytes) 785 { 786 uint32_t tdqc = 0; 787 788 /* Caller must provide dst only. */ 789 KASSERT((task->ct_flags & TASK_KEY) == 0); 790 KASSERT((task->ct_flags & TASK_IV) == 0); 791 KASSERT((task->ct_flags & TASK_CTR) == 0); 792 KASSERT((task->ct_flags & TASK_SRC) == 0); 793 794 /* Set up the task descriptor queue control words. */ 795 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 796 SUN8I_CRYPTO_TDQC_METHOD); 797 798 /* Fill in the descriptor. */ 799 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 800 } 801 802 static int 803 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 804 struct sun8i_crypto_task *task, 805 uint32_t nbytes, uint32_t keysize, uint32_t dir) 806 { 807 uint32_t tdqc = 0, tdqs = 0; 808 809 /* Caller must provide key, src, and dst only. */ 810 KASSERT(task->ct_flags & TASK_KEY); 811 KASSERT((task->ct_flags & TASK_IV) == 0); 812 KASSERT((task->ct_flags & TASK_CTR) == 0); 813 KASSERT(task->ct_flags & TASK_SRC); 814 815 /* Set up the task descriptor queue control word. */ 816 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 817 SUN8I_CRYPTO_TDQC_METHOD); 818 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 819 820 #ifdef DIAGNOSTIC 821 switch (keysize) { 822 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 823 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 824 break; 825 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 826 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 827 break; 828 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 829 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 830 break; 831 } 832 #endif 833 834 /* Set up the symmetric control word. */ 835 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 836 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 837 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 838 SUN8I_CRYPTO_TDQS_OP_MODE); 839 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 840 841 /* Fill in the descriptor. */ 842 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 843 } 844 845 /* 846 * sun8i_crypto_submit(sc, task) 847 * 848 * Submit a task to the crypto engine after it has been loaded 849 * with sun8i_crypto_task_load. On success, guarantees to 850 * eventually call the task's callback. 851 */ 852 static int 853 sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 854 struct sun8i_crypto_task *task) 855 { 856 unsigned i, retries = 0; 857 uint32_t icr; 858 int error = 0; 859 860 /* One at a time at the device registers, please. */ 861 mutex_enter(&sc->sc_lock); 862 863 /* Find a channel. */ 864 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 865 if (sc->sc_chan[i].cc_task == NULL) 866 break; 867 } 868 if (i == SUN8I_CRYPTO_NCHAN) { 869 device_printf(sc->sc_dev, "no free channels\n"); 870 error = ERESTART; 871 goto out; 872 } 873 874 /* 875 * Set the channel id. Caller is responsible for setting up 876 * all other parts of the descriptor. 877 */ 878 task->ct_desc->td_cid = htole32(i); 879 880 /* 881 * Prepare to send the descriptor to the device by DMA. 882 * Matches POSTWRITE in sun8i_crypto_chan_done. 883 */ 884 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 885 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 886 887 /* Confirm we're ready to go. */ 888 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 889 device_printf(sc->sc_dev, "TLR not clear\n"); 890 error = EIO; 891 goto out; 892 } 893 894 /* Enable interrupts for this channel. */ 895 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 896 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 897 SUN8I_CRYPTO_ICR_INTR_EN); 898 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 899 900 /* Set the task descriptor queue address. */ 901 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 902 task->ct_descmap->dm_segs[0].ds_addr); 903 904 /* Notify the engine to load it, and wait for acknowledgement. */ 905 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 906 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 907 { 908 /* 909 * XXX Timeout pulled from arse. Is it even important 910 * to wait here? 911 */ 912 if (++retries == 1000) { 913 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 914 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 915 /* 916 * Hope it clears eventually; if not, we'll 917 * time out. 918 */ 919 break; 920 } 921 DELAY(1); 922 } 923 924 /* 925 * Loaded up and ready to go. Start a timer ticking if it's 926 * not already. 927 */ 928 sc->sc_chan[i].cc_task = task; 929 sc->sc_chan[i].cc_starttime = getticks(); 930 if (!callout_pending(&sc->sc_timeout)) 931 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 932 933 /* XXX Consider polling if cold to get entropy earlier. */ 934 935 out: /* Done! */ 936 if (error) 937 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 938 sc, task, error); 939 else 940 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 941 sc, task, i); 942 mutex_exit(&sc->sc_lock); 943 return error; 944 } 945 946 /* 947 * sun8i_crypto_timeout(cookie) 948 * 949 * Timeout handler. Schedules work in a thread to cancel all 950 * pending tasks that were started long enough ago we're bored of 951 * waiting for them, and reschedules another timeout unless the 952 * channels are all idle. 953 */ 954 static void 955 sun8i_crypto_timeout(void *cookie) 956 { 957 struct sun8i_crypto_softc *sc = cookie; 958 unsigned i; 959 960 mutex_enter(&sc->sc_lock); 961 962 /* Check whether there are any tasks pending. */ 963 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 964 if (sc->sc_chan[i].cc_task) 965 break; 966 } 967 if (i == SUN8I_CRYPTO_NCHAN) 968 /* None pending, so nothing to do. */ 969 goto out; 970 971 /* 972 * Schedule the worker to check for timeouts, and schedule 973 * another timeout in case we need it. 974 */ 975 sun8i_crypto_schedule_worker(sc); 976 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 977 978 out: mutex_exit(&sc->sc_lock); 979 } 980 981 /* 982 * sun8i_crypto_intr(cookie) 983 * 984 * Device interrupt handler. Find what channels have completed, 985 * whether with success or with failure, and schedule work in 986 * thread context to invoke the appropriate callbacks. 987 */ 988 static int 989 sun8i_crypto_intr(void *cookie) 990 { 991 struct sun8i_crypto_softc *sc = cookie; 992 uint32_t isr, esr; 993 994 mutex_enter(&sc->sc_lock); 995 996 /* 997 * Get and acknowledge the interrupts and error status. 998 * 999 * XXX Data sheet says the error status register is read-only, 1000 * but then advises writing 1 to bit x1xx (keysram access error 1001 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 1002 */ 1003 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 1004 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 1005 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 1006 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 1007 1008 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 1009 1010 /* Start the worker if necessary. */ 1011 sun8i_crypto_schedule_worker(sc); 1012 1013 /* Tell the worker what to do. */ 1014 sc->sc_done |= __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 1015 sc->sc_esr |= esr; 1016 1017 mutex_exit(&sc->sc_lock); 1018 1019 return __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE) != 0; 1020 } 1021 1022 /* 1023 * sun8i_crypto_schedule_worker(sc) 1024 * 1025 * Ensure that crypto engine thread context work to invoke task 1026 * callbacks will run promptly. Idempotent. 1027 */ 1028 static void 1029 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 1030 { 1031 1032 KASSERT(mutex_owned(&sc->sc_lock)); 1033 1034 /* Start the worker if necessary. */ 1035 if (!sc->sc_work_pending) { 1036 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1037 sc->sc_work_pending = true; 1038 } 1039 } 1040 1041 /* 1042 * sun8i_crypto_worker(wk, cookie) 1043 * 1044 * Thread-context worker: Invoke all task callbacks for which the 1045 * device has notified us of completion or for which we gave up 1046 * waiting. 1047 */ 1048 static void 1049 sun8i_crypto_worker(struct work *wk, void *cookie) 1050 { 1051 struct sun8i_crypto_softc *sc = cookie; 1052 uint32_t done, esr, esr_chan; 1053 unsigned i, now; 1054 int error; 1055 1056 /* 1057 * Acquire the lock. Note: We will be releasing and 1058 * reacquiring it throughout the loop. 1059 */ 1060 mutex_enter(&sc->sc_lock); 1061 1062 /* Acknowledge the work. */ 1063 KASSERT(sc->sc_work_pending); 1064 sc->sc_work_pending = false; 1065 1066 /* 1067 * Claim the done mask and error status once; we will be 1068 * releasing and reacquiring the lock for the callbacks, so 1069 * they may change. 1070 */ 1071 done = sc->sc_done; 1072 esr = sc->sc_esr; 1073 sc->sc_done = 0; 1074 sc->sc_esr = 0; 1075 1076 /* Check the time to determine what's timed out. */ 1077 now = getticks(); 1078 1079 /* Process the channels. */ 1080 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 1081 /* Check whether the channel is done. */ 1082 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 1083 /* Nope. Do we have a task to time out? */ 1084 if ((sc->sc_chan[i].cc_task != NULL) && 1085 ((now - sc->sc_chan[i].cc_starttime) >= 1086 SUN8I_CRYPTO_TIMEOUT)) 1087 sun8i_crypto_chan_done(sc, i, ETIMEDOUT); 1088 continue; 1089 } 1090 1091 /* Channel is done. Interpret the error if any. */ 1092 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1093 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1094 device_printf(sc->sc_dev, "channel %u:" 1095 " alg not supported\n", i); 1096 error = ENODEV; 1097 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1098 device_printf(sc->sc_dev, "channel %u:" 1099 " data length error\n", i); 1100 error = EIO; /* XXX */ 1101 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1102 device_printf(sc->sc_dev, "channel %u:" 1103 " key sram error\n", i); 1104 error = EIO; /* XXX */ 1105 } else if (esr_chan != 0) { 1106 error = EIO; /* generic I/O error */ 1107 } else { 1108 error = 0; 1109 } 1110 1111 /* 1112 * Notify the task of completion. May release the lock 1113 * to invoke a callback. 1114 */ 1115 sun8i_crypto_chan_done(sc, i, error); 1116 } 1117 1118 /* All one; release the lock one last time. */ 1119 mutex_exit(&sc->sc_lock); 1120 } 1121 1122 /* 1123 * sun8i_crypto_chan_done(sc, i, error) 1124 * 1125 * Notify the callback for the task on channel i, if there is one, 1126 * of the specified error, or 0 for success. 1127 */ 1128 static void 1129 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1130 { 1131 struct sun8i_crypto_task *task; 1132 uint32_t nbytes; 1133 uint32_t icr; 1134 1135 KASSERT(mutex_owned(&sc->sc_lock)); 1136 1137 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 1138 1139 /* Claim the task if there is one; bail if not. */ 1140 if ((task = sc->sc_chan[i].cc_task) == NULL) { 1141 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1142 i, error); 1143 return; 1144 } 1145 sc->sc_chan[i].cc_task = NULL; 1146 1147 /* Disable interrupts on this channel. */ 1148 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1149 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1150 SUN8I_CRYPTO_ICR_INTR_EN); 1151 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1152 1153 /* 1154 * Finished sending the descriptor to the device by DMA. 1155 * Matches PREWRITE in sun8i_crypto_task_submit. 1156 */ 1157 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1158 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1159 1160 /* 1161 * Finished with all the other bits of DMA too. Matches 1162 * sun8i_crypto_task_load. 1163 */ 1164 nbytes = task->ct_nbytes; 1165 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 1166 BUS_DMASYNC_POSTREAD); 1167 if (task->ct_flags & TASK_SRC) 1168 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 1169 BUS_DMASYNC_POSTWRITE); 1170 if (task->ct_flags & TASK_CTR) 1171 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 1172 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 1173 if (task->ct_flags & TASK_IV) 1174 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 1175 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1176 if (task->ct_flags & TASK_KEY) 1177 /* XXX Can we zero the bounce buffer if there is one? */ 1178 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 1179 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1180 1181 /* Temporarily release the lock to invoke the callback. */ 1182 mutex_exit(&sc->sc_lock); 1183 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1184 (*task->ct_callback)(sc, task, task->ct_cookie, error); 1185 mutex_enter(&sc->sc_lock); 1186 } 1187 1188 /* 1189 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 1190 * 1191 * Allocate a single-segment DMA-safe buffer and map it into KVA. 1192 * May fail if dmaflags is BUS_DMA_NOWAIT. 1193 */ 1194 static int 1195 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 1196 struct sun8i_crypto_buf *buf, int dmaflags) 1197 { 1198 int error; 1199 1200 /* Allocate a DMA-safe buffer. */ 1201 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 1202 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1203 if (error) 1204 goto fail0; 1205 1206 /* Map the buffer into kernel virtual address space. */ 1207 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 1208 size, &buf->cb_kva, dmaflags); 1209 if (error) 1210 goto fail1; 1211 1212 /* Success! */ 1213 return 0; 1214 1215 fail2: __unused 1216 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1217 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1218 fail0: return error; 1219 } 1220 1221 /* 1222 * sun8i_crypto_freebuf(sc, buf) 1223 * 1224 * Unmap buf and free it. 1225 */ 1226 static void 1227 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1228 struct sun8i_crypto_buf *buf) 1229 { 1230 1231 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1232 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1233 } 1234 1235 /* 1236 * sun8i_crypto_rng_attach(sc) 1237 * 1238 * Attach an rndsource for the crypto engine's TRNG. 1239 */ 1240 static void 1241 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1242 { 1243 device_t self = sc->sc_dev; 1244 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1245 struct sun8i_crypto_task *task; 1246 int error; 1247 1248 /* Preallocate a buffer to reuse. */ 1249 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 1250 BUS_DMA_WAITOK); 1251 if (error) { 1252 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 1253 error); 1254 goto fail0; 1255 } 1256 1257 /* Create a task to reuse. */ 1258 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 1259 rng, PR_WAITOK); 1260 if (rng->cr_task == NULL) { 1261 aprint_error_dev(self, "failed to allocate RNG task\n"); 1262 error = ENOMEM; 1263 goto fail1; 1264 } 1265 1266 /* Preload the destination map. */ 1267 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1268 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1269 if (error) { 1270 aprint_error_dev(self, "failed to load RNG buffer: %d\n", 1271 error); 1272 goto fail2; 1273 } 1274 1275 /* 1276 * Attach the rndsource. This is _not_ marked as RND_TYPE_RNG 1277 * because the output is not uniformly distributed. The bits 1278 * are heavily weighted toward 0 or 1, at different times, and 1279 * I haven't scienced a satisfactory story out of it yet. 1280 */ 1281 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1282 rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1283 RND_TYPE_UNKNOWN, 1284 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1285 1286 /* Success! */ 1287 return; 1288 1289 fail3: __unused 1290 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1291 fail2: sun8i_crypto_task_put(sc, task); 1292 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 1293 fail0: return; 1294 } 1295 1296 /* 1297 * sun8i_crypto_rng_get(nbytes, cookie) 1298 * 1299 * On-demand rndsource callback: try to gather nbytes of entropy 1300 * and enter them into the pool ASAP. 1301 */ 1302 static void 1303 sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1304 { 1305 struct sun8i_crypto_softc *sc = cookie; 1306 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1307 struct sun8i_crypto_task *task = rng->cr_task; 1308 bool pending; 1309 int error; 1310 1311 /* 1312 * Test and set the RNG-pending flag. If it's already in 1313 * progress, nothing to do here. 1314 */ 1315 mutex_enter(&sc->sc_lock); 1316 pending = rng->cr_pending; 1317 rng->cr_pending = true; 1318 mutex_exit(&sc->sc_lock); 1319 if (pending) 1320 return; 1321 1322 /* Load the task descriptor. */ 1323 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1324 if (error) 1325 goto fail; 1326 1327 /* Submit! */ 1328 error = sun8i_crypto_submit(sc, task); 1329 if (error) 1330 goto fail; 1331 1332 /* All done! */ 1333 return; 1334 1335 fail: mutex_enter(&sc->sc_lock); 1336 rng->cr_pending = false; 1337 mutex_exit(&sc->sc_lock); 1338 } 1339 1340 static void 1341 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1342 struct sun8i_crypto_task *task, void *cookie, int error) 1343 { 1344 struct sun8i_crypto_rng *rng = cookie; 1345 uint8_t *buf = rng->cr_buf.cb_kva; 1346 uint32_t entropybits; 1347 1348 KASSERT(rng == &sc->sc_rng); 1349 1350 /* If anything went wrong, forget about it. */ 1351 if (error) 1352 goto out; 1353 1354 /* 1355 * This TRNG has quite low entropy at best. But if it fails a 1356 * repeated output test, then assume it's busted. 1357 */ 1358 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1359 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1360 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1361 SUN8I_CRYPTO_RNGBYTES/2)) { 1362 device_printf(sc->sc_dev, "failed repeated output test\n"); 1363 entropybits = 0; 1364 } 1365 1366 /* 1367 * Actually we don't believe in any of the entropy until this 1368 * device has had more scrutiny. 1369 */ 1370 entropybits = 0; 1371 1372 /* Success! Enter and erase the data. */ 1373 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1374 entropybits); 1375 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1376 1377 out: /* Done -- clear the RNG-pending flag. */ 1378 mutex_enter(&sc->sc_lock); 1379 rng->cr_pending = false; 1380 mutex_exit(&sc->sc_lock); 1381 } 1382 1383 /* 1384 * Self-test 1385 */ 1386 1387 static const uint8_t selftest_input[16]; 1388 static const uint8_t selftest_key[16]; 1389 static const uint8_t selftest_output[16] = { 1390 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1391 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1392 }; 1393 1394 static void 1395 sun8i_crypto_selftest(device_t self) 1396 { 1397 const size_t keybytes = sizeof selftest_key; 1398 const size_t nbytes = sizeof selftest_input; 1399 struct sun8i_crypto_softc *sc = device_private(self); 1400 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 1401 struct sun8i_crypto_task *task; 1402 int error; 1403 1404 CTASSERT(sizeof selftest_input == sizeof selftest_output); 1405 1406 /* Allocate an input buffer. */ 1407 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 1408 BUS_DMA_WAITOK); 1409 if (error) 1410 goto fail0; 1411 1412 /* Allocate a key buffer. */ 1413 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 1414 BUS_DMA_WAITOK); 1415 if (error) 1416 goto fail1; 1417 1418 /* Allocate an output buffer. */ 1419 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 1420 BUS_DMA_WAITOK); 1421 if (error) 1422 goto fail2; 1423 1424 /* Allocate a task descriptor. */ 1425 task = selftest->cs_task = sun8i_crypto_task_get(sc, 1426 sun8i_crypto_selftest_done, selftest, PR_WAITOK); 1427 if (selftest->cs_task == NULL) { 1428 error = ENOMEM; 1429 goto fail3; 1430 } 1431 1432 /* Copy the input and key into their buffers. */ 1433 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 1434 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1435 1436 /* Load the key, src, and dst for DMA transfers. */ 1437 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 1438 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1439 if (error) 1440 goto fail4; 1441 task->ct_flags |= TASK_KEY; 1442 1443 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 1444 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1445 if (error) 1446 goto fail5; 1447 task->ct_flags |= TASK_SRC; 1448 1449 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1450 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1451 if (error) 1452 goto fail6; 1453 1454 /* Set up the task descriptor. */ 1455 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 1456 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 1457 if (error) 1458 goto fail7; 1459 1460 /* Submit! */ 1461 error = sun8i_crypto_submit(sc, task); 1462 if (error) 1463 goto fail7; 1464 1465 device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1466 1467 /* Success! */ 1468 return; 1469 1470 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1471 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1472 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1473 fail4: sun8i_crypto_task_put(sc, task); 1474 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1475 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1476 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1477 fail0: aprint_error_dev(self, "failed to run self-test, error=%d\n", error); 1478 } 1479 1480 static bool 1481 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1482 size_t n, const void *expected, const void *actual) 1483 { 1484 const uint8_t *e = expected; 1485 const uint8_t *a = actual; 1486 size_t i; 1487 1488 if (memcmp(e, a, n) == 0) 1489 return true; 1490 1491 device_printf(sc->sc_dev, "self-test: %s\n", title); 1492 printf("expected: "); 1493 for (i = 0; i < n; i++) 1494 printf("%02hhx", e[i]); 1495 printf("\n"); 1496 printf("actual: "); 1497 for (i = 0; i < n; i++) 1498 printf("%02hhx", a[i]); 1499 printf("\n"); 1500 return false; 1501 } 1502 1503 static void 1504 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1505 struct sun8i_crypto_task *task, void *cookie, int error) 1506 { 1507 const size_t keybytes = sizeof selftest_key; 1508 const size_t nbytes = sizeof selftest_input; 1509 struct sun8i_crypto_selftest *selftest = cookie; 1510 bool ok = true; 1511 1512 KASSERT(selftest == &sc->sc_selftest); 1513 1514 /* If anything went wrong, fail now. */ 1515 if (error) { 1516 device_printf(sc->sc_dev, "self-test error=%d\n", error); 1517 goto out; 1518 } 1519 1520 /* 1521 * Verify the input and key weren't clobbered, and verify the 1522 * output matches what we expect. 1523 */ 1524 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 1525 selftest_input, selftest->cs_in.cb_kva); 1526 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 1527 selftest_key, selftest->cs_key.cb_kva); 1528 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 1529 selftest_output, selftest->cs_out.cb_kva); 1530 1531 /* XXX Disable the RNG and other stuff if this fails... */ 1532 if (ok) 1533 device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1534 1535 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1536 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1537 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1538 sun8i_crypto_task_put(sc, task); 1539 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1540 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1541 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1542 } 1543 1544 /* 1545 * Sysctl for testing 1546 */ 1547 1548 struct sun8i_crypto_userreq { 1549 kmutex_t cu_lock; 1550 kcondvar_t cu_cv; 1551 size_t cu_size; 1552 struct sun8i_crypto_buf cu_buf; 1553 struct sun8i_crypto_task *cu_task; 1554 int cu_error; 1555 bool cu_done; 1556 bool cu_cancel; 1557 }; 1558 1559 static void 1560 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1561 { 1562 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1563 int error; 1564 1565 /* hw.sun8icryptoN (node) */ 1566 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1567 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1568 SYSCTL_DESCR("sun8i crypto engine knobs"), 1569 NULL, 0, NULL, 0, 1570 CTL_HW, CTL_CREATE, CTL_EOL); 1571 if (error) { 1572 aprint_error_dev(sc->sc_dev, 1573 "failed to set up sysctl hw.%s: %d\n", 1574 device_xname(sc->sc_dev), error); 1575 return; 1576 } 1577 1578 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1579 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1580 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1581 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1582 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1583 if (error) { 1584 aprint_error_dev(sc->sc_dev, 1585 "failed to set up sysctl hw.%s.rng: %d\n", 1586 device_xname(sc->sc_dev), error); 1587 return; 1588 } 1589 } 1590 1591 static int 1592 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1593 { 1594 struct sysctlnode node = *rnode; 1595 struct sun8i_crypto_softc *sc = node.sysctl_data; 1596 struct sun8i_crypto_userreq *req; 1597 struct sun8i_crypto_task *task; 1598 size_t size; 1599 int error; 1600 1601 /* If oldp == NULL, the caller wants to learn the size. */ 1602 if (oldp == NULL) { 1603 *oldlenp = 4096; 1604 return 0; 1605 } 1606 1607 /* Truncate to 4096 bytes. */ 1608 size = MIN(4096, *oldlenp); 1609 if (size == 0) 1610 return 0; /* nothing to do */ 1611 1612 /* Allocate a request context. */ 1613 req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1614 if (req == NULL) 1615 return ENOMEM; 1616 1617 /* Initialize the request context. */ 1618 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1619 cv_init(&req->cu_cv, "sun8isy"); 1620 req->cu_size = size; 1621 req->cu_error = EIO; 1622 req->cu_done = false; 1623 req->cu_cancel = false; 1624 1625 /* Allocate a buffer for the RNG output. */ 1626 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1627 if (error) 1628 goto out0; 1629 1630 /* Allocate a task. */ 1631 task = req->cu_task = sun8i_crypto_task_get(sc, 1632 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 1633 if (task == NULL) { 1634 error = ENOMEM; 1635 goto out1; 1636 } 1637 1638 /* Set the task up for TRNG to our buffer. */ 1639 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1640 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1641 if (error) 1642 goto out2; 1643 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1644 if (error) 1645 goto out3; 1646 1647 /* Submit! */ 1648 error = sun8i_crypto_submit(sc, task); 1649 if (error) { 1650 /* Make sure we don't restart the syscall -- just fail. */ 1651 if (error == ERESTART) 1652 error = EBUSY; 1653 goto out3; 1654 } 1655 1656 /* Wait for the request to complete. */ 1657 mutex_enter(&req->cu_lock); 1658 while (!req->cu_done) { 1659 error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1660 if (error) { 1661 /* 1662 * If we finished while waiting to acquire the 1663 * lock, ignore the error and just return now. 1664 * Otherwise, notify the callback that it has 1665 * to clean up after us. 1666 */ 1667 if (req->cu_done) 1668 error = 0; 1669 else 1670 req->cu_cancel = true; 1671 break; 1672 } 1673 } 1674 mutex_exit(&req->cu_lock); 1675 1676 /* 1677 * Return early on error from cv_wait_sig, which means 1678 * interruption; the callback will clean up instead. 1679 */ 1680 if (error) 1681 return error; 1682 1683 /* Check for error from the device. */ 1684 error = req->cu_error; 1685 if (error) 1686 goto out3; 1687 1688 /* Copy out the data. */ 1689 node.sysctl_data = req->cu_buf.cb_kva; 1690 node.sysctl_size = size; 1691 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1692 1693 /* Clear the buffer. */ 1694 explicit_memset(req->cu_buf.cb_kva, 0, size); 1695 1696 /* Clean up. */ 1697 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1698 out2: sun8i_crypto_task_put(sc, task); 1699 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1700 out0: cv_destroy(&req->cu_cv); 1701 mutex_destroy(&req->cu_lock); 1702 kmem_free(req, sizeof(*req)); 1703 return error; 1704 } 1705 1706 static void 1707 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1708 struct sun8i_crypto_task *task, void *cookie, int error) 1709 { 1710 struct sun8i_crypto_userreq *req = cookie; 1711 bool cancel; 1712 1713 /* 1714 * Notify the waiting thread of the error, and find out whether 1715 * that thread cancelled. 1716 */ 1717 mutex_enter(&req->cu_lock); 1718 cancel = req->cu_cancel; 1719 req->cu_error = error; 1720 req->cu_done = true; 1721 cv_broadcast(&req->cu_cv); 1722 mutex_exit(&req->cu_lock); 1723 1724 /* 1725 * If it wasn't cancelled, we're done -- the main thread will 1726 * clean up after itself. 1727 */ 1728 if (!cancel) 1729 return; 1730 1731 /* Clean up after the main thread cancelled. */ 1732 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1733 sun8i_crypto_task_put(sc, task); 1734 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1735 cv_destroy(&req->cu_cv); 1736 mutex_destroy(&req->cu_lock); 1737 kmem_free(req, sizeof(*req)); 1738 } 1739 1740 /* 1741 * sun8i_crypto_register(sc) 1742 * 1743 * Register opencrypto algorithms supported by the crypto engine. 1744 */ 1745 static void 1746 sun8i_crypto_register(struct sun8i_crypto_softc *sc) 1747 { 1748 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 1749 1750 co->co_driverid = crypto_get_driverid(0); 1751 if (co->co_driverid == (uint32_t)-1) { 1752 aprint_error_dev(sc->sc_dev, 1753 "failed to register crypto driver\n"); 1754 return; 1755 } 1756 1757 sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 1758 sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 1759 #ifdef CRYPTO_AES_ECB 1760 sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 1761 #endif 1762 #ifdef CRYPTO_AES_XTS 1763 sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 1764 #endif 1765 #ifdef CRYPTO_DES_CBC 1766 sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 1767 #endif 1768 #ifdef CRYPTO_DES_ECB 1769 sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 1770 #endif 1771 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 1772 #ifdef CRYPTO_3DES_ECB 1773 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 1774 #endif 1775 1776 sun8i_crypto_register1(sc, CRYPTO_MD5); 1777 sun8i_crypto_register1(sc, CRYPTO_SHA1); 1778 #ifdef CRYPTO_SHA224 1779 sun8i_crypto_register1(sc, CRYPTO_SHA224); 1780 #endif 1781 #ifdef CRYPTO_SHA256 1782 sun8i_crypto_register1(sc, CRYPTO_SHA256); 1783 #endif 1784 1785 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 1786 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 1787 1788 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 1789 } 1790 1791 /* 1792 * sun8i_crypto_register1(sc, alg) 1793 * 1794 * Register support for one algorithm alg using 1795 * sun8i_crypto_newsession/freesession/process. 1796 */ 1797 static void 1798 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 1799 { 1800 1801 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 1802 sun8i_crypto_newsession, 1803 sun8i_crypto_freesession, 1804 sun8i_crypto_process, 1805 sc); 1806 } 1807 1808 /* 1809 * sun8i_crypto_newsession(cookie, sidp, cri) 1810 * 1811 * Called by opencrypto to allocate a new session. We don't keep 1812 * track of sessions, since there are no persistent keys in the 1813 * hardware that we take advantage of, so this only validates the 1814 * crypto operations and returns a zero session id. 1815 */ 1816 static int 1817 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 1818 { 1819 1820 /* No composition of operations is supported here. */ 1821 if (cri->cri_next) 1822 return EINVAL; 1823 1824 /* 1825 * No variation of rounds is supported here. (XXX Unused and 1826 * unimplemented in opencrypto(9) altogether? 1827 */ 1828 if (cri->cri_rnd) 1829 return EINVAL; 1830 1831 /* 1832 * Validate per-algorithm key length. 1833 * 1834 * XXX Does opencrypto(9) do this internally? 1835 */ 1836 switch (cri->cri_alg) { 1837 case CRYPTO_MD5: 1838 case CRYPTO_SHA1: 1839 #ifdef CRYPTO_SHA224 1840 case CRYPTO_SHA224: 1841 #endif 1842 #ifdef CRYPTO_SHA256 1843 case CRYPTO_SHA256: 1844 #endif 1845 if (cri->cri_klen) 1846 return EINVAL; 1847 break; 1848 case CRYPTO_AES_CBC: 1849 #ifdef CRYPTO_AES_ECB 1850 case CRYPTO_AES_ECB: 1851 #endif 1852 switch (cri->cri_klen) { 1853 case 128: 1854 case 192: 1855 case 256: 1856 break; 1857 default: 1858 return EINVAL; 1859 } 1860 break; 1861 case CRYPTO_AES_CTR: 1862 /* 1863 * opencrypto `AES-CTR' takes four bytes of the input 1864 * block as the last four bytes of the key, for reasons 1865 * that are not entirely clear. 1866 */ 1867 switch (cri->cri_klen) { 1868 case 128 + 32: 1869 case 192 + 32: 1870 case 256 + 32: 1871 break; 1872 default: 1873 return EINVAL; 1874 } 1875 break; 1876 #ifdef CRYPTO_AES_XTS 1877 case CRYPTO_AES_XTS: 1878 switch (cri->cri_klen) { 1879 case 256: 1880 case 384: 1881 case 512: 1882 break; 1883 default: 1884 return EINVAL; 1885 } 1886 break; 1887 #endif 1888 case CRYPTO_DES_CBC: 1889 #ifdef CRYPTO_DES_ECB 1890 case CRYPTO_DES_ECB: 1891 #endif 1892 switch (cri->cri_klen) { 1893 case 64: 1894 break; 1895 default: 1896 return EINVAL; 1897 } 1898 break; 1899 case CRYPTO_3DES_CBC: 1900 #ifdef CRYPTO_3DES_ECB 1901 case CRYPTO_3DES_ECB: 1902 #endif 1903 switch (cri->cri_klen) { 1904 case 192: 1905 break; 1906 default: 1907 return EINVAL; 1908 } 1909 break; 1910 case CRYPTO_SHA1_HMAC: 1911 /* 1912 * XXX Unclear what the length limit is, but since HMAC 1913 * behaves qualitatively different for a key of at 1914 * least the full block size -- and is generally best 1915 * to use with half the block size -- let's limit it to 1916 * one block. 1917 */ 1918 if (cri->cri_klen % 8) 1919 return EINVAL; 1920 if (cri->cri_klen > 512) 1921 return EINVAL; 1922 break; 1923 case CRYPTO_SHA2_256_HMAC: 1924 if (cri->cri_klen % 8) 1925 return EINVAL; 1926 if (cri->cri_klen > 512) 1927 return EINVAL; 1928 break; 1929 default: 1930 panic("unsupported algorithm %d", cri->cri_alg); 1931 } 1932 1933 KASSERT(cri->cri_klen % 8 == 0); 1934 1935 /* Success! */ 1936 *sidp = 1; 1937 return 0; 1938 } 1939 1940 /* 1941 * sun8i_crypto_freesession(cookie, dsid) 1942 * 1943 * Called by opencrypto to free a session. We don't keep track of 1944 * sessions, since there are no persistent keys in the hardware 1945 * that we take advantage of, so this is a no-op. 1946 * 1947 * Note: dsid is actually a 64-bit quantity containing both the 1948 * driver id in the high half and the session id in the low half. 1949 */ 1950 static int 1951 sun8i_crypto_freesession(void *cookie, uint64_t dsid) 1952 { 1953 1954 KASSERT((dsid & 0xffffffff) == 1); 1955 1956 /* Success! */ 1957 return 0; 1958 } 1959 1960 /* 1961 * sun8i_crypto_ivlen(crd) 1962 * 1963 * Return the crypto engine's notion of `IV length', in bytes, for 1964 * an opencrypto operation. 1965 */ 1966 static u_int 1967 sun8i_crypto_ivlen(const struct cryptodesc *crd) 1968 { 1969 1970 switch (crd->crd_alg) { 1971 case CRYPTO_AES_CBC: 1972 return 16; 1973 #ifdef CRYPTO_AES_XTS 1974 case CRYPTO_AES_XTS: 1975 return 16; 1976 #endif 1977 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 1978 return 8; 1979 #ifdef CRYPTO_DES_CBC 1980 case CRYPTO_DES_CBC: 1981 return 8; 1982 #endif 1983 case CRYPTO_3DES_CBC: 1984 return 8; 1985 case CRYPTO_MD5: 1986 return 16; 1987 #ifdef CRYPTO_SHA224 1988 case CRYPTO_SHA224: 1989 return 32; 1990 #endif 1991 #ifdef CRYPTO_SHA256 1992 case CRYPTO_SHA256: 1993 return 32; 1994 #endif 1995 case CRYPTO_SHA1_HMAC: 1996 return 20; 1997 case CRYPTO_SHA2_256_HMAC: 1998 return 32; 1999 default: 2000 return 0; 2001 } 2002 } 2003 2004 /* 2005 * sun8i_crypto_process(cookie, crp, hint) 2006 * 2007 * Main opencrypto processing dispatch. 2008 */ 2009 static int 2010 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 2011 { 2012 struct sun8i_crypto_softc *sc = cookie; 2013 struct sun8i_crypto_task *task; 2014 struct cryptodesc *crd = crp->crp_desc; 2015 unsigned klen, ivlen; 2016 uint32_t tdqc = 0, tdqs = 0; 2017 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 2018 const uint32_t tdqa = 0; 2019 int error; 2020 2021 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 2022 2023 /* Reject compositions -- we do not handle them. */ 2024 if (crd->crd_next != NULL) { 2025 error = EOPNOTSUPP; 2026 goto fail0; 2027 } 2028 2029 /* Reject transfers with nonsense skip. */ 2030 if (crd->crd_skip < 0) { 2031 error = EINVAL; 2032 goto fail0; 2033 } 2034 2035 /* 2036 * Actually just reject any nonzero skip, because it requires 2037 * DMA segment bookkeeping that we don't do yet. 2038 */ 2039 if (crd->crd_skip) { 2040 error = EOPNOTSUPP; 2041 goto fail0; 2042 } 2043 2044 /* Reject large transfers. */ 2045 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 2046 error = EFBIG; 2047 goto fail0; 2048 } 2049 2050 /* Reject nonsense, unaligned, or mismatched lengths. */ 2051 if (crd->crd_len < 0 || 2052 crd->crd_len % 4 || 2053 crd->crd_len != crp->crp_ilen) { 2054 error = EINVAL; 2055 goto fail0; 2056 } 2057 2058 /* Reject mismatched buffer lengths. */ 2059 /* XXX Handle crd_skip. */ 2060 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2061 struct mbuf *m = crp->crp_buf; 2062 uint32_t nbytes = 0; 2063 while (m != NULL) { 2064 KASSERT(m->m_len >= 0); 2065 if (m->m_len > crd->crd_len || 2066 nbytes > crd->crd_len - m->m_len) { 2067 error = EINVAL; 2068 goto fail0; 2069 } 2070 nbytes += m->m_len; 2071 m = m->m_next; 2072 } 2073 if (nbytes != crd->crd_len) { 2074 error = EINVAL; 2075 goto fail0; 2076 } 2077 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2078 struct uio *uio = crp->crp_buf; 2079 if (uio->uio_resid != crd->crd_len) { 2080 error = EINVAL; 2081 goto fail0; 2082 } 2083 } 2084 2085 /* Get a task, or fail with ERESTART if we can't. */ 2086 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 2087 PR_NOWAIT); 2088 if (task == NULL) { 2089 /* 2090 * Don't invoke crypto_done -- we are asking the 2091 * opencrypto(9) machinery to queue the request and get 2092 * back to us. 2093 */ 2094 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 2095 return ERESTART; 2096 } 2097 2098 /* Load key in, if relevant. */ 2099 klen = crd->crd_klen; 2100 if (klen) { 2101 if (crd->crd_alg == CRYPTO_AES_CTR) 2102 /* AES-CTR is special -- see IV processing below. */ 2103 klen -= 32; 2104 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 2105 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 2106 if (error) 2107 goto fail1; 2108 task->ct_flags |= TASK_KEY; 2109 } 2110 2111 /* Handle the IV, if relevant. */ 2112 ivlen = sun8i_crypto_ivlen(crd); 2113 if (ivlen) { 2114 void *iv; 2115 2116 /* 2117 * If there's an explicit IV, use it; otherwise 2118 * randomly generate one. 2119 */ 2120 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2121 iv = crd->crd_iv; 2122 } else { 2123 cprng_fast(task->ct_iv, ivlen); 2124 iv = task->ct_iv; 2125 } 2126 2127 /* 2128 * If the IV is not already present in the user's 2129 * buffer, copy it over. 2130 */ 2131 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 2132 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2133 m_copyback(crp->crp_buf, crd->crd_inject, 2134 ivlen, iv); 2135 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2136 cuio_copyback(crp->crp_buf, crd->crd_inject, 2137 ivlen, iv); 2138 } else { 2139 panic("invalid buffer type %x", 2140 crp->crp_flags); 2141 } 2142 } 2143 2144 /* 2145 * opencrypto's idea of `AES-CTR' is special. 2146 * 2147 * - The low 4 bytes of the input block are drawn from 2148 * an extra 4 bytes at the end of the key. 2149 * 2150 * - The next 8 bytes of the input block are drawn from 2151 * the opencrypto iv. 2152 * 2153 * - The high 4 bytes are the big-endian block counter, 2154 * which starts at 1 because why not. 2155 */ 2156 if (crd->crd_alg == CRYPTO_AES_CTR) { 2157 uint8_t block[16]; 2158 uint32_t blkno = 1; 2159 2160 /* Format the initial input block. */ 2161 memcpy(block, crd->crd_key + klen/8, 4); 2162 memcpy(block + 4, iv, 8); 2163 be32enc(block + 12, blkno); 2164 2165 /* Copy it into the DMA buffer. */ 2166 memcpy(task->ct_iv, block, 16); 2167 iv = task->ct_iv; 2168 ivlen = 16; 2169 } 2170 2171 /* Load the IV. */ 2172 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 2173 NULL, BUS_DMA_NOWAIT); 2174 if (error) 2175 goto fail1; 2176 task->ct_flags |= TASK_IV; 2177 } 2178 2179 /* Load the src and dst. */ 2180 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2181 struct mbuf *m = crp->crp_buf; 2182 2183 /* XXX Handle crd_skip. */ 2184 KASSERT(crd->crd_skip == 0); 2185 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 2186 BUS_DMA_NOWAIT); 2187 if (error) 2188 goto fail1; 2189 task->ct_flags |= TASK_SRC; 2190 2191 /* XXX Handle crd_skip. */ 2192 KASSERT(crd->crd_skip == 0); 2193 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 2194 BUS_DMA_NOWAIT); 2195 if (error) 2196 goto fail1; 2197 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2198 struct uio *uio = crp->crp_buf; 2199 2200 /* XXX Handle crd_skip. */ 2201 KASSERT(crd->crd_skip == 0); 2202 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 2203 BUS_DMA_NOWAIT); 2204 if (error) 2205 goto fail1; 2206 task->ct_flags |= TASK_SRC; 2207 2208 /* XXX Handle crd_skip. */ 2209 KASSERT(crd->crd_skip == 0); 2210 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 2211 BUS_DMA_NOWAIT); 2212 if (error) 2213 goto fail1; 2214 } else { 2215 panic("invalid buffer type %x", crp->crp_flags); 2216 } 2217 2218 /* Set the encryption direction. */ 2219 if (crd->crd_flags & CRD_F_ENCRYPT) 2220 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 2221 else 2222 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 2223 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 2224 2225 /* Set the method. */ 2226 switch (crd->crd_alg) { 2227 case CRYPTO_AES_CBC: 2228 case CRYPTO_AES_CTR: 2229 #ifdef CRYPTO_AES_ECB 2230 case CRYPTO_AES_ECB: 2231 #endif 2232 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2233 break; 2234 #ifdef CRYPTO_AES_XTS 2235 case CRYPTO_AES_XTS: 2236 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2237 break; 2238 #endif 2239 case CRYPTO_DES_CBC: 2240 #ifdef CRYPTO_DES_ECB 2241 case CRYPTO_DES_ECB: 2242 #endif 2243 method = SUN8I_CRYPTO_TDQC_METHOD_DES; 2244 break; 2245 case CRYPTO_3DES_CBC: 2246 #ifdef CRYPTO_3DES_ECB 2247 case CRYPTO_3DES_ECB: 2248 #endif 2249 method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 2250 break; 2251 case CRYPTO_MD5: 2252 method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 2253 break; 2254 case CRYPTO_SHA1: 2255 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 2256 break; 2257 #ifdef CRYPTO_SHA224 2258 case CRYPTO_SHA224: 2259 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 2260 break; 2261 #endif 2262 #ifdef CRYPTO_SHA256 2263 case CRYPTO_SHA256: 2264 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 2265 break; 2266 #endif 2267 case CRYPTO_SHA1_HMAC: 2268 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 2269 break; 2270 case CRYPTO_SHA2_256_HMAC: 2271 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 2272 break; 2273 default: 2274 panic("unknown algorithm %d", crd->crd_alg); 2275 } 2276 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 2277 2278 /* Set the key selector. No idea how to use the internal keys. */ 2279 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 2280 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 2281 2282 /* XXX Deal with AES_CTS_Last_Block_Flag. */ 2283 2284 /* Set the mode. */ 2285 switch (crd->crd_alg) { 2286 #ifdef CRYPTO_AES_ECB 2287 case CRYPTO_AES_ECB: 2288 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2289 break; 2290 #endif 2291 #ifdef CRYPTO_DES_ECB 2292 case CRYPTO_DES_ECB: 2293 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2294 break; 2295 #endif 2296 #ifdef CRYPTO_3DES_ECB 2297 case CRYPTO_3DES_ECB: 2298 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2299 break; 2300 #endif 2301 case CRYPTO_AES_CBC: 2302 case CRYPTO_DES_CBC: 2303 case CRYPTO_3DES_CBC: 2304 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 2305 break; 2306 case CRYPTO_AES_CTR: 2307 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 2308 break; 2309 #ifdef CRYPTO_AES_XTS 2310 case CRYPTO_AES_XTS: 2311 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 2312 break; 2313 #endif 2314 default: 2315 panic("unknown algorithm %d", crd->crd_alg); 2316 } 2317 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 2318 2319 /* Set the CTR width. */ 2320 switch (crd->crd_alg) { 2321 case CRYPTO_AES_CTR: 2322 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 2323 break; 2324 } 2325 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 2326 2327 /* Set the AES key size. */ 2328 switch (crd->crd_alg) { 2329 case CRYPTO_AES_CBC: 2330 #ifdef CRYPTO_AES_ECB 2331 case CRYPTO_AES_ECB: 2332 #endif 2333 switch (crd->crd_klen) { 2334 case 128: 2335 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2336 break; 2337 case 192: 2338 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2339 break; 2340 case 256: 2341 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2342 break; 2343 default: 2344 panic("invalid AES key size in bits: %u", 2345 crd->crd_klen); 2346 } 2347 break; 2348 case CRYPTO_AES_CTR: 2349 switch (crd->crd_klen) { 2350 case 128 + 32: 2351 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2352 break; 2353 case 192 + 32: 2354 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2355 break; 2356 case 256 + 32: 2357 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2358 break; 2359 default: 2360 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 2361 crd->crd_klen); 2362 } 2363 break; 2364 #ifdef CRYPTO_AES_XTS 2365 case CRYPTO_AES_XTS: 2366 switch (crd->crd_klen) { 2367 case 256: 2368 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2369 break; 2370 case 384: 2371 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2372 break; 2373 case 512: 2374 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2375 break; 2376 default: 2377 panic("invalid AES-XTS key size in bits: %u", 2378 crd->crd_klen); 2379 } 2380 break; 2381 #endif 2382 } 2383 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 2384 2385 /* Set up the task descriptor. */ 2386 error = sun8i_crypto_task_load(sc, task, crd->crd_len, 2387 tdqc, tdqs, tdqa); 2388 if (error) 2389 goto fail2; 2390 2391 /* Submit! */ 2392 error = sun8i_crypto_submit(sc, task); 2393 if (error) 2394 goto fail2; 2395 2396 /* Success! */ 2397 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 2398 return 0; 2399 2400 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2401 fail1: if (task->ct_flags & TASK_SRC) 2402 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2403 if (task->ct_flags & TASK_CTR) 2404 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2405 if (task->ct_flags & TASK_IV) 2406 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2407 if (task->ct_flags & TASK_KEY) 2408 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2409 sun8i_crypto_task_put(sc, task); 2410 fail0: KASSERT(error); 2411 KASSERT(error != ERESTART); 2412 crp->crp_etype = error; 2413 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2414 crypto_done(crp); 2415 return 0; 2416 } 2417 2418 /* 2419 * sun8i_crypto_callback(sc, task, cookie, error) 2420 * 2421 * Completion callback for a task submitted via opencrypto. 2422 * Release the task and pass the error on to opencrypto with 2423 * crypto_done. 2424 */ 2425 static void 2426 sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 2427 struct sun8i_crypto_task *task, void *cookie, int error) 2428 { 2429 struct cryptop *crp = cookie; 2430 struct cryptodesc *crd __diagused = crp->crp_desc; 2431 2432 KASSERT(error != ERESTART); 2433 KASSERT(crd != NULL); 2434 KASSERT(crd->crd_next == NULL); 2435 2436 /* Return the number of bytes processed. */ 2437 crp->crp_olen = error ? 0 : crp->crp_ilen; 2438 2439 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2440 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2441 if (task->ct_flags & TASK_CTR) 2442 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2443 if (task->ct_flags & TASK_IV) 2444 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2445 if (task->ct_flags & TASK_KEY) 2446 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2447 sun8i_crypto_task_put(sc, task); 2448 KASSERT(error != ERESTART); 2449 crp->crp_etype = error; 2450 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2451 crypto_done(crp); 2452 } 2453
Indexes created Sun Sep 28 16:09:52 GMT 2025