sun8i_crypto.c revision 1.26
1 /* $NetBSD: sun8i_crypto.c,v 1.26 2021/08/07 15:41:00 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * sun8i_crypto -- Allwinner Crypto Engine driver 34 * 35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37 * the moment, but in principle it could be wired up with opencrypto(9) 38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41 * 42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43 */ 44 45 #include <sys/cdefs.h> 46 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.26 2021/08/07 15:41:00 riastradh Exp $"); 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/atomic.h> 51 #include <sys/bus.h> 52 #include <sys/callout.h> 53 #include <sys/conf.h> 54 #include <sys/cprng.h> 55 #include <sys/device.h> 56 #include <sys/kernel.h> 57 #include <sys/kmem.h> 58 #include <sys/mbuf.h> 59 #include <sys/mutex.h> 60 #include <sys/rndsource.h> 61 #include <sys/sdt.h> 62 #include <sys/sysctl.h> 63 #include <sys/workqueue.h> 64 65 #include <dev/fdt/fdtvar.h> 66 67 #include <opencrypto/cryptodev.h> 68 69 #include <arm/sunxi/sun8i_crypto.h> 70 71 #define SUN8I_CRYPTO_TIMEOUT hz 72 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 73 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 74 75 struct sun8i_crypto_config { 76 u_int mod_rate; /* module clock rate */ 77 }; 78 79 /* 80 * The module clock is set to 50 MHz on H3, 300 MHz otherwise. 81 * From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c: 82 * Module clock is lower on H3 than other SoC due to some DMA 83 * timeout occurring with high value. 84 */ 85 static const struct sun8i_crypto_config sun50i_a64_crypto_config = { 86 .mod_rate = 300*1000*1000, 87 }; 88 89 static const struct sun8i_crypto_config sun50i_h5_crypto_config = { 90 .mod_rate = 300*1000*1000, 91 }; 92 93 static const struct sun8i_crypto_config sun8i_h3_crypto_config = { 94 .mod_rate = 50*1000*1000, 95 }; 96 97 struct sun8i_crypto_task; 98 99 struct sun8i_crypto_buf { 100 bus_dma_segment_t cb_seg[1]; 101 int cb_nsegs; 102 void *cb_kva; 103 }; 104 105 struct sun8i_crypto_softc { 106 device_t sc_dev; 107 bus_space_tag_t sc_bst; 108 bus_space_handle_t sc_bsh; 109 bus_dma_tag_t sc_dmat; 110 struct pool_cache *sc_taskpool; 111 112 const struct sun8i_crypto_config *sc_cfg; 113 114 kmutex_t sc_lock; 115 struct sun8i_crypto_chan { 116 struct sun8i_crypto_task *cc_task; 117 unsigned cc_starttime; 118 } sc_chan[SUN8I_CRYPTO_NCHAN]; 119 struct callout sc_timeout; 120 struct workqueue *sc_wq; 121 struct work sc_work; 122 void *sc_ih; 123 uint32_t sc_done; 124 uint32_t sc_esr; 125 bool sc_work_pending; 126 struct sun8i_crypto_rng { 127 struct sun8i_crypto_buf cr_buf; 128 struct sun8i_crypto_task *cr_task; 129 struct krndsource cr_rndsource; 130 bool cr_pending; 131 } sc_rng; 132 struct sun8i_crypto_selftest { 133 struct sun8i_crypto_buf cs_in; 134 struct sun8i_crypto_buf cs_key; 135 struct sun8i_crypto_buf cs_out; 136 struct sun8i_crypto_task *cs_task; 137 } sc_selftest; 138 struct sun8i_crypto_sysctl { 139 struct sysctllog *cy_log; 140 const struct sysctlnode *cy_root_node; 141 const struct sysctlnode *cy_trng_node; 142 } sc_sysctl; 143 struct sun8i_crypto_opencrypto { 144 uint32_t co_driverid; 145 } sc_opencrypto; 146 }; 147 148 struct sun8i_crypto_task { 149 struct sun8i_crypto_buf ct_descbuf; 150 struct sun8i_crypto_taskdesc *ct_desc; 151 struct sun8i_crypto_buf ct_ivbuf; 152 void *ct_iv; 153 struct sun8i_crypto_buf ct_ctrbuf; 154 void *ct_ctr; 155 bus_dmamap_t ct_descmap; 156 bus_dmamap_t ct_keymap; 157 bus_dmamap_t ct_ivmap; /* IV input */ 158 bus_dmamap_t ct_ctrmap; /* updated IV output */ 159 bus_dmamap_t ct_srcmap; 160 bus_dmamap_t ct_dstmap; 161 uint32_t ct_nbytes; 162 int ct_flags; 163 #define TASK_KEY __BIT(0) 164 #define TASK_IV __BIT(1) 165 #define TASK_CTR __BIT(2) 166 #define TASK_SRC __BIT(3) 167 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 168 void (*ct_callback)(struct sun8i_crypto_softc *, 169 struct sun8i_crypto_task *, void *, int); 170 void *ct_cookie; 171 }; 172 173 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 174 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 175 176 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 177 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 178 179 /* 180 * Forward declarations 181 */ 182 183 static int sun8i_crypto_match(device_t, cfdata_t, void *); 184 static void sun8i_crypto_attach(device_t, device_t, void *); 185 186 static int sun8i_crypto_task_ctor(void *, void *, int); 187 static void sun8i_crypto_task_dtor(void *, void *); 188 static struct sun8i_crypto_task * 189 sun8i_crypto_task_get(struct sun8i_crypto_softc *, 190 void (*)(struct sun8i_crypto_softc *, 191 struct sun8i_crypto_task *, void *, int), 192 void *, int); 193 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 194 struct sun8i_crypto_task *); 195 196 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 197 struct sun8i_crypto_task *, uint32_t, 198 uint32_t, uint32_t, uint32_t); 199 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 200 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 201 202 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 203 struct sun8i_crypto_task *, uint32_t); 204 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 205 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 206 207 static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 208 struct sun8i_crypto_task *); 209 210 static void sun8i_crypto_timeout(void *); 211 static int sun8i_crypto_intr(void *); 212 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 213 static void sun8i_crypto_worker(struct work *, void *); 214 static bool sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 215 int); 216 217 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 218 struct sun8i_crypto_buf *, int); 219 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 220 struct sun8i_crypto_buf *); 221 222 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 223 static void sun8i_crypto_rng_get(size_t, void *); 224 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 225 struct sun8i_crypto_task *, void *, int); 226 227 static void sun8i_crypto_selftest(device_t); 228 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 229 struct sun8i_crypto_task *, void *, int); 230 231 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 232 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 233 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 234 struct sun8i_crypto_task *, void *, int); 235 236 static void sun8i_crypto_register(struct sun8i_crypto_softc *); 237 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 238 static int sun8i_crypto_newsession(void *, uint32_t *, 239 struct cryptoini *); 240 static int sun8i_crypto_freesession(void *, uint64_t); 241 static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 242 static int sun8i_crypto_process(void *, struct cryptop *, int); 243 static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 244 struct sun8i_crypto_task *, void *, int); 245 246 /* 247 * Probes 248 */ 249 250 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 251 "bus_size_t"/*reg*/, 252 "uint32_t"/*value*/); 253 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 254 "bus_size_t"/*reg*/, 255 "uint32_t"/*write*/); 256 257 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 258 "struct sun8i_crypto_task *"/*task*/); 259 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 260 "int"/*error*/); 261 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 262 "struct sun8i_crypto_task *"/*task*/); 263 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 264 "struct sun8i_crypto_task *"/*task*/); 265 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 266 "struct sun8i_crypto_task *"/*task*/); 267 268 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 269 "struct sun8i_crypto_task *"/*task*/, 270 "uint32_t"/*tdqc*/, 271 "uint32_t"/*tdqs*/, 272 "uint32_t"/*tdqa*/, 273 "struct sun8i_crypto_taskdesc *"/*desc*/, 274 "int"/*error*/); 275 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 276 "struct sun8i_crypto_task *"/*task*/, 277 "bus_addr_t"/*ds_addr*/, 278 "bus_size_t"/*ds_len*/); 279 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 280 "struct sun8i_crypto_task *"/*task*/, 281 "int"/*error*/); 282 283 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 284 "struct sun8i_crypto_softc *"/*sc*/, 285 "struct sun8i_crypto_task *"/*task*/, 286 "int"/*error*/); 287 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 288 "struct sun8i_crypto_softc *"/*sc*/, 289 "struct sun8i_crypto_task *"/*task*/, 290 "unsigned"/*chan*/); 291 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 292 "struct sun8i_crypto_softc *"/*sc*/, 293 "uint32_t"/*isr*/, 294 "uint32_t"/*esr*/); 295 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 296 "struct sun8i_crypto_softc *"/*sc*/, 297 "unsigned"/*chan*/, 298 "int"/*error*/); 299 300 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 301 "struct sun8i_crypto_softc *"/*sc*/, 302 "struct cryptop *"/*crp*/, 303 "int"/*hint*/); 304 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 305 "struct sun8i_crypto_softc *"/*sc*/, 306 "struct cryptop *"/*crp*/, 307 "int"/*hint*/); 308 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 309 "struct sun8i_crypto_softc *"/*sc*/, 310 "struct cryptop *"/*crp*/, 311 "int"/*hint*/, 312 "struct sun8i_crypto_task *"/*task*/); 313 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 314 "struct sun8i_crypto_softc *"/*sc*/, 315 "struct cryptop *"/*crp*/, 316 "int"/*error*/); 317 318 /* 319 * Register access 320 */ 321 322 static uint32_t 323 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 324 { 325 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 326 327 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 328 return v; 329 } 330 331 static void 332 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 333 { 334 335 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 336 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 337 } 338 339 /* 340 * Autoconf goo 341 */ 342 343 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 344 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 345 346 static const struct device_compatible_entry compat_data[] = { 347 { .compat = "allwinner,sun50i-a64-crypto", 348 .data = &sun50i_a64_crypto_config }, 349 { .compat = "allwinner,sun50i-h5-crypto", 350 .data = &sun50i_h5_crypto_config }, 351 { .compat = "allwinner,sun8i-h3-crypto", 352 .data = &sun8i_h3_crypto_config }, 353 DEVICE_COMPAT_EOL 354 }; 355 356 static int 357 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 358 { 359 const struct fdt_attach_args *const faa = aux; 360 361 return of_compatible_match(faa->faa_phandle, compat_data); 362 } 363 364 static void 365 sun8i_crypto_attach(device_t parent, device_t self, void *aux) 366 { 367 struct sun8i_crypto_softc *const sc = device_private(self); 368 const struct fdt_attach_args *const faa = aux; 369 bus_addr_t addr; 370 bus_size_t size; 371 const int phandle = faa->faa_phandle; 372 char intrstr[128]; 373 struct clk *clk; 374 struct fdtbus_reset *rst; 375 u_int mod_rate; 376 377 sc->sc_dev = self; 378 sc->sc_dmat = faa->faa_dmat; 379 sc->sc_bst = faa->faa_bst; 380 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 381 0, 0, 0, "sun8icry", NULL, IPL_VM, 382 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 383 sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data; 384 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM); 385 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 386 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 387 if (workqueue_create(&sc->sc_wq, device_xname(self), 388 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 389 aprint_error(": couldn't create workqueue\n"); 390 return; 391 } 392 393 /* 394 * Prime the pool with enough tasks that each channel can be 395 * busy with a task as we prepare another task for when it's 396 * done. 397 */ 398 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 399 400 /* Get and map device registers. */ 401 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 402 aprint_error(": couldn't get registers\n"); 403 return; 404 } 405 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 406 aprint_error(": couldn't map registers\n"); 407 return; 408 } 409 410 /* Get an interrupt handle. */ 411 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 412 aprint_error(": failed to decode interrupt\n"); 413 return; 414 } 415 416 /* Enable the bus clock. */ 417 if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 418 aprint_error(": couldn't enable bus clock\n"); 419 return; 420 } 421 422 /* Get the module clock and set it. */ 423 mod_rate = sc->sc_cfg->mod_rate; 424 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 425 if (clk_enable(clk) != 0) { 426 aprint_error(": couldn't enable CE clock\n"); 427 return; 428 } 429 if (clk_set_rate(clk, mod_rate) != 0) { 430 aprint_error(": couldn't set CE clock to %d MHz\n", 431 mod_rate / (1000 * 1000)); 432 return; 433 } 434 } 435 436 /* Get a reset handle if we need and try to deassert it. */ 437 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 438 if (fdtbus_reset_deassert(rst) != 0) { 439 aprint_error(": couldn't de-assert reset\n"); 440 return; 441 } 442 } 443 444 aprint_naive("\n"); 445 aprint_normal(": Crypto Engine\n"); 446 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 447 448 /* Disable and clear interrupts. */ 449 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 450 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 451 452 /* Establish an interrupt handler. */ 453 sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM, 454 FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self)); 455 if (sc->sc_ih == NULL) { 456 aprint_error_dev(self, "failed to establish interrupt on %s\n", 457 intrstr); 458 return; 459 } 460 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 461 462 /* Set up the RNG. */ 463 sun8i_crypto_rng_attach(sc); 464 465 /* Attach the sysctl. */ 466 sun8i_crypto_sysctl_attach(sc); 467 468 /* Perform self-tests. */ 469 config_interrupts(self, sun8i_crypto_selftest); 470 471 /* Register opencrypto handlers. */ 472 sun8i_crypto_register(sc); 473 } 474 475 static int 476 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 477 { 478 struct sun8i_crypto_softc *sc = cookie; 479 struct sun8i_crypto_task *task = vtask; 480 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 481 int error; 482 483 /* Create a DMA buffer for the task descriptor. */ 484 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 485 &task->ct_descbuf, dmaflags); 486 if (error) 487 goto fail0; 488 task->ct_desc = task->ct_descbuf.cb_kva; 489 490 /* Create DMA buffers for the IV and CTR. */ 491 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 492 &task->ct_ivbuf, dmaflags); 493 if (error) 494 goto fail1; 495 task->ct_iv = task->ct_ivbuf.cb_kva; 496 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 497 &task->ct_ctrbuf, dmaflags); 498 if (error) 499 goto fail2; 500 task->ct_ctr = task->ct_ctrbuf.cb_kva; 501 502 /* Create a DMA map for the task descriptor and preload it. */ 503 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 504 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 505 if (error) 506 goto fail3; 507 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 508 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 509 if (error) 510 goto fail4; 511 512 /* Create DMA maps for the key, IV, and CTR. */ 513 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 514 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 515 if (error) 516 goto fail5; 517 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 518 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 519 if (error) 520 goto fail6; 521 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 522 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 523 if (error) 524 goto fail7; 525 526 /* Create DMA maps for the src and dst scatter/gather vectors. */ 527 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 528 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 529 &task->ct_srcmap); 530 if (error) 531 goto fail8; 532 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 533 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 534 &task->ct_dstmap); 535 if (error) 536 goto fail9; 537 538 /* Success! */ 539 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 540 return 0; 541 542 fail10: __unused 543 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 544 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 545 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 546 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 547 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 548 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 549 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 550 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 551 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 552 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 553 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 554 return error; 555 } 556 557 static void 558 sun8i_crypto_task_dtor(void *cookie, void *vtask) 559 { 560 struct sun8i_crypto_softc *sc = cookie; 561 struct sun8i_crypto_task *task = vtask; 562 563 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 564 565 /* XXX Zero the bounce buffers if there are any. */ 566 567 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 568 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 569 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 570 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 571 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 572 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 573 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 574 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 575 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 576 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 577 } 578 579 /* 580 * sun8i_crypto_task_get(sc, callback, cookie, pflags) 581 * 582 * Allocate a task that will call callback(sc, task, cookie, 583 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 584 * PR_NOWAIT, may fail and return NULL. No further allocation is 585 * needed to submit the task if this succeeds (although task 586 * submission may still fail if all channels are busy). 587 */ 588 static struct sun8i_crypto_task * 589 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 590 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 591 void *, int), 592 void *cookie, int pflags) 593 { 594 struct sun8i_crypto_task *task; 595 596 /* Allocate a task, or fail if we can't. */ 597 task = pool_cache_get(sc->sc_taskpool, pflags); 598 if (task == NULL) 599 goto out; 600 601 /* Set up flags and the callback. */ 602 task->ct_flags = 0; 603 task->ct_callback = callback; 604 task->ct_cookie = cookie; 605 606 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 607 return task; 608 } 609 610 /* 611 * sun8i_crypto_task_invalid(sc, task, cookie, error) 612 * 613 * Callback for a task not currently in use, to detect errors. 614 */ 615 static void 616 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 617 struct sun8i_crypto_task *task, void *cookie, int error) 618 { 619 void (*callback)(struct sun8i_crypto_softc *, 620 struct sun8i_crypto_task *, void *, int) = cookie; 621 622 panic("task for callback %p used after free", callback); 623 } 624 625 /* 626 * sun8i_crypto_task_put(sc, task) 627 * 628 * Free a task obtained with sun8i_crypto_task_get. 629 */ 630 static void 631 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 632 struct sun8i_crypto_task *task) 633 { 634 635 SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 636 637 task->ct_cookie = task->ct_callback; 638 task->ct_callback = &sun8i_crypto_task_invalid; 639 pool_cache_put(sc->sc_taskpool, task); 640 } 641 642 /* 643 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 644 * 645 * Set up the task descriptor after the relevant DMA maps have 646 * been loaded for a transfer of nbytes. bus_dmamap_sync matches 647 * sun8i_crypto_chan_done. May fail if input is inadequately 648 * aligned. 649 * 650 * XXX Teach this to support task chains. 651 */ 652 static int 653 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 654 struct sun8i_crypto_task *task, uint32_t nbytes, 655 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 656 { 657 struct sun8i_crypto_taskdesc *desc = task->ct_desc; 658 int error; 659 660 KASSERT(tdqs == 0 || tdqa == 0); 661 KASSERT(nbytes % 4 == 0); 662 663 memset(desc, 0, sizeof(*desc)); 664 665 /* Always enable interrupt for the task. */ 666 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 667 668 desc->td_tdqc = htole32(tdqc); 669 desc->td_tdqs = htole32(tdqs); 670 desc->td_tdqa = htole32(tdqa); 671 672 if (task->ct_flags & TASK_KEY) { 673 bus_dmamap_t keymap = task->ct_keymap; 674 KASSERT(keymap->dm_nsegs == 1); 675 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 676 bus_dmamap_sync(sc->sc_dmat, keymap, 0, 677 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 678 } 679 if (task->ct_flags & TASK_IV) { 680 bus_dmamap_t ivmap = task->ct_ivmap; 681 KASSERT(ivmap->dm_nsegs == 1); 682 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 683 bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 684 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 685 } 686 if (task->ct_flags & TASK_CTR) { 687 bus_dmamap_t ctrmap = task->ct_ctrmap; 688 KASSERT(ctrmap->dm_nsegs == 1); 689 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 690 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 691 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 692 } 693 694 if (task->ct_flags & TASK_BYTES) 695 desc->td_datalen = htole32(nbytes); 696 else 697 desc->td_datalen = htole32(nbytes/4); 698 699 if (task->ct_flags & TASK_SRC) { 700 bus_dmamap_t srcmap = task->ct_srcmap; 701 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 702 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 703 nbytes); 704 if (error) 705 return error; 706 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 707 BUS_DMASYNC_PREWRITE); 708 } 709 710 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 711 nbytes); 712 if (error) 713 goto out; 714 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 715 BUS_DMASYNC_PREREAD); 716 717 task->ct_nbytes = nbytes; 718 719 /* Success! */ 720 error = 0; 721 722 out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 723 task, tdqc, tdqs, tdqa, desc, error); 724 return error; 725 } 726 727 /* 728 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 729 * 730 * Set up a task's scatter/gather vector -- src or dst -- with the 731 * given DMA map for a transfer of nbytes. May fail if input is 732 * inadequately aligned. 733 */ 734 static int 735 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 736 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 737 uint32_t nbytes __diagused) 738 { 739 uint32_t total __diagused = 0; 740 unsigned i; 741 742 /* 743 * Verify that the alignment is correct and initialize the 744 * scatter/gather vector. 745 */ 746 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 747 for (i = 0; i < map->dm_nsegs; i++) { 748 if ((map->dm_segs[i].ds_addr % 4) | 749 (map->dm_segs[i].ds_len % 4)) { 750 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 751 task, 752 map->dm_segs[i].ds_addr, 753 map->dm_segs[i].ds_len); 754 return EINVAL; 755 } 756 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 757 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 758 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 759 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 760 total += map->dm_segs[i].ds_len; 761 } 762 763 /* Set the remainder to zero. */ 764 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 765 adrlen[i].adr = 0; 766 adrlen[i].len = 0; 767 } 768 769 /* Verify the total size matches the transfer length. */ 770 KASSERT(total == nbytes); 771 772 /* Success! */ 773 return 0; 774 } 775 776 /* 777 * sun8i_crypto_task_load_trng(task, nbytes) 778 * 779 * Set up the task descriptor for a transfer of nbytes from the 780 * TRNG. 781 */ 782 static int 783 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 784 struct sun8i_crypto_task *task, uint32_t nbytes) 785 { 786 uint32_t tdqc = 0; 787 788 /* Caller must provide dst only. */ 789 KASSERT((task->ct_flags & TASK_KEY) == 0); 790 KASSERT((task->ct_flags & TASK_IV) == 0); 791 KASSERT((task->ct_flags & TASK_CTR) == 0); 792 KASSERT((task->ct_flags & TASK_SRC) == 0); 793 794 /* Set up the task descriptor queue control words. */ 795 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 796 SUN8I_CRYPTO_TDQC_METHOD); 797 798 /* Fill in the descriptor. */ 799 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 800 } 801 802 static int 803 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 804 struct sun8i_crypto_task *task, 805 uint32_t nbytes, uint32_t keysize, uint32_t dir) 806 { 807 uint32_t tdqc = 0, tdqs = 0; 808 809 /* Caller must provide key, src, and dst only. */ 810 KASSERT(task->ct_flags & TASK_KEY); 811 KASSERT((task->ct_flags & TASK_IV) == 0); 812 KASSERT((task->ct_flags & TASK_CTR) == 0); 813 KASSERT(task->ct_flags & TASK_SRC); 814 815 /* Set up the task descriptor queue control word. */ 816 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 817 SUN8I_CRYPTO_TDQC_METHOD); 818 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 819 820 #ifdef DIAGNOSTIC 821 switch (keysize) { 822 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 823 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 824 break; 825 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 826 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 827 break; 828 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 829 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 830 break; 831 } 832 #endif 833 834 /* Set up the symmetric control word. */ 835 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 836 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 837 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 838 SUN8I_CRYPTO_TDQS_OP_MODE); 839 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 840 841 /* Fill in the descriptor. */ 842 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 843 } 844 845 /* 846 * sun8i_crypto_submit(sc, task) 847 * 848 * Submit a task to the crypto engine after it has been loaded 849 * with sun8i_crypto_task_load. On success, guarantees to 850 * eventually call the task's callback. 851 */ 852 static int 853 sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 854 struct sun8i_crypto_task *task) 855 { 856 unsigned i, retries = 0; 857 uint32_t icr; 858 int error = 0; 859 860 /* One at a time at the device registers, please. */ 861 mutex_enter(&sc->sc_lock); 862 863 /* Find a channel. */ 864 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 865 if (sc->sc_chan[i].cc_task == NULL) 866 break; 867 } 868 if (i == SUN8I_CRYPTO_NCHAN) { 869 device_printf(sc->sc_dev, "no free channels\n"); 870 error = ERESTART; 871 goto out; 872 } 873 874 /* 875 * Set the channel id. Caller is responsible for setting up 876 * all other parts of the descriptor. 877 */ 878 task->ct_desc->td_cid = htole32(i); 879 880 /* 881 * Prepare to send the descriptor to the device by DMA. 882 * Matches POSTWRITE in sun8i_crypto_chan_done. 883 */ 884 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 885 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 886 887 /* Confirm we're ready to go. */ 888 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 889 device_printf(sc->sc_dev, "TLR not clear\n"); 890 error = EIO; 891 goto out; 892 } 893 894 /* Enable interrupts for this channel. */ 895 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 896 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 897 SUN8I_CRYPTO_ICR_INTR_EN); 898 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 899 900 /* Set the task descriptor queue address. */ 901 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 902 task->ct_descmap->dm_segs[0].ds_addr); 903 904 /* Notify the engine to load it, and wait for acknowledgement. */ 905 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 906 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 907 { 908 /* 909 * XXX Timeout pulled from arse. Is it even important 910 * to wait here? 911 */ 912 if (++retries == 1000) { 913 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 914 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 915 /* 916 * Hope it clears eventually; if not, we'll 917 * time out. 918 */ 919 break; 920 } 921 DELAY(1); 922 } 923 924 /* 925 * Loaded up and ready to go. Start a timer ticking if it's 926 * not already. 927 */ 928 sc->sc_chan[i].cc_task = task; 929 sc->sc_chan[i].cc_starttime = getticks(); 930 if (!callout_pending(&sc->sc_timeout)) 931 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 932 933 /* XXX Consider polling if cold to get entropy earlier. */ 934 935 out: /* Done! */ 936 if (error) 937 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 938 sc, task, error); 939 else 940 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 941 sc, task, i); 942 mutex_exit(&sc->sc_lock); 943 return error; 944 } 945 946 /* 947 * sun8i_crypto_timeout(cookie) 948 * 949 * Timeout handler. Schedules work in a thread to cancel all 950 * pending tasks that were started long enough ago we're bored of 951 * waiting for them, and reschedules another timeout unless the 952 * channels are all idle. 953 */ 954 static void 955 sun8i_crypto_timeout(void *cookie) 956 { 957 struct sun8i_crypto_softc *sc = cookie; 958 unsigned i; 959 960 mutex_enter(&sc->sc_lock); 961 962 /* Check whether there are any tasks pending. */ 963 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 964 if (sc->sc_chan[i].cc_task) 965 break; 966 } 967 if (i == SUN8I_CRYPTO_NCHAN) 968 /* None pending, so nothing to do. */ 969 goto out; 970 971 /* 972 * Schedule the worker to check for timeouts, and schedule 973 * another timeout in case we need it. 974 */ 975 sun8i_crypto_schedule_worker(sc); 976 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 977 978 out: mutex_exit(&sc->sc_lock); 979 } 980 981 /* 982 * sun8i_crypto_intr(cookie) 983 * 984 * Device interrupt handler. Find what channels have completed, 985 * whether with success or with failure, and schedule work in 986 * thread context to invoke the appropriate callbacks. 987 */ 988 static int 989 sun8i_crypto_intr(void *cookie) 990 { 991 struct sun8i_crypto_softc *sc = cookie; 992 uint32_t isr, esr; 993 994 mutex_enter(&sc->sc_lock); 995 996 /* 997 * Get and acknowledge the interrupts and error status. 998 * 999 * XXX Data sheet says the error status register is read-only, 1000 * but then advises writing 1 to bit x1xx (keysram access error 1001 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 1002 */ 1003 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 1004 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 1005 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 1006 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 1007 1008 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 1009 1010 /* Start the worker if necessary. */ 1011 sun8i_crypto_schedule_worker(sc); 1012 1013 /* Tell the worker what to do. */ 1014 sc->sc_done |= __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 1015 sc->sc_esr |= esr; 1016 1017 mutex_exit(&sc->sc_lock); 1018 1019 return __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE) != 0; 1020 } 1021 1022 /* 1023 * sun8i_crypto_schedule_worker(sc) 1024 * 1025 * Ensure that crypto engine thread context work to invoke task 1026 * callbacks will run promptly. Idempotent. 1027 */ 1028 static void 1029 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 1030 { 1031 1032 KASSERT(mutex_owned(&sc->sc_lock)); 1033 1034 /* Start the worker if necessary. */ 1035 if (!sc->sc_work_pending) { 1036 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1037 sc->sc_work_pending = true; 1038 } 1039 } 1040 1041 /* 1042 * sun8i_crypto_worker(wk, cookie) 1043 * 1044 * Thread-context worker: Invoke all task callbacks for which the 1045 * device has notified us of completion or for which we gave up 1046 * waiting. 1047 */ 1048 static void 1049 sun8i_crypto_worker(struct work *wk, void *cookie) 1050 { 1051 struct sun8i_crypto_softc *sc = cookie; 1052 uint32_t done, esr, esr_chan; 1053 unsigned i, now; 1054 bool unblock = false; 1055 int error; 1056 1057 /* 1058 * Acquire the lock. Note: We will be releasing and 1059 * reacquiring it throughout the loop. 1060 */ 1061 mutex_enter(&sc->sc_lock); 1062 1063 /* Acknowledge the work. */ 1064 KASSERT(sc->sc_work_pending); 1065 sc->sc_work_pending = false; 1066 1067 /* 1068 * Claim the done mask and error status once; we will be 1069 * releasing and reacquiring the lock for the callbacks, so 1070 * they may change. 1071 */ 1072 done = sc->sc_done; 1073 esr = sc->sc_esr; 1074 sc->sc_done = 0; 1075 sc->sc_esr = 0; 1076 1077 /* Check the time to determine what's timed out. */ 1078 now = getticks(); 1079 1080 /* Process the channels. */ 1081 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 1082 /* Check whether the channel is done. */ 1083 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 1084 /* Nope. Do we have a task to time out? */ 1085 if ((sc->sc_chan[i].cc_task != NULL) && 1086 ((now - sc->sc_chan[i].cc_starttime) >= 1087 SUN8I_CRYPTO_TIMEOUT)) 1088 unblock |= sun8i_crypto_chan_done(sc, i, 1089 ETIMEDOUT); 1090 continue; 1091 } 1092 1093 /* Channel is done. Interpret the error if any. */ 1094 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1095 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1096 device_printf(sc->sc_dev, "channel %u:" 1097 " alg not supported\n", i); 1098 error = ENODEV; 1099 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1100 device_printf(sc->sc_dev, "channel %u:" 1101 " data length error\n", i); 1102 error = EIO; /* XXX */ 1103 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1104 device_printf(sc->sc_dev, "channel %u:" 1105 " key sram error\n", i); 1106 error = EIO; /* XXX */ 1107 } else if (esr_chan != 0) { 1108 error = EIO; /* generic I/O error */ 1109 } else { 1110 error = 0; 1111 } 1112 1113 /* 1114 * Notify the task of completion. May release the lock 1115 * to invoke a callback. 1116 */ 1117 unblock |= sun8i_crypto_chan_done(sc, i, error); 1118 } 1119 1120 /* All one; release the lock one last time. */ 1121 mutex_exit(&sc->sc_lock); 1122 1123 /* 1124 * If we cleared any channels, it is time to allow opencrypto 1125 * to issue new operations. Asymmetric operations (which we 1126 * don't support, at the moment, but we could) and symmetric 1127 * operations (which we do) use the same task channels, so we 1128 * unblock both kinds. 1129 */ 1130 if (unblock) { 1131 crypto_unblock(sc->sc_opencrypto.co_driverid, 1132 CRYPTO_SYMQ|CRYPTO_ASYMQ); 1133 } 1134 } 1135 1136 /* 1137 * sun8i_crypto_chan_done(sc, i, error) 1138 * 1139 * Notify the callback for the task on channel i, if there is one, 1140 * of the specified error, or 0 for success. 1141 */ 1142 static bool 1143 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1144 { 1145 struct sun8i_crypto_task *task; 1146 uint32_t nbytes; 1147 uint32_t icr; 1148 1149 KASSERT(mutex_owned(&sc->sc_lock)); 1150 1151 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 1152 1153 /* Claim the task if there is one; bail if not. */ 1154 if ((task = sc->sc_chan[i].cc_task) == NULL) { 1155 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1156 i, error); 1157 /* We did not clear a channel. */ 1158 return false; 1159 } 1160 sc->sc_chan[i].cc_task = NULL; 1161 1162 /* Disable interrupts on this channel. */ 1163 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1164 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1165 SUN8I_CRYPTO_ICR_INTR_EN); 1166 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1167 1168 /* 1169 * Finished sending the descriptor to the device by DMA. 1170 * Matches PREWRITE in sun8i_crypto_task_submit. 1171 */ 1172 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1173 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1174 1175 /* 1176 * Finished with all the other bits of DMA too. Matches 1177 * sun8i_crypto_task_load. 1178 */ 1179 nbytes = task->ct_nbytes; 1180 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 1181 BUS_DMASYNC_POSTREAD); 1182 if (task->ct_flags & TASK_SRC) 1183 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 1184 BUS_DMASYNC_POSTWRITE); 1185 if (task->ct_flags & TASK_CTR) 1186 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 1187 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 1188 if (task->ct_flags & TASK_IV) 1189 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 1190 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1191 if (task->ct_flags & TASK_KEY) 1192 /* XXX Can we zero the bounce buffer if there is one? */ 1193 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 1194 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1195 1196 /* Temporarily release the lock to invoke the callback. */ 1197 mutex_exit(&sc->sc_lock); 1198 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1199 (*task->ct_callback)(sc, task, task->ct_cookie, error); 1200 mutex_enter(&sc->sc_lock); 1201 1202 /* We cleared a channel. */ 1203 return true; 1204 } 1205 1206 /* 1207 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 1208 * 1209 * Allocate a single-segment DMA-safe buffer and map it into KVA. 1210 * May fail if dmaflags is BUS_DMA_NOWAIT. 1211 */ 1212 static int 1213 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 1214 struct sun8i_crypto_buf *buf, int dmaflags) 1215 { 1216 int error; 1217 1218 /* Allocate a DMA-safe buffer. */ 1219 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 1220 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1221 if (error) 1222 goto fail0; 1223 1224 /* Map the buffer into kernel virtual address space. */ 1225 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 1226 size, &buf->cb_kva, dmaflags); 1227 if (error) 1228 goto fail1; 1229 1230 /* Success! */ 1231 return 0; 1232 1233 fail2: __unused 1234 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1235 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1236 fail0: return error; 1237 } 1238 1239 /* 1240 * sun8i_crypto_freebuf(sc, buf) 1241 * 1242 * Unmap buf and free it. 1243 */ 1244 static void 1245 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1246 struct sun8i_crypto_buf *buf) 1247 { 1248 1249 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1250 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1251 } 1252 1253 /* 1254 * sun8i_crypto_rng_attach(sc) 1255 * 1256 * Attach an rndsource for the crypto engine's TRNG. 1257 */ 1258 static void 1259 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1260 { 1261 device_t self = sc->sc_dev; 1262 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1263 struct sun8i_crypto_task *task; 1264 int error; 1265 1266 /* Preallocate a buffer to reuse. */ 1267 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 1268 BUS_DMA_WAITOK); 1269 if (error) { 1270 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 1271 error); 1272 goto fail0; 1273 } 1274 1275 /* Create a task to reuse. */ 1276 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 1277 rng, PR_WAITOK); 1278 if (rng->cr_task == NULL) { 1279 aprint_error_dev(self, "failed to allocate RNG task\n"); 1280 error = ENOMEM; 1281 goto fail1; 1282 } 1283 1284 /* Preload the destination map. */ 1285 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1286 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1287 if (error) { 1288 aprint_error_dev(self, "failed to load RNG buffer: %d\n", 1289 error); 1290 goto fail2; 1291 } 1292 1293 /* 1294 * Attach the rndsource. This is _not_ marked as RND_TYPE_RNG 1295 * because the output is not uniformly distributed. The bits 1296 * are heavily weighted toward 0 or 1, at different times, and 1297 * I haven't scienced a satisfactory story out of it yet. 1298 */ 1299 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1300 rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1301 RND_TYPE_UNKNOWN, 1302 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1303 1304 /* Success! */ 1305 return; 1306 1307 fail3: __unused 1308 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1309 fail2: sun8i_crypto_task_put(sc, task); 1310 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 1311 fail0: return; 1312 } 1313 1314 /* 1315 * sun8i_crypto_rng_get(nbytes, cookie) 1316 * 1317 * On-demand rndsource callback: try to gather nbytes of entropy 1318 * and enter them into the pool ASAP. 1319 */ 1320 static void 1321 sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1322 { 1323 struct sun8i_crypto_softc *sc = cookie; 1324 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1325 struct sun8i_crypto_task *task = rng->cr_task; 1326 bool pending; 1327 int error; 1328 1329 /* 1330 * Test and set the RNG-pending flag. If it's already in 1331 * progress, nothing to do here. 1332 */ 1333 mutex_enter(&sc->sc_lock); 1334 pending = rng->cr_pending; 1335 rng->cr_pending = true; 1336 mutex_exit(&sc->sc_lock); 1337 if (pending) 1338 return; 1339 1340 /* Load the task descriptor. */ 1341 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1342 if (error) 1343 goto fail; 1344 1345 /* Submit! */ 1346 error = sun8i_crypto_submit(sc, task); 1347 if (error) 1348 goto fail; 1349 1350 /* All done! */ 1351 return; 1352 1353 fail: mutex_enter(&sc->sc_lock); 1354 rng->cr_pending = false; 1355 mutex_exit(&sc->sc_lock); 1356 } 1357 1358 static void 1359 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1360 struct sun8i_crypto_task *task, void *cookie, int error) 1361 { 1362 struct sun8i_crypto_rng *rng = cookie; 1363 uint8_t *buf = rng->cr_buf.cb_kva; 1364 uint32_t entropybits; 1365 1366 KASSERT(rng == &sc->sc_rng); 1367 1368 /* If anything went wrong, forget about it. */ 1369 if (error) 1370 goto out; 1371 1372 /* 1373 * This TRNG has quite low entropy at best. But if it fails a 1374 * repeated output test, then assume it's busted. 1375 */ 1376 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1377 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1378 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1379 SUN8I_CRYPTO_RNGBYTES/2)) { 1380 device_printf(sc->sc_dev, "failed repeated output test\n"); 1381 entropybits = 0; 1382 } 1383 1384 /* 1385 * Actually we don't believe in any of the entropy until this 1386 * device has had more scrutiny. 1387 */ 1388 entropybits = 0; 1389 1390 /* Success! Enter and erase the data. */ 1391 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1392 entropybits); 1393 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1394 1395 out: /* Done -- clear the RNG-pending flag. */ 1396 mutex_enter(&sc->sc_lock); 1397 rng->cr_pending = false; 1398 mutex_exit(&sc->sc_lock); 1399 } 1400 1401 /* 1402 * Self-test 1403 */ 1404 1405 static const uint8_t selftest_input[16]; 1406 static const uint8_t selftest_key[16]; 1407 static const uint8_t selftest_output[16] = { 1408 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1409 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1410 }; 1411 1412 static void 1413 sun8i_crypto_selftest(device_t self) 1414 { 1415 const size_t keybytes = sizeof selftest_key; 1416 const size_t nbytes = sizeof selftest_input; 1417 struct sun8i_crypto_softc *sc = device_private(self); 1418 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 1419 struct sun8i_crypto_task *task; 1420 int error; 1421 1422 CTASSERT(sizeof selftest_input == sizeof selftest_output); 1423 1424 /* Allocate an input buffer. */ 1425 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 1426 BUS_DMA_WAITOK); 1427 if (error) 1428 goto fail0; 1429 1430 /* Allocate a key buffer. */ 1431 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 1432 BUS_DMA_WAITOK); 1433 if (error) 1434 goto fail1; 1435 1436 /* Allocate an output buffer. */ 1437 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 1438 BUS_DMA_WAITOK); 1439 if (error) 1440 goto fail2; 1441 1442 /* Allocate a task descriptor. */ 1443 task = selftest->cs_task = sun8i_crypto_task_get(sc, 1444 sun8i_crypto_selftest_done, selftest, PR_WAITOK); 1445 if (selftest->cs_task == NULL) { 1446 error = ENOMEM; 1447 goto fail3; 1448 } 1449 1450 /* Copy the input and key into their buffers. */ 1451 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 1452 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1453 1454 /* Load the key, src, and dst for DMA transfers. */ 1455 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 1456 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1457 if (error) 1458 goto fail4; 1459 task->ct_flags |= TASK_KEY; 1460 1461 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 1462 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1463 if (error) 1464 goto fail5; 1465 task->ct_flags |= TASK_SRC; 1466 1467 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1468 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1469 if (error) 1470 goto fail6; 1471 1472 /* Set up the task descriptor. */ 1473 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 1474 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 1475 if (error) 1476 goto fail7; 1477 1478 /* Submit! */ 1479 error = sun8i_crypto_submit(sc, task); 1480 if (error) 1481 goto fail7; 1482 1483 device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1484 1485 /* Success! */ 1486 return; 1487 1488 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1489 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1490 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1491 fail4: sun8i_crypto_task_put(sc, task); 1492 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1493 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1494 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1495 fail0: aprint_error_dev(self, "failed to run self-test, error=%d\n", error); 1496 } 1497 1498 static bool 1499 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1500 size_t n, const void *expected, const void *actual) 1501 { 1502 const uint8_t *e = expected; 1503 const uint8_t *a = actual; 1504 size_t i; 1505 1506 if (memcmp(e, a, n) == 0) 1507 return true; 1508 1509 device_printf(sc->sc_dev, "self-test: %s\n", title); 1510 printf("expected: "); 1511 for (i = 0; i < n; i++) 1512 printf("%02hhx", e[i]); 1513 printf("\n"); 1514 printf("actual: "); 1515 for (i = 0; i < n; i++) 1516 printf("%02hhx", a[i]); 1517 printf("\n"); 1518 return false; 1519 } 1520 1521 static void 1522 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1523 struct sun8i_crypto_task *task, void *cookie, int error) 1524 { 1525 const size_t keybytes = sizeof selftest_key; 1526 const size_t nbytes = sizeof selftest_input; 1527 struct sun8i_crypto_selftest *selftest = cookie; 1528 bool ok = true; 1529 1530 KASSERT(selftest == &sc->sc_selftest); 1531 1532 /* If anything went wrong, fail now. */ 1533 if (error) { 1534 device_printf(sc->sc_dev, "self-test error=%d\n", error); 1535 goto out; 1536 } 1537 1538 /* 1539 * Verify the input and key weren't clobbered, and verify the 1540 * output matches what we expect. 1541 */ 1542 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 1543 selftest_input, selftest->cs_in.cb_kva); 1544 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 1545 selftest_key, selftest->cs_key.cb_kva); 1546 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 1547 selftest_output, selftest->cs_out.cb_kva); 1548 1549 /* XXX Disable the RNG and other stuff if this fails... */ 1550 if (ok) 1551 device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1552 1553 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1554 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1555 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1556 sun8i_crypto_task_put(sc, task); 1557 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1558 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1559 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1560 } 1561 1562 /* 1563 * Sysctl for testing 1564 */ 1565 1566 struct sun8i_crypto_userreq { 1567 kmutex_t cu_lock; 1568 kcondvar_t cu_cv; 1569 size_t cu_size; 1570 struct sun8i_crypto_buf cu_buf; 1571 struct sun8i_crypto_task *cu_task; 1572 int cu_error; 1573 bool cu_done; 1574 bool cu_cancel; 1575 }; 1576 1577 static void 1578 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1579 { 1580 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1581 int error; 1582 1583 /* hw.sun8icryptoN (node) */ 1584 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1585 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1586 SYSCTL_DESCR("sun8i crypto engine knobs"), 1587 NULL, 0, NULL, 0, 1588 CTL_HW, CTL_CREATE, CTL_EOL); 1589 if (error) { 1590 aprint_error_dev(sc->sc_dev, 1591 "failed to set up sysctl hw.%s: %d\n", 1592 device_xname(sc->sc_dev), error); 1593 return; 1594 } 1595 1596 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1597 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1598 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1599 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1600 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1601 if (error) { 1602 aprint_error_dev(sc->sc_dev, 1603 "failed to set up sysctl hw.%s.rng: %d\n", 1604 device_xname(sc->sc_dev), error); 1605 return; 1606 } 1607 } 1608 1609 static int 1610 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1611 { 1612 struct sysctlnode node = *rnode; 1613 struct sun8i_crypto_softc *sc = node.sysctl_data; 1614 struct sun8i_crypto_userreq *req; 1615 struct sun8i_crypto_task *task; 1616 size_t size; 1617 int error; 1618 1619 /* If oldp == NULL, the caller wants to learn the size. */ 1620 if (oldp == NULL) { 1621 *oldlenp = 4096; 1622 return 0; 1623 } 1624 1625 /* Truncate to 4096 bytes. */ 1626 size = MIN(4096, *oldlenp); 1627 if (size == 0) 1628 return 0; /* nothing to do */ 1629 1630 /* Allocate a request context. */ 1631 req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1632 if (req == NULL) 1633 return ENOMEM; 1634 1635 /* Initialize the request context. */ 1636 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1637 cv_init(&req->cu_cv, "sun8isy"); 1638 req->cu_size = size; 1639 req->cu_error = EIO; 1640 req->cu_done = false; 1641 req->cu_cancel = false; 1642 1643 /* Allocate a buffer for the RNG output. */ 1644 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1645 if (error) 1646 goto out0; 1647 1648 /* Allocate a task. */ 1649 task = req->cu_task = sun8i_crypto_task_get(sc, 1650 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 1651 if (task == NULL) { 1652 error = ENOMEM; 1653 goto out1; 1654 } 1655 1656 /* Set the task up for TRNG to our buffer. */ 1657 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1658 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1659 if (error) 1660 goto out2; 1661 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1662 if (error) 1663 goto out3; 1664 1665 /* Submit! */ 1666 error = sun8i_crypto_submit(sc, task); 1667 if (error) { 1668 /* Make sure we don't restart the syscall -- just fail. */ 1669 if (error == ERESTART) 1670 error = EBUSY; 1671 goto out3; 1672 } 1673 1674 /* Wait for the request to complete. */ 1675 mutex_enter(&req->cu_lock); 1676 while (!req->cu_done) { 1677 error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1678 if (error) { 1679 /* 1680 * If we finished while waiting to acquire the 1681 * lock, ignore the error and just return now. 1682 * Otherwise, notify the callback that it has 1683 * to clean up after us. 1684 */ 1685 if (req->cu_done) 1686 error = 0; 1687 else 1688 req->cu_cancel = true; 1689 break; 1690 } 1691 } 1692 mutex_exit(&req->cu_lock); 1693 1694 /* 1695 * Return early on error from cv_wait_sig, which means 1696 * interruption; the callback will clean up instead. 1697 */ 1698 if (error) 1699 return error; 1700 1701 /* Check for error from the device. */ 1702 error = req->cu_error; 1703 if (error) 1704 goto out3; 1705 1706 /* Copy out the data. */ 1707 node.sysctl_data = req->cu_buf.cb_kva; 1708 node.sysctl_size = size; 1709 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1710 1711 /* Clear the buffer. */ 1712 explicit_memset(req->cu_buf.cb_kva, 0, size); 1713 1714 /* Clean up. */ 1715 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1716 out2: sun8i_crypto_task_put(sc, task); 1717 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1718 out0: cv_destroy(&req->cu_cv); 1719 mutex_destroy(&req->cu_lock); 1720 kmem_free(req, sizeof(*req)); 1721 return error; 1722 } 1723 1724 static void 1725 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1726 struct sun8i_crypto_task *task, void *cookie, int error) 1727 { 1728 struct sun8i_crypto_userreq *req = cookie; 1729 bool cancel; 1730 1731 /* 1732 * Notify the waiting thread of the error, and find out whether 1733 * that thread cancelled. 1734 */ 1735 mutex_enter(&req->cu_lock); 1736 cancel = req->cu_cancel; 1737 req->cu_error = error; 1738 req->cu_done = true; 1739 cv_broadcast(&req->cu_cv); 1740 mutex_exit(&req->cu_lock); 1741 1742 /* 1743 * If it wasn't cancelled, we're done -- the main thread will 1744 * clean up after itself. 1745 */ 1746 if (!cancel) 1747 return; 1748 1749 /* Clean up after the main thread cancelled. */ 1750 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1751 sun8i_crypto_task_put(sc, task); 1752 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1753 cv_destroy(&req->cu_cv); 1754 mutex_destroy(&req->cu_lock); 1755 kmem_free(req, sizeof(*req)); 1756 } 1757 1758 /* 1759 * sun8i_crypto_register(sc) 1760 * 1761 * Register opencrypto algorithms supported by the crypto engine. 1762 */ 1763 static void 1764 sun8i_crypto_register(struct sun8i_crypto_softc *sc) 1765 { 1766 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 1767 1768 co->co_driverid = crypto_get_driverid(0); 1769 if (co->co_driverid == (uint32_t)-1) { 1770 aprint_error_dev(sc->sc_dev, 1771 "failed to register crypto driver\n"); 1772 return; 1773 } 1774 1775 sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 1776 sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 1777 #ifdef CRYPTO_AES_ECB 1778 sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 1779 #endif 1780 #ifdef CRYPTO_AES_XTS 1781 sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 1782 #endif 1783 #ifdef CRYPTO_DES_CBC 1784 sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 1785 #endif 1786 #ifdef CRYPTO_DES_ECB 1787 sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 1788 #endif 1789 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 1790 #ifdef CRYPTO_3DES_ECB 1791 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 1792 #endif 1793 1794 sun8i_crypto_register1(sc, CRYPTO_MD5); 1795 sun8i_crypto_register1(sc, CRYPTO_SHA1); 1796 #ifdef CRYPTO_SHA224 1797 sun8i_crypto_register1(sc, CRYPTO_SHA224); 1798 #endif 1799 #ifdef CRYPTO_SHA256 1800 sun8i_crypto_register1(sc, CRYPTO_SHA256); 1801 #endif 1802 1803 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 1804 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 1805 1806 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 1807 } 1808 1809 /* 1810 * sun8i_crypto_register1(sc, alg) 1811 * 1812 * Register support for one algorithm alg using 1813 * sun8i_crypto_newsession/freesession/process. 1814 */ 1815 static void 1816 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 1817 { 1818 1819 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 1820 sun8i_crypto_newsession, 1821 sun8i_crypto_freesession, 1822 sun8i_crypto_process, 1823 sc); 1824 } 1825 1826 /* 1827 * sun8i_crypto_newsession(cookie, sidp, cri) 1828 * 1829 * Called by opencrypto to allocate a new session. We don't keep 1830 * track of sessions, since there are no persistent keys in the 1831 * hardware that we take advantage of, so this only validates the 1832 * crypto operations and returns a dummy session id of 1. 1833 */ 1834 static int 1835 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 1836 { 1837 1838 /* No composition of operations is supported here. */ 1839 if (cri->cri_next) 1840 return EINVAL; 1841 1842 /* 1843 * No variation of rounds is supported here. (XXX Unused and 1844 * unimplemented in opencrypto(9) altogether?) 1845 */ 1846 if (cri->cri_rnd) 1847 return EINVAL; 1848 1849 /* 1850 * Validate per-algorithm key length. 1851 * 1852 * XXX Does opencrypto(9) do this internally? 1853 */ 1854 switch (cri->cri_alg) { 1855 case CRYPTO_MD5: 1856 case CRYPTO_SHA1: 1857 #ifdef CRYPTO_SHA224 1858 case CRYPTO_SHA224: 1859 #endif 1860 #ifdef CRYPTO_SHA256 1861 case CRYPTO_SHA256: 1862 #endif 1863 if (cri->cri_klen) 1864 return EINVAL; 1865 break; 1866 case CRYPTO_AES_CBC: 1867 #ifdef CRYPTO_AES_ECB 1868 case CRYPTO_AES_ECB: 1869 #endif 1870 switch (cri->cri_klen) { 1871 case 128: 1872 case 192: 1873 case 256: 1874 break; 1875 default: 1876 return EINVAL; 1877 } 1878 break; 1879 case CRYPTO_AES_CTR: 1880 /* 1881 * opencrypto `AES-CTR' takes four bytes of the input 1882 * block as the last four bytes of the key, for reasons 1883 * that are not entirely clear. 1884 */ 1885 switch (cri->cri_klen) { 1886 case 128 + 32: 1887 case 192 + 32: 1888 case 256 + 32: 1889 break; 1890 default: 1891 return EINVAL; 1892 } 1893 break; 1894 #ifdef CRYPTO_AES_XTS 1895 case CRYPTO_AES_XTS: 1896 switch (cri->cri_klen) { 1897 case 256: 1898 case 384: 1899 case 512: 1900 break; 1901 default: 1902 return EINVAL; 1903 } 1904 break; 1905 #endif 1906 case CRYPTO_DES_CBC: 1907 #ifdef CRYPTO_DES_ECB 1908 case CRYPTO_DES_ECB: 1909 #endif 1910 switch (cri->cri_klen) { 1911 case 64: 1912 break; 1913 default: 1914 return EINVAL; 1915 } 1916 break; 1917 case CRYPTO_3DES_CBC: 1918 #ifdef CRYPTO_3DES_ECB 1919 case CRYPTO_3DES_ECB: 1920 #endif 1921 switch (cri->cri_klen) { 1922 case 192: 1923 break; 1924 default: 1925 return EINVAL; 1926 } 1927 break; 1928 case CRYPTO_SHA1_HMAC: 1929 /* 1930 * XXX Unclear what the length limit is, but since HMAC 1931 * behaves qualitatively different for a key of at 1932 * least the full block size -- and is generally best 1933 * to use with half the block size -- let's limit it to 1934 * one block. 1935 */ 1936 if (cri->cri_klen % 8) 1937 return EINVAL; 1938 if (cri->cri_klen > 512) 1939 return EINVAL; 1940 break; 1941 case CRYPTO_SHA2_256_HMAC: 1942 if (cri->cri_klen % 8) 1943 return EINVAL; 1944 if (cri->cri_klen > 512) 1945 return EINVAL; 1946 break; 1947 default: 1948 panic("unsupported algorithm %d", cri->cri_alg); 1949 } 1950 1951 KASSERT(cri->cri_klen % 8 == 0); 1952 1953 /* Success! */ 1954 *sidp = 1; 1955 return 0; 1956 } 1957 1958 /* 1959 * sun8i_crypto_freesession(cookie, dsid) 1960 * 1961 * Called by opencrypto to free a session. We don't keep track of 1962 * sessions, since there are no persistent keys in the hardware 1963 * that we take advantage of, so this is a no-op. 1964 * 1965 * Note: dsid is actually a 64-bit quantity containing both the 1966 * driver id in the high half and the session id in the low half. 1967 */ 1968 static int 1969 sun8i_crypto_freesession(void *cookie, uint64_t dsid) 1970 { 1971 1972 KASSERT((dsid & 0xffffffff) == 1); 1973 1974 /* Success! */ 1975 return 0; 1976 } 1977 1978 /* 1979 * sun8i_crypto_ivlen(crd) 1980 * 1981 * Return the crypto engine's notion of `IV length', in bytes, for 1982 * an opencrypto operation. 1983 */ 1984 static u_int 1985 sun8i_crypto_ivlen(const struct cryptodesc *crd) 1986 { 1987 1988 switch (crd->crd_alg) { 1989 case CRYPTO_AES_CBC: 1990 return 16; 1991 #ifdef CRYPTO_AES_XTS 1992 case CRYPTO_AES_XTS: 1993 return 16; 1994 #endif 1995 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 1996 return 8; 1997 #ifdef CRYPTO_DES_CBC 1998 case CRYPTO_DES_CBC: 1999 return 8; 2000 #endif 2001 case CRYPTO_3DES_CBC: 2002 return 8; 2003 case CRYPTO_MD5: 2004 return 16; 2005 #ifdef CRYPTO_SHA224 2006 case CRYPTO_SHA224: 2007 return 32; 2008 #endif 2009 #ifdef CRYPTO_SHA256 2010 case CRYPTO_SHA256: 2011 return 32; 2012 #endif 2013 case CRYPTO_SHA1_HMAC: 2014 return 20; 2015 case CRYPTO_SHA2_256_HMAC: 2016 return 32; 2017 default: 2018 return 0; 2019 } 2020 } 2021 2022 /* 2023 * sun8i_crypto_process(cookie, crp, hint) 2024 * 2025 * Main opencrypto processing dispatch. 2026 */ 2027 static int 2028 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 2029 { 2030 struct sun8i_crypto_softc *sc = cookie; 2031 struct sun8i_crypto_task *task; 2032 struct cryptodesc *crd = crp->crp_desc; 2033 unsigned klen, ivlen; 2034 uint32_t tdqc = 0, tdqs = 0; 2035 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 2036 const uint32_t tdqa = 0; 2037 int error; 2038 2039 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 2040 2041 /* Reject compositions -- we do not handle them. */ 2042 if (crd->crd_next != NULL) { 2043 error = EOPNOTSUPP; 2044 goto fail0; 2045 } 2046 2047 /* Reject transfers with nonsense skip. */ 2048 if (crd->crd_skip < 0) { 2049 error = EINVAL; 2050 goto fail0; 2051 } 2052 2053 /* 2054 * Actually just reject any nonzero skip, because it requires 2055 * DMA segment bookkeeping that we don't do yet. 2056 */ 2057 if (crd->crd_skip) { 2058 error = EOPNOTSUPP; 2059 goto fail0; 2060 } 2061 2062 /* Reject large transfers. */ 2063 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 2064 error = EFBIG; 2065 goto fail0; 2066 } 2067 2068 /* Reject nonsense, unaligned, or mismatched lengths. */ 2069 if (crd->crd_len < 0 || 2070 crd->crd_len % 4 || 2071 crd->crd_len != crp->crp_ilen) { 2072 error = EINVAL; 2073 goto fail0; 2074 } 2075 2076 /* Reject mismatched buffer lengths. */ 2077 /* XXX Handle crd_skip. */ 2078 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2079 struct mbuf *m = crp->crp_buf; 2080 uint32_t nbytes = 0; 2081 while (m != NULL) { 2082 KASSERT(m->m_len >= 0); 2083 if (m->m_len > crd->crd_len || 2084 nbytes > crd->crd_len - m->m_len) { 2085 error = EINVAL; 2086 goto fail0; 2087 } 2088 nbytes += m->m_len; 2089 m = m->m_next; 2090 } 2091 if (nbytes != crd->crd_len) { 2092 error = EINVAL; 2093 goto fail0; 2094 } 2095 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2096 struct uio *uio = crp->crp_buf; 2097 if (uio->uio_resid != crd->crd_len) { 2098 error = EINVAL; 2099 goto fail0; 2100 } 2101 } 2102 2103 /* Get a task, or fail with ERESTART if we can't. */ 2104 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 2105 PR_NOWAIT); 2106 if (task == NULL) { 2107 /* 2108 * Don't invoke crypto_done -- we are asking the 2109 * opencrypto(9) machinery to queue the request and get 2110 * back to us. 2111 */ 2112 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 2113 return ERESTART; 2114 } 2115 2116 /* Load key in, if relevant. */ 2117 klen = crd->crd_klen; 2118 if (klen) { 2119 if (crd->crd_alg == CRYPTO_AES_CTR) 2120 /* AES-CTR is special -- see IV processing below. */ 2121 klen -= 32; 2122 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 2123 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 2124 if (error) 2125 goto fail1; 2126 task->ct_flags |= TASK_KEY; 2127 } 2128 2129 /* Handle the IV, if relevant. */ 2130 ivlen = sun8i_crypto_ivlen(crd); 2131 if (ivlen) { 2132 void *iv; 2133 2134 /* 2135 * If there's an explicit IV, use it; otherwise 2136 * randomly generate one. 2137 */ 2138 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2139 iv = crd->crd_iv; 2140 } else { 2141 cprng_fast(task->ct_iv, ivlen); 2142 iv = task->ct_iv; 2143 } 2144 2145 /* 2146 * If the IV is not already present in the user's 2147 * buffer, copy it over. 2148 */ 2149 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 2150 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2151 m_copyback(crp->crp_buf, crd->crd_inject, 2152 ivlen, iv); 2153 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2154 cuio_copyback(crp->crp_buf, crd->crd_inject, 2155 ivlen, iv); 2156 } else { 2157 panic("invalid buffer type %x", 2158 crp->crp_flags); 2159 } 2160 } 2161 2162 /* 2163 * opencrypto's idea of `AES-CTR' is special. 2164 * 2165 * - The low 4 bytes of the input block are drawn from 2166 * an extra 4 bytes at the end of the key. 2167 * 2168 * - The next 8 bytes of the input block are drawn from 2169 * the opencrypto iv. 2170 * 2171 * - The high 4 bytes are the big-endian block counter, 2172 * which starts at 1 because why not. 2173 */ 2174 if (crd->crd_alg == CRYPTO_AES_CTR) { 2175 uint8_t block[16]; 2176 uint32_t blkno = 1; 2177 2178 /* Format the initial input block. */ 2179 memcpy(block, crd->crd_key + klen/8, 4); 2180 memcpy(block + 4, iv, 8); 2181 be32enc(block + 12, blkno); 2182 2183 /* Copy it into the DMA buffer. */ 2184 memcpy(task->ct_iv, block, 16); 2185 iv = task->ct_iv; 2186 ivlen = 16; 2187 } 2188 2189 /* Load the IV. */ 2190 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 2191 NULL, BUS_DMA_NOWAIT); 2192 if (error) 2193 goto fail1; 2194 task->ct_flags |= TASK_IV; 2195 } 2196 2197 /* Load the src and dst. */ 2198 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2199 struct mbuf *m = crp->crp_buf; 2200 2201 /* XXX Handle crd_skip. */ 2202 KASSERT(crd->crd_skip == 0); 2203 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 2204 BUS_DMA_NOWAIT); 2205 if (error) 2206 goto fail1; 2207 task->ct_flags |= TASK_SRC; 2208 2209 /* XXX Handle crd_skip. */ 2210 KASSERT(crd->crd_skip == 0); 2211 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 2212 BUS_DMA_NOWAIT); 2213 if (error) 2214 goto fail1; 2215 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2216 struct uio *uio = crp->crp_buf; 2217 2218 /* XXX Handle crd_skip. */ 2219 KASSERT(crd->crd_skip == 0); 2220 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 2221 BUS_DMA_NOWAIT); 2222 if (error) 2223 goto fail1; 2224 task->ct_flags |= TASK_SRC; 2225 2226 /* XXX Handle crd_skip. */ 2227 KASSERT(crd->crd_skip == 0); 2228 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 2229 BUS_DMA_NOWAIT); 2230 if (error) 2231 goto fail1; 2232 } else { 2233 panic("invalid buffer type %x", crp->crp_flags); 2234 } 2235 2236 /* Set the encryption direction. */ 2237 if (crd->crd_flags & CRD_F_ENCRYPT) 2238 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 2239 else 2240 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 2241 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 2242 2243 /* Set the method. */ 2244 switch (crd->crd_alg) { 2245 case CRYPTO_AES_CBC: 2246 case CRYPTO_AES_CTR: 2247 #ifdef CRYPTO_AES_ECB 2248 case CRYPTO_AES_ECB: 2249 #endif 2250 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2251 break; 2252 #ifdef CRYPTO_AES_XTS 2253 case CRYPTO_AES_XTS: 2254 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2255 break; 2256 #endif 2257 case CRYPTO_DES_CBC: 2258 #ifdef CRYPTO_DES_ECB 2259 case CRYPTO_DES_ECB: 2260 #endif 2261 method = SUN8I_CRYPTO_TDQC_METHOD_DES; 2262 break; 2263 case CRYPTO_3DES_CBC: 2264 #ifdef CRYPTO_3DES_ECB 2265 case CRYPTO_3DES_ECB: 2266 #endif 2267 method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 2268 break; 2269 case CRYPTO_MD5: 2270 method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 2271 break; 2272 case CRYPTO_SHA1: 2273 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 2274 break; 2275 #ifdef CRYPTO_SHA224 2276 case CRYPTO_SHA224: 2277 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 2278 break; 2279 #endif 2280 #ifdef CRYPTO_SHA256 2281 case CRYPTO_SHA256: 2282 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 2283 break; 2284 #endif 2285 case CRYPTO_SHA1_HMAC: 2286 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 2287 break; 2288 case CRYPTO_SHA2_256_HMAC: 2289 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 2290 break; 2291 default: 2292 panic("unknown algorithm %d", crd->crd_alg); 2293 } 2294 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 2295 2296 /* Set the key selector. No idea how to use the internal keys. */ 2297 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 2298 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 2299 2300 /* XXX Deal with AES_CTS_Last_Block_Flag. */ 2301 2302 /* Set the mode. */ 2303 switch (crd->crd_alg) { 2304 #ifdef CRYPTO_AES_ECB 2305 case CRYPTO_AES_ECB: 2306 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2307 break; 2308 #endif 2309 #ifdef CRYPTO_DES_ECB 2310 case CRYPTO_DES_ECB: 2311 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2312 break; 2313 #endif 2314 #ifdef CRYPTO_3DES_ECB 2315 case CRYPTO_3DES_ECB: 2316 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2317 break; 2318 #endif 2319 case CRYPTO_AES_CBC: 2320 case CRYPTO_DES_CBC: 2321 case CRYPTO_3DES_CBC: 2322 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 2323 break; 2324 case CRYPTO_AES_CTR: 2325 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 2326 break; 2327 #ifdef CRYPTO_AES_XTS 2328 case CRYPTO_AES_XTS: 2329 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 2330 break; 2331 #endif 2332 default: 2333 panic("unknown algorithm %d", crd->crd_alg); 2334 } 2335 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 2336 2337 /* Set the CTR width. */ 2338 switch (crd->crd_alg) { 2339 case CRYPTO_AES_CTR: 2340 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 2341 break; 2342 } 2343 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 2344 2345 /* Set the AES key size. */ 2346 switch (crd->crd_alg) { 2347 case CRYPTO_AES_CBC: 2348 #ifdef CRYPTO_AES_ECB 2349 case CRYPTO_AES_ECB: 2350 #endif 2351 switch (crd->crd_klen) { 2352 case 128: 2353 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2354 break; 2355 case 192: 2356 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2357 break; 2358 case 256: 2359 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2360 break; 2361 default: 2362 panic("invalid AES key size in bits: %u", 2363 crd->crd_klen); 2364 } 2365 break; 2366 case CRYPTO_AES_CTR: 2367 switch (crd->crd_klen) { 2368 case 128 + 32: 2369 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2370 break; 2371 case 192 + 32: 2372 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2373 break; 2374 case 256 + 32: 2375 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2376 break; 2377 default: 2378 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 2379 crd->crd_klen); 2380 } 2381 break; 2382 #ifdef CRYPTO_AES_XTS 2383 case CRYPTO_AES_XTS: 2384 switch (crd->crd_klen) { 2385 case 256: 2386 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2387 break; 2388 case 384: 2389 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2390 break; 2391 case 512: 2392 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2393 break; 2394 default: 2395 panic("invalid AES-XTS key size in bits: %u", 2396 crd->crd_klen); 2397 } 2398 break; 2399 #endif 2400 } 2401 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 2402 2403 /* Set up the task descriptor. */ 2404 error = sun8i_crypto_task_load(sc, task, crd->crd_len, 2405 tdqc, tdqs, tdqa); 2406 if (error) 2407 goto fail2; 2408 2409 /* Submit! */ 2410 error = sun8i_crypto_submit(sc, task); 2411 if (error) 2412 goto fail2; 2413 2414 /* Success! */ 2415 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 2416 return 0; 2417 2418 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2419 fail1: if (task->ct_flags & TASK_SRC) 2420 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2421 if (task->ct_flags & TASK_CTR) 2422 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2423 if (task->ct_flags & TASK_IV) 2424 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2425 if (task->ct_flags & TASK_KEY) 2426 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2427 sun8i_crypto_task_put(sc, task); 2428 fail0: KASSERT(error); 2429 KASSERT(error != ERESTART); 2430 crp->crp_etype = error; 2431 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2432 crypto_done(crp); 2433 return 0; 2434 } 2435 2436 /* 2437 * sun8i_crypto_callback(sc, task, cookie, error) 2438 * 2439 * Completion callback for a task submitted via opencrypto. 2440 * Release the task and pass the error on to opencrypto with 2441 * crypto_done. 2442 */ 2443 static void 2444 sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 2445 struct sun8i_crypto_task *task, void *cookie, int error) 2446 { 2447 struct cryptop *crp = cookie; 2448 struct cryptodesc *crd __diagused = crp->crp_desc; 2449 2450 KASSERT(error != ERESTART); 2451 KASSERT(crd != NULL); 2452 KASSERT(crd->crd_next == NULL); 2453 2454 /* Return the number of bytes processed. */ 2455 crp->crp_olen = error ? 0 : crp->crp_ilen; 2456 2457 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2458 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2459 if (task->ct_flags & TASK_CTR) 2460 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2461 if (task->ct_flags & TASK_IV) 2462 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2463 if (task->ct_flags & TASK_KEY) 2464 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2465 sun8i_crypto_task_put(sc, task); 2466 KASSERT(error != ERESTART); 2467 crp->crp_etype = error; 2468 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2469 crypto_done(crp); 2470 } 2471
Indexes created Thu Sep 25 16:09:42 GMT 2025