sun8i_crypto.c revision 1.20
1 /* $NetBSD: sun8i_crypto.c,v 1.20 2021/01/18 02:35:49 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * sun8i_crypto -- Allwinner Crypto Engine driver 34 * 35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37 * the moment, but in principle it could be wired up with opencrypto(9) 38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41 * 42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43 */ 44 45 #include <sys/cdefs.h> 46 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.20 2021/01/18 02:35:49 thorpej Exp $"); 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/atomic.h> 51 #include <sys/bus.h> 52 #include <sys/callout.h> 53 #include <sys/conf.h> 54 #include <sys/cprng.h> 55 #include <sys/device.h> 56 #include <sys/kernel.h> 57 #include <sys/kmem.h> 58 #include <sys/mbuf.h> 59 #include <sys/mutex.h> 60 #include <sys/rndsource.h> 61 #include <sys/sdt.h> 62 #include <sys/sysctl.h> 63 #include <sys/workqueue.h> 64 65 #include <dev/fdt/fdtvar.h> 66 67 #include <opencrypto/cryptodev.h> 68 69 #include <arm/sunxi/sun8i_crypto.h> 70 71 #define SUN8I_CRYPTO_TIMEOUT hz 72 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 73 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 74 75 struct sun8i_crypto_task; 76 77 struct sun8i_crypto_buf { 78 bus_dma_segment_t cb_seg[1]; 79 int cb_nsegs; 80 void *cb_kva; 81 }; 82 83 struct sun8i_crypto_softc { 84 device_t sc_dev; 85 bus_space_tag_t sc_bst; 86 bus_space_handle_t sc_bsh; 87 bus_dma_tag_t sc_dmat; 88 struct pool_cache *sc_taskpool; 89 kmutex_t sc_lock; 90 struct sun8i_crypto_chan { 91 struct sun8i_crypto_task *cc_task; 92 unsigned cc_starttime; 93 } sc_chan[SUN8I_CRYPTO_NCHAN]; 94 struct callout sc_timeout; 95 struct workqueue *sc_wq; 96 struct work sc_work; 97 void *sc_ih; 98 uint32_t sc_done; 99 uint32_t sc_esr; 100 bool sc_work_pending; 101 struct sun8i_crypto_rng { 102 struct sun8i_crypto_buf cr_buf; 103 struct sun8i_crypto_task *cr_task; 104 struct krndsource cr_rndsource; 105 bool cr_pending; 106 } sc_rng; 107 struct sun8i_crypto_selftest { 108 struct sun8i_crypto_buf cs_in; 109 struct sun8i_crypto_buf cs_key; 110 struct sun8i_crypto_buf cs_out; 111 struct sun8i_crypto_task *cs_task; 112 } sc_selftest; 113 struct sun8i_crypto_sysctl { 114 struct sysctllog *cy_log; 115 const struct sysctlnode *cy_root_node; 116 const struct sysctlnode *cy_trng_node; 117 } sc_sysctl; 118 struct sun8i_crypto_opencrypto { 119 uint32_t co_driverid; 120 } sc_opencrypto; 121 }; 122 123 struct sun8i_crypto_task { 124 struct sun8i_crypto_buf ct_descbuf; 125 struct sun8i_crypto_taskdesc *ct_desc; 126 struct sun8i_crypto_buf ct_ivbuf; 127 void *ct_iv; 128 struct sun8i_crypto_buf ct_ctrbuf; 129 void *ct_ctr; 130 bus_dmamap_t ct_descmap; 131 bus_dmamap_t ct_keymap; 132 bus_dmamap_t ct_ivmap; /* IV input */ 133 bus_dmamap_t ct_ctrmap; /* updated IV output */ 134 bus_dmamap_t ct_srcmap; 135 bus_dmamap_t ct_dstmap; 136 uint32_t ct_nbytes; 137 int ct_flags; 138 #define TASK_KEY __BIT(0) 139 #define TASK_IV __BIT(1) 140 #define TASK_CTR __BIT(2) 141 #define TASK_SRC __BIT(3) 142 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 143 void (*ct_callback)(struct sun8i_crypto_softc *, 144 struct sun8i_crypto_task *, void *, int); 145 void *ct_cookie; 146 }; 147 148 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 149 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 150 151 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 152 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 153 154 /* 155 * Forward declarations 156 */ 157 158 static int sun8i_crypto_match(device_t, cfdata_t, void *); 159 static void sun8i_crypto_attach(device_t, device_t, void *); 160 161 static int sun8i_crypto_task_ctor(void *, void *, int); 162 static void sun8i_crypto_task_dtor(void *, void *); 163 static struct sun8i_crypto_task * 164 sun8i_crypto_task_get(struct sun8i_crypto_softc *, 165 void (*)(struct sun8i_crypto_softc *, 166 struct sun8i_crypto_task *, void *, int), 167 void *, int); 168 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 169 struct sun8i_crypto_task *); 170 171 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 172 struct sun8i_crypto_task *, uint32_t, 173 uint32_t, uint32_t, uint32_t); 174 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 175 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 176 177 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 178 struct sun8i_crypto_task *, uint32_t); 179 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 180 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 181 182 static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 183 struct sun8i_crypto_task *); 184 185 static void sun8i_crypto_timeout(void *); 186 static int sun8i_crypto_intr(void *); 187 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 188 static void sun8i_crypto_worker(struct work *, void *); 189 static void sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 190 int); 191 192 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 193 struct sun8i_crypto_buf *, int); 194 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 195 struct sun8i_crypto_buf *); 196 197 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 198 static void sun8i_crypto_rng_get(size_t, void *); 199 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 200 struct sun8i_crypto_task *, void *, int); 201 202 static void sun8i_crypto_selftest(device_t); 203 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 204 struct sun8i_crypto_task *, void *, int); 205 206 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 207 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 208 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 209 struct sun8i_crypto_task *, void *, int); 210 211 static void sun8i_crypto_register(struct sun8i_crypto_softc *); 212 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 213 static int sun8i_crypto_newsession(void *, uint32_t *, 214 struct cryptoini *); 215 static int sun8i_crypto_freesession(void *, uint64_t); 216 static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 217 static int sun8i_crypto_process(void *, struct cryptop *, int); 218 static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 219 struct sun8i_crypto_task *, void *, int); 220 221 /* 222 * Probes 223 */ 224 225 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 226 "bus_size_t"/*reg*/, 227 "uint32_t"/*value*/); 228 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 229 "bus_size_t"/*reg*/, 230 "uint32_t"/*write*/); 231 232 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 233 "struct sun8i_crypto_task *"/*task*/); 234 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 235 "int"/*error*/); 236 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 237 "struct sun8i_crypto_task *"/*task*/); 238 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 239 "struct sun8i_crypto_task *"/*task*/); 240 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 241 "struct sun8i_crypto_task *"/*task*/); 242 243 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 244 "struct sun8i_crypto_task *"/*task*/, 245 "uint32_t"/*tdqc*/, 246 "uint32_t"/*tdqs*/, 247 "uint32_t"/*tdqa*/, 248 "struct sun8i_crypto_taskdesc *"/*desc*/, 249 "int"/*error*/); 250 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 251 "struct sun8i_crypto_task *"/*task*/, 252 "bus_addr_t"/*ds_addr*/, 253 "bus_size_t"/*ds_len*/); 254 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 255 "struct sun8i_crypto_task *"/*task*/, 256 "int"/*error*/); 257 258 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 259 "struct sun8i_crypto_softc *"/*sc*/, 260 "struct sun8i_crypto_task *"/*task*/, 261 "int"/*error*/); 262 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 263 "struct sun8i_crypto_softc *"/*sc*/, 264 "struct sun8i_crypto_task *"/*task*/, 265 "unsigned"/*chan*/); 266 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 267 "struct sun8i_crypto_softc *"/*sc*/, 268 "uint32_t"/*isr*/, 269 "uint32_t"/*esr*/); 270 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 271 "struct sun8i_crypto_softc *"/*sc*/, 272 "unsigned"/*chan*/, 273 "int"/*error*/); 274 275 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 276 "struct sun8i_crypto_softc *"/*sc*/, 277 "struct cryptop *"/*crp*/, 278 "int"/*hint*/); 279 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 280 "struct sun8i_crypto_softc *"/*sc*/, 281 "struct cryptop *"/*crp*/, 282 "int"/*hint*/); 283 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 284 "struct sun8i_crypto_softc *"/*sc*/, 285 "struct cryptop *"/*crp*/, 286 "int"/*hint*/, 287 "struct sun8i_crypto_task *"/*task*/); 288 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 289 "struct sun8i_crypto_softc *"/*sc*/, 290 "struct cryptop *"/*crp*/, 291 "int"/*error*/); 292 293 /* 294 * Register access 295 */ 296 297 static uint32_t 298 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 299 { 300 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 301 302 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 303 return v; 304 } 305 306 static void 307 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 308 { 309 310 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 311 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 312 } 313 314 /* 315 * Autoconf goo 316 */ 317 318 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 319 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 320 321 static const struct device_compatible_entry compat_data[] = { 322 { .compat = "allwinner,sun50i-a64-crypto" }, 323 324 { 0 } 325 }; 326 327 static int 328 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 329 { 330 const struct fdt_attach_args *const faa = aux; 331 332 return of_match_compat_data(faa->faa_phandle, compat_data); 333 } 334 335 static void 336 sun8i_crypto_attach(device_t parent, device_t self, void *aux) 337 { 338 struct sun8i_crypto_softc *const sc = device_private(self); 339 const struct fdt_attach_args *const faa = aux; 340 bus_addr_t addr; 341 bus_size_t size; 342 const int phandle = faa->faa_phandle; 343 char intrstr[128]; 344 struct clk *clk; 345 struct fdtbus_reset *rst; 346 347 sc->sc_dev = self; 348 sc->sc_dmat = faa->faa_dmat; 349 sc->sc_bst = faa->faa_bst; 350 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 351 0, 0, 0, "sun8icry", NULL, IPL_VM, 352 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 353 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM); 354 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 355 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 356 if (workqueue_create(&sc->sc_wq, device_xname(self), 357 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 358 aprint_error(": couldn't create workqueue\n"); 359 return; 360 } 361 362 /* 363 * Prime the pool with enough tasks that each channel can be 364 * busy with a task as we prepare another task for when it's 365 * done. 366 */ 367 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 368 369 /* Get and map device registers. */ 370 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 371 aprint_error(": couldn't get registers\n"); 372 return; 373 } 374 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 375 aprint_error(": couldn't map registers\n"); 376 return; 377 } 378 379 /* Get an interrupt handle. */ 380 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 381 aprint_error(": failed to decode interrupt\n"); 382 return; 383 } 384 385 /* Enable the bus clock. */ 386 if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 387 aprint_error(": couldn't enable bus clock\n"); 388 return; 389 } 390 391 /* Get the module clock and set it to 300 MHz. */ 392 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 393 if (clk_enable(clk) != 0) { 394 aprint_error(": couldn't enable CE clock\n"); 395 return; 396 } 397 if (clk_set_rate(clk, 300*1000*1000) != 0) { 398 aprint_error(": couldn't set CE clock to 300MHz\n"); 399 return; 400 } 401 } 402 403 /* Get a reset handle if we need and try to deassert it. */ 404 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 405 if (fdtbus_reset_deassert(rst) != 0) { 406 aprint_error(": couldn't de-assert reset\n"); 407 return; 408 } 409 } 410 411 aprint_naive("\n"); 412 aprint_normal(": Crypto Engine\n"); 413 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 414 415 /* Disable and clear interrupts. */ 416 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 417 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 418 419 /* Establish an interrupt handler. */ 420 sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM, 421 FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self)); 422 if (sc->sc_ih == NULL) { 423 aprint_error_dev(self, "failed to establish interrupt on %s\n", 424 intrstr); 425 return; 426 } 427 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 428 429 /* Set up the RNG. */ 430 sun8i_crypto_rng_attach(sc); 431 432 /* Attach the sysctl. */ 433 sun8i_crypto_sysctl_attach(sc); 434 435 /* Perform self-tests. */ 436 config_interrupts(self, sun8i_crypto_selftest); 437 438 /* Register opencrypto handlers. */ 439 sun8i_crypto_register(sc); 440 } 441 442 static int 443 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 444 { 445 struct sun8i_crypto_softc *sc = cookie; 446 struct sun8i_crypto_task *task = vtask; 447 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 448 int error; 449 450 /* Create a DMA buffer for the task descriptor. */ 451 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 452 &task->ct_descbuf, dmaflags); 453 if (error) 454 goto fail0; 455 task->ct_desc = task->ct_descbuf.cb_kva; 456 457 /* Create DMA buffers for the IV and CTR. */ 458 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 459 &task->ct_ivbuf, dmaflags); 460 if (error) 461 goto fail1; 462 task->ct_iv = task->ct_ivbuf.cb_kva; 463 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 464 &task->ct_ctrbuf, dmaflags); 465 if (error) 466 goto fail2; 467 task->ct_ctr = task->ct_ctrbuf.cb_kva; 468 469 /* Create a DMA map for the task descriptor and preload it. */ 470 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 471 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 472 if (error) 473 goto fail3; 474 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 475 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 476 if (error) 477 goto fail4; 478 479 /* Create DMA maps for the key, IV, and CTR. */ 480 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 481 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 482 if (error) 483 goto fail5; 484 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 485 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 486 if (error) 487 goto fail6; 488 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 489 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 490 if (error) 491 goto fail7; 492 493 /* Create DMA maps for the src and dst scatter/gather vectors. */ 494 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 495 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 496 &task->ct_srcmap); 497 if (error) 498 goto fail8; 499 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 500 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 501 &task->ct_dstmap); 502 if (error) 503 goto fail9; 504 505 /* Success! */ 506 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 507 return 0; 508 509 fail10: __unused 510 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 511 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 512 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 513 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 514 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 515 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 516 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 517 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 518 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 519 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 520 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 521 return error; 522 } 523 524 static void 525 sun8i_crypto_task_dtor(void *cookie, void *vtask) 526 { 527 struct sun8i_crypto_softc *sc = cookie; 528 struct sun8i_crypto_task *task = vtask; 529 530 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 531 532 /* XXX Zero the bounce buffers if there are any. */ 533 534 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 535 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 536 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 537 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 538 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 539 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 540 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 541 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 542 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 543 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 544 } 545 546 /* 547 * sun8i_crypto_task_get(sc, callback, cookie, pflags) 548 * 549 * Allocate a task that will call callback(sc, task, cookie, 550 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 551 * PR_NOWAIT, may fail and return NULL. No further allocation is 552 * needed to submit the task if this succeeds (although task 553 * submission may still fail if all channels are busy). 554 */ 555 static struct sun8i_crypto_task * 556 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 557 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 558 void *, int), 559 void *cookie, int pflags) 560 { 561 struct sun8i_crypto_task *task; 562 563 /* Allocate a task, or fail if we can't. */ 564 task = pool_cache_get(sc->sc_taskpool, pflags); 565 if (task == NULL) 566 goto out; 567 568 /* Set up flags and the callback. */ 569 task->ct_flags = 0; 570 task->ct_callback = callback; 571 task->ct_cookie = cookie; 572 573 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 574 return task; 575 } 576 577 /* 578 * sun8i_crypto_task_invalid(sc, task, cookie, error) 579 * 580 * Callback for a task not currently in use, to detect errors. 581 */ 582 static void 583 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 584 struct sun8i_crypto_task *task, void *cookie, int error) 585 { 586 void (*callback)(struct sun8i_crypto_softc *, 587 struct sun8i_crypto_task *, void *, int) = cookie; 588 589 panic("task for callback %p used after free", callback); 590 } 591 592 /* 593 * sun8i_crypto_task_put(sc, task) 594 * 595 * Free a task obtained with sun8i_crypto_task_get. 596 */ 597 static void 598 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 599 struct sun8i_crypto_task *task) 600 { 601 602 SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 603 604 task->ct_cookie = task->ct_callback; 605 task->ct_callback = &sun8i_crypto_task_invalid; 606 pool_cache_put(sc->sc_taskpool, task); 607 } 608 609 /* 610 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 611 * 612 * Set up the task descriptor after the relevant DMA maps have 613 * been loaded for a transfer of nbytes. bus_dmamap_sync matches 614 * sun8i_crypto_chan_done. May fail if input is inadequately 615 * aligned. 616 * 617 * XXX Teach this to support task chains. 618 */ 619 static int 620 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 621 struct sun8i_crypto_task *task, uint32_t nbytes, 622 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 623 { 624 struct sun8i_crypto_taskdesc *desc = task->ct_desc; 625 int error; 626 627 KASSERT(tdqs == 0 || tdqa == 0); 628 KASSERT(nbytes % 4 == 0); 629 630 memset(desc, 0, sizeof(*desc)); 631 632 /* Always enable interrupt for the task. */ 633 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 634 635 desc->td_tdqc = htole32(tdqc); 636 desc->td_tdqs = htole32(tdqs); 637 desc->td_tdqa = htole32(tdqa); 638 639 if (task->ct_flags & TASK_KEY) { 640 bus_dmamap_t keymap = task->ct_keymap; 641 KASSERT(keymap->dm_nsegs == 1); 642 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 643 bus_dmamap_sync(sc->sc_dmat, keymap, 0, 644 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 645 } 646 if (task->ct_flags & TASK_IV) { 647 bus_dmamap_t ivmap = task->ct_ivmap; 648 KASSERT(ivmap->dm_nsegs == 1); 649 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 650 bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 651 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 652 } 653 if (task->ct_flags & TASK_CTR) { 654 bus_dmamap_t ctrmap = task->ct_ctrmap; 655 KASSERT(ctrmap->dm_nsegs == 1); 656 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 657 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 658 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 659 } 660 661 if (task->ct_flags & TASK_BYTES) 662 desc->td_datalen = htole32(nbytes); 663 else 664 desc->td_datalen = htole32(nbytes/4); 665 666 if (task->ct_flags & TASK_SRC) { 667 bus_dmamap_t srcmap = task->ct_srcmap; 668 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 669 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 670 nbytes); 671 if (error) 672 return error; 673 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 674 BUS_DMASYNC_PREWRITE); 675 } 676 677 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 678 nbytes); 679 if (error) 680 goto out; 681 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 682 BUS_DMASYNC_PREREAD); 683 684 task->ct_nbytes = nbytes; 685 686 /* Success! */ 687 error = 0; 688 689 out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 690 task, tdqc, tdqs, tdqa, desc, error); 691 return error; 692 } 693 694 /* 695 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 696 * 697 * Set up a task's scatter/gather vector -- src or dst -- with the 698 * given DMA map for a transfer of nbytes. May fail if input is 699 * inadequately aligned. 700 */ 701 static int 702 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 703 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 704 uint32_t nbytes __diagused) 705 { 706 uint32_t total __diagused = 0; 707 unsigned i; 708 709 /* 710 * Verify that the alignment is correct and initialize the 711 * scatter/gather vector. 712 */ 713 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 714 for (i = 0; i < map->dm_nsegs; i++) { 715 if ((map->dm_segs[i].ds_addr % 4) | 716 (map->dm_segs[i].ds_len % 4)) { 717 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 718 task, 719 map->dm_segs[i].ds_addr, 720 map->dm_segs[i].ds_len); 721 return EINVAL; 722 } 723 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 724 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 725 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 726 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 727 total += map->dm_segs[i].ds_len; 728 } 729 730 /* Set the remainder to zero. */ 731 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 732 adrlen[i].adr = 0; 733 adrlen[i].len = 0; 734 } 735 736 /* Verify the total size matches the transfer length. */ 737 KASSERT(total == nbytes); 738 739 /* Success! */ 740 return 0; 741 } 742 743 /* 744 * sun8i_crypto_task_load_trng(task, nbytes) 745 * 746 * Set up the task descriptor for a transfer of nbytes from the 747 * TRNG. 748 */ 749 static int 750 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 751 struct sun8i_crypto_task *task, uint32_t nbytes) 752 { 753 uint32_t tdqc = 0; 754 755 /* Caller must provide dst only. */ 756 KASSERT((task->ct_flags & TASK_KEY) == 0); 757 KASSERT((task->ct_flags & TASK_IV) == 0); 758 KASSERT((task->ct_flags & TASK_CTR) == 0); 759 KASSERT((task->ct_flags & TASK_SRC) == 0); 760 761 /* Set up the task descriptor queue control words. */ 762 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 763 SUN8I_CRYPTO_TDQC_METHOD); 764 765 /* Fill in the descriptor. */ 766 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 767 } 768 769 static int 770 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 771 struct sun8i_crypto_task *task, 772 uint32_t nbytes, uint32_t keysize, uint32_t dir) 773 { 774 uint32_t tdqc = 0, tdqs = 0; 775 776 /* Caller must provide key, src, and dst only. */ 777 KASSERT(task->ct_flags & TASK_KEY); 778 KASSERT((task->ct_flags & TASK_IV) == 0); 779 KASSERT((task->ct_flags & TASK_CTR) == 0); 780 KASSERT(task->ct_flags & TASK_SRC); 781 782 /* Set up the task descriptor queue control word. */ 783 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 784 SUN8I_CRYPTO_TDQC_METHOD); 785 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 786 787 #ifdef DIAGNOSTIC 788 switch (keysize) { 789 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 790 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 791 break; 792 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 793 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 794 break; 795 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 796 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 797 break; 798 } 799 #endif 800 801 /* Set up the symmetric control word. */ 802 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 803 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 804 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 805 SUN8I_CRYPTO_TDQS_OP_MODE); 806 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 807 808 /* Fill in the descriptor. */ 809 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 810 } 811 812 /* 813 * sun8i_crypto_submit(sc, task) 814 * 815 * Submit a task to the crypto engine after it has been loaded 816 * with sun8i_crypto_task_load. On success, guarantees to 817 * eventually call the task's callback. 818 */ 819 static int 820 sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 821 struct sun8i_crypto_task *task) 822 { 823 unsigned i, retries = 0; 824 uint32_t icr; 825 int error = 0; 826 827 /* One at a time at the device registers, please. */ 828 mutex_enter(&sc->sc_lock); 829 830 /* Find a channel. */ 831 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 832 if (sc->sc_chan[i].cc_task == NULL) 833 break; 834 } 835 if (i == SUN8I_CRYPTO_NCHAN) { 836 device_printf(sc->sc_dev, "no free channels\n"); 837 error = ERESTART; 838 goto out; 839 } 840 841 /* 842 * Set the channel id. Caller is responsible for setting up 843 * all other parts of the descriptor. 844 */ 845 task->ct_desc->td_cid = htole32(i); 846 847 /* 848 * Prepare to send the descriptor to the device by DMA. 849 * Matches POSTWRITE in sun8i_crypto_chan_done. 850 */ 851 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 852 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 853 854 /* Confirm we're ready to go. */ 855 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 856 device_printf(sc->sc_dev, "TLR not clear\n"); 857 error = EIO; 858 goto out; 859 } 860 861 /* Enable interrupts for this channel. */ 862 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 863 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 864 SUN8I_CRYPTO_ICR_INTR_EN); 865 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 866 867 /* Set the task descriptor queue address. */ 868 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 869 task->ct_descmap->dm_segs[0].ds_addr); 870 871 /* Notify the engine to load it, and wait for acknowledgement. */ 872 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 873 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 874 { 875 /* 876 * XXX Timeout pulled from arse. Is it even important 877 * to wait here? 878 */ 879 if (++retries == 1000) { 880 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 881 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 882 /* 883 * Hope it clears eventually; if not, we'll 884 * time out. 885 */ 886 break; 887 } 888 DELAY(1); 889 } 890 891 /* 892 * Loaded up and ready to go. Start a timer ticking if it's 893 * not already. 894 */ 895 sc->sc_chan[i].cc_task = task; 896 sc->sc_chan[i].cc_starttime = getticks(); 897 if (!callout_pending(&sc->sc_timeout)) 898 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 899 900 /* XXX Consider polling if cold to get entropy earlier. */ 901 902 out: /* Done! */ 903 if (error) 904 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 905 sc, task, error); 906 else 907 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 908 sc, task, i); 909 mutex_exit(&sc->sc_lock); 910 return error; 911 } 912 913 /* 914 * sun8i_crypto_timeout(cookie) 915 * 916 * Timeout handler. Schedules work in a thread to cancel all 917 * pending tasks that were started long enough ago we're bored of 918 * waiting for them, and reschedules another timeout unless the 919 * channels are all idle. 920 */ 921 static void 922 sun8i_crypto_timeout(void *cookie) 923 { 924 struct sun8i_crypto_softc *sc = cookie; 925 unsigned i; 926 927 mutex_enter(&sc->sc_lock); 928 929 /* Check whether there are any tasks pending. */ 930 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 931 if (sc->sc_chan[i].cc_task) 932 break; 933 } 934 if (i == SUN8I_CRYPTO_NCHAN) 935 /* None pending, so nothing to do. */ 936 goto out; 937 938 /* 939 * Schedule the worker to check for timeouts, and schedule 940 * another timeout in case we need it. 941 */ 942 sun8i_crypto_schedule_worker(sc); 943 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 944 945 out: mutex_exit(&sc->sc_lock); 946 } 947 948 /* 949 * sun8i_crypto_intr(cookie) 950 * 951 * Device interrupt handler. Find what channels have completed, 952 * whether with success or with failure, and schedule work in 953 * thread context to invoke the appropriate callbacks. 954 */ 955 static int 956 sun8i_crypto_intr(void *cookie) 957 { 958 struct sun8i_crypto_softc *sc = cookie; 959 uint32_t isr, esr; 960 961 mutex_enter(&sc->sc_lock); 962 963 /* 964 * Get and acknowledge the interrupts and error status. 965 * 966 * XXX Data sheet says the error status register is read-only, 967 * but then advises writing 1 to bit x1xx (keysram access error 968 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 969 */ 970 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 971 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 972 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 973 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 974 975 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 976 977 /* Start the worker if necessary. */ 978 sun8i_crypto_schedule_worker(sc); 979 980 /* Tell the worker what to do. */ 981 sc->sc_done |= __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 982 sc->sc_esr |= esr; 983 984 mutex_exit(&sc->sc_lock); 985 986 return __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE) != 0; 987 } 988 989 /* 990 * sun8i_crypto_schedule_worker(sc) 991 * 992 * Ensure that crypto engine thread context work to invoke task 993 * callbacks will run promptly. Idempotent. 994 */ 995 static void 996 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 997 { 998 999 KASSERT(mutex_owned(&sc->sc_lock)); 1000 1001 /* Start the worker if necessary. */ 1002 if (!sc->sc_work_pending) { 1003 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1004 sc->sc_work_pending = true; 1005 } 1006 } 1007 1008 /* 1009 * sun8i_crypto_worker(wk, cookie) 1010 * 1011 * Thread-context worker: Invoke all task callbacks for which the 1012 * device has notified us of completion or for which we gave up 1013 * waiting. 1014 */ 1015 static void 1016 sun8i_crypto_worker(struct work *wk, void *cookie) 1017 { 1018 struct sun8i_crypto_softc *sc = cookie; 1019 uint32_t done, esr, esr_chan; 1020 unsigned i, now; 1021 int error; 1022 1023 /* 1024 * Acquire the lock. Note: We will be releasing and 1025 * reacquiring it throughout the loop. 1026 */ 1027 mutex_enter(&sc->sc_lock); 1028 1029 /* Acknowledge the work. */ 1030 KASSERT(sc->sc_work_pending); 1031 sc->sc_work_pending = false; 1032 1033 /* 1034 * Claim the done mask and error status once; we will be 1035 * releasing and reacquiring the lock for the callbacks, so 1036 * they may change. 1037 */ 1038 done = sc->sc_done; 1039 esr = sc->sc_esr; 1040 sc->sc_done = 0; 1041 sc->sc_esr = 0; 1042 1043 /* Check the time to determine what's timed out. */ 1044 now = getticks(); 1045 1046 /* Process the channels. */ 1047 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 1048 /* Check whether the channel is done. */ 1049 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 1050 /* Nope. Do we have a task to time out? */ 1051 if ((sc->sc_chan[i].cc_task != NULL) && 1052 ((now - sc->sc_chan[i].cc_starttime) >= 1053 SUN8I_CRYPTO_TIMEOUT)) 1054 sun8i_crypto_chan_done(sc, i, ETIMEDOUT); 1055 continue; 1056 } 1057 1058 /* Channel is done. Interpret the error if any. */ 1059 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1060 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1061 device_printf(sc->sc_dev, "channel %u:" 1062 " alg not supported\n", i); 1063 error = ENODEV; 1064 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1065 device_printf(sc->sc_dev, "channel %u:" 1066 " data length error\n", i); 1067 error = EIO; /* XXX */ 1068 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1069 device_printf(sc->sc_dev, "channel %u:" 1070 " key sram error\n", i); 1071 error = EIO; /* XXX */ 1072 } else if (esr_chan != 0) { 1073 error = EIO; /* generic I/O error */ 1074 } else { 1075 error = 0; 1076 } 1077 1078 /* 1079 * Notify the task of completion. May release the lock 1080 * to invoke a callback. 1081 */ 1082 sun8i_crypto_chan_done(sc, i, error); 1083 } 1084 1085 /* All one; release the lock one last time. */ 1086 mutex_exit(&sc->sc_lock); 1087 } 1088 1089 /* 1090 * sun8i_crypto_chan_done(sc, i, error) 1091 * 1092 * Notify the callback for the task on channel i, if there is one, 1093 * of the specified error, or 0 for success. 1094 */ 1095 static void 1096 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1097 { 1098 struct sun8i_crypto_task *task; 1099 uint32_t nbytes; 1100 uint32_t icr; 1101 1102 KASSERT(mutex_owned(&sc->sc_lock)); 1103 1104 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 1105 1106 /* Claim the task if there is one; bail if not. */ 1107 if ((task = sc->sc_chan[i].cc_task) == NULL) { 1108 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1109 i, error); 1110 return; 1111 } 1112 sc->sc_chan[i].cc_task = NULL; 1113 1114 /* Disable interrupts on this channel. */ 1115 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1116 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1117 SUN8I_CRYPTO_ICR_INTR_EN); 1118 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1119 1120 /* 1121 * Finished sending the descriptor to the device by DMA. 1122 * Matches PREWRITE in sun8i_crypto_task_submit. 1123 */ 1124 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1125 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1126 1127 /* 1128 * Finished with all the other bits of DMA too. Matches 1129 * sun8i_crypto_task_load. 1130 */ 1131 nbytes = task->ct_nbytes; 1132 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 1133 BUS_DMASYNC_POSTREAD); 1134 if (task->ct_flags & TASK_SRC) 1135 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 1136 BUS_DMASYNC_POSTWRITE); 1137 if (task->ct_flags & TASK_CTR) 1138 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 1139 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 1140 if (task->ct_flags & TASK_IV) 1141 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 1142 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1143 if (task->ct_flags & TASK_KEY) 1144 /* XXX Can we zero the bounce buffer if there is one? */ 1145 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 1146 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1147 1148 /* Temporarily release the lock to invoke the callback. */ 1149 mutex_exit(&sc->sc_lock); 1150 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1151 (*task->ct_callback)(sc, task, task->ct_cookie, error); 1152 mutex_enter(&sc->sc_lock); 1153 } 1154 1155 /* 1156 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 1157 * 1158 * Allocate a single-segment DMA-safe buffer and map it into KVA. 1159 * May fail if dmaflags is BUS_DMA_NOWAIT. 1160 */ 1161 static int 1162 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 1163 struct sun8i_crypto_buf *buf, int dmaflags) 1164 { 1165 int error; 1166 1167 /* Allocate a DMA-safe buffer. */ 1168 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 1169 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1170 if (error) 1171 goto fail0; 1172 1173 /* Map the buffer into kernel virtual address space. */ 1174 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 1175 size, &buf->cb_kva, dmaflags); 1176 if (error) 1177 goto fail1; 1178 1179 /* Success! */ 1180 return 0; 1181 1182 fail2: __unused 1183 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1184 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1185 fail0: return error; 1186 } 1187 1188 /* 1189 * sun8i_crypto_freebuf(sc, buf) 1190 * 1191 * Unmap buf and free it. 1192 */ 1193 static void 1194 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1195 struct sun8i_crypto_buf *buf) 1196 { 1197 1198 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1199 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1200 } 1201 1202 /* 1203 * sun8i_crypto_rng_attach(sc) 1204 * 1205 * Attach an rndsource for the crypto engine's TRNG. 1206 */ 1207 static void 1208 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1209 { 1210 device_t self = sc->sc_dev; 1211 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1212 struct sun8i_crypto_task *task; 1213 int error; 1214 1215 /* Preallocate a buffer to reuse. */ 1216 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 1217 BUS_DMA_WAITOK); 1218 if (error) { 1219 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 1220 error); 1221 goto fail0; 1222 } 1223 1224 /* Create a task to reuse. */ 1225 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 1226 rng, PR_WAITOK); 1227 if (rng->cr_task == NULL) { 1228 aprint_error_dev(self, "failed to allocate RNG task\n"); 1229 error = ENOMEM; 1230 goto fail1; 1231 } 1232 1233 /* Preload the destination map. */ 1234 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1235 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1236 if (error) { 1237 aprint_error_dev(self, "failed to load RNG buffer: %d\n", 1238 error); 1239 goto fail2; 1240 } 1241 1242 /* 1243 * Attach the rndsource. This is _not_ marked as RND_TYPE_RNG 1244 * because the output is not uniformly distributed. The bits 1245 * are heavily weighted toward 0 or 1, at different times, and 1246 * I haven't scienced a satisfactory story out of it yet. 1247 */ 1248 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1249 rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1250 RND_TYPE_UNKNOWN, 1251 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1252 1253 /* Success! */ 1254 return; 1255 1256 fail3: __unused 1257 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1258 fail2: sun8i_crypto_task_put(sc, task); 1259 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 1260 fail0: return; 1261 } 1262 1263 /* 1264 * sun8i_crypto_rng_get(nbytes, cookie) 1265 * 1266 * On-demand rndsource callback: try to gather nbytes of entropy 1267 * and enter them into the pool ASAP. 1268 */ 1269 static void 1270 sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1271 { 1272 struct sun8i_crypto_softc *sc = cookie; 1273 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1274 struct sun8i_crypto_task *task = rng->cr_task; 1275 bool pending; 1276 int error; 1277 1278 /* 1279 * Test and set the RNG-pending flag. If it's already in 1280 * progress, nothing to do here. 1281 */ 1282 mutex_enter(&sc->sc_lock); 1283 pending = rng->cr_pending; 1284 rng->cr_pending = true; 1285 mutex_exit(&sc->sc_lock); 1286 if (pending) 1287 return; 1288 1289 /* Load the task descriptor. */ 1290 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1291 if (error) 1292 goto fail; 1293 1294 /* Submit! */ 1295 error = sun8i_crypto_submit(sc, task); 1296 if (error) 1297 goto fail; 1298 1299 /* All done! */ 1300 return; 1301 1302 fail: mutex_enter(&sc->sc_lock); 1303 rng->cr_pending = false; 1304 mutex_exit(&sc->sc_lock); 1305 } 1306 1307 static void 1308 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1309 struct sun8i_crypto_task *task, void *cookie, int error) 1310 { 1311 struct sun8i_crypto_rng *rng = cookie; 1312 uint8_t *buf = rng->cr_buf.cb_kva; 1313 uint32_t entropybits; 1314 1315 KASSERT(rng == &sc->sc_rng); 1316 1317 /* If anything went wrong, forget about it. */ 1318 if (error) 1319 goto out; 1320 1321 /* 1322 * This TRNG has quite low entropy at best. But if it fails a 1323 * repeated output test, then assume it's busted. 1324 */ 1325 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1326 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1327 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1328 SUN8I_CRYPTO_RNGBYTES/2)) { 1329 device_printf(sc->sc_dev, "failed repeated output test\n"); 1330 entropybits = 0; 1331 } 1332 1333 /* 1334 * Actually we don't believe in any of the entropy until this 1335 * device has had more scrutiny. 1336 */ 1337 entropybits = 0; 1338 1339 /* Success! Enter and erase the data. */ 1340 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1341 entropybits); 1342 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1343 1344 out: /* Done -- clear the RNG-pending flag. */ 1345 mutex_enter(&sc->sc_lock); 1346 rng->cr_pending = false; 1347 mutex_exit(&sc->sc_lock); 1348 } 1349 1350 /* 1351 * Self-test 1352 */ 1353 1354 static const uint8_t selftest_input[16]; 1355 static const uint8_t selftest_key[16]; 1356 static const uint8_t selftest_output[16] = { 1357 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1358 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1359 }; 1360 1361 static void 1362 sun8i_crypto_selftest(device_t self) 1363 { 1364 const size_t keybytes = sizeof selftest_key; 1365 const size_t nbytes = sizeof selftest_input; 1366 struct sun8i_crypto_softc *sc = device_private(self); 1367 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 1368 struct sun8i_crypto_task *task; 1369 int error; 1370 1371 CTASSERT(sizeof selftest_input == sizeof selftest_output); 1372 1373 /* Allocate an input buffer. */ 1374 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 1375 BUS_DMA_WAITOK); 1376 if (error) 1377 goto fail0; 1378 1379 /* Allocate a key buffer. */ 1380 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 1381 BUS_DMA_WAITOK); 1382 if (error) 1383 goto fail1; 1384 1385 /* Allocate an output buffer. */ 1386 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 1387 BUS_DMA_WAITOK); 1388 if (error) 1389 goto fail2; 1390 1391 /* Allocate a task descriptor. */ 1392 task = selftest->cs_task = sun8i_crypto_task_get(sc, 1393 sun8i_crypto_selftest_done, selftest, PR_WAITOK); 1394 if (selftest->cs_task == NULL) { 1395 error = ENOMEM; 1396 goto fail3; 1397 } 1398 1399 /* Copy the input and key into their buffers. */ 1400 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 1401 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1402 1403 /* Load the key, src, and dst for DMA transfers. */ 1404 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 1405 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1406 if (error) 1407 goto fail4; 1408 task->ct_flags |= TASK_KEY; 1409 1410 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 1411 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1412 if (error) 1413 goto fail5; 1414 task->ct_flags |= TASK_SRC; 1415 1416 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1417 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1418 if (error) 1419 goto fail6; 1420 1421 /* Set up the task descriptor. */ 1422 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 1423 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 1424 if (error) 1425 goto fail7; 1426 1427 /* Submit! */ 1428 error = sun8i_crypto_submit(sc, task); 1429 if (error) 1430 goto fail7; 1431 1432 device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1433 1434 /* Success! */ 1435 return; 1436 1437 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1438 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1439 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1440 fail4: sun8i_crypto_task_put(sc, task); 1441 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1442 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1443 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1444 fail0: aprint_error_dev(self, "failed to run self-test, error=%d\n", error); 1445 } 1446 1447 static bool 1448 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1449 size_t n, const void *expected, const void *actual) 1450 { 1451 const uint8_t *e = expected; 1452 const uint8_t *a = actual; 1453 size_t i; 1454 1455 if (memcmp(e, a, n) == 0) 1456 return true; 1457 1458 device_printf(sc->sc_dev, "self-test: %s\n", title); 1459 printf("expected: "); 1460 for (i = 0; i < n; i++) 1461 printf("%02hhx", e[i]); 1462 printf("\n"); 1463 printf("actual: "); 1464 for (i = 0; i < n; i++) 1465 printf("%02hhx", a[i]); 1466 printf("\n"); 1467 return false; 1468 } 1469 1470 static void 1471 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1472 struct sun8i_crypto_task *task, void *cookie, int error) 1473 { 1474 const size_t keybytes = sizeof selftest_key; 1475 const size_t nbytes = sizeof selftest_input; 1476 struct sun8i_crypto_selftest *selftest = cookie; 1477 bool ok = true; 1478 1479 KASSERT(selftest == &sc->sc_selftest); 1480 1481 /* If anything went wrong, fail now. */ 1482 if (error) { 1483 device_printf(sc->sc_dev, "self-test error=%d\n", error); 1484 goto out; 1485 } 1486 1487 /* 1488 * Verify the input and key weren't clobbered, and verify the 1489 * output matches what we expect. 1490 */ 1491 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 1492 selftest_input, selftest->cs_in.cb_kva); 1493 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 1494 selftest_key, selftest->cs_key.cb_kva); 1495 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 1496 selftest_output, selftest->cs_out.cb_kva); 1497 1498 /* XXX Disable the RNG and other stuff if this fails... */ 1499 if (ok) 1500 device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1501 1502 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1503 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1504 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1505 sun8i_crypto_task_put(sc, task); 1506 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1507 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1508 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1509 } 1510 1511 /* 1512 * Sysctl for testing 1513 */ 1514 1515 struct sun8i_crypto_userreq { 1516 kmutex_t cu_lock; 1517 kcondvar_t cu_cv; 1518 size_t cu_size; 1519 struct sun8i_crypto_buf cu_buf; 1520 struct sun8i_crypto_task *cu_task; 1521 int cu_error; 1522 bool cu_done; 1523 bool cu_cancel; 1524 }; 1525 1526 static void 1527 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1528 { 1529 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1530 int error; 1531 1532 /* hw.sun8icryptoN (node) */ 1533 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1534 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1535 SYSCTL_DESCR("sun8i crypto engine knobs"), 1536 NULL, 0, NULL, 0, 1537 CTL_HW, CTL_CREATE, CTL_EOL); 1538 if (error) { 1539 aprint_error_dev(sc->sc_dev, 1540 "failed to set up sysctl hw.%s: %d\n", 1541 device_xname(sc->sc_dev), error); 1542 return; 1543 } 1544 1545 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1546 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1547 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1548 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1549 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1550 if (error) { 1551 aprint_error_dev(sc->sc_dev, 1552 "failed to set up sysctl hw.%s.rng: %d\n", 1553 device_xname(sc->sc_dev), error); 1554 return; 1555 } 1556 } 1557 1558 static int 1559 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1560 { 1561 struct sysctlnode node = *rnode; 1562 struct sun8i_crypto_softc *sc = node.sysctl_data; 1563 struct sun8i_crypto_userreq *req; 1564 struct sun8i_crypto_task *task; 1565 size_t size; 1566 int error; 1567 1568 /* If oldp == NULL, the caller wants to learn the size. */ 1569 if (oldp == NULL) { 1570 *oldlenp = 4096; 1571 return 0; 1572 } 1573 1574 /* Truncate to 4096 bytes. */ 1575 size = MIN(4096, *oldlenp); 1576 if (size == 0) 1577 return 0; /* nothing to do */ 1578 1579 /* Allocate a request context. */ 1580 req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1581 if (req == NULL) 1582 return ENOMEM; 1583 1584 /* Initialize the request context. */ 1585 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1586 cv_init(&req->cu_cv, "sun8isy"); 1587 req->cu_size = size; 1588 req->cu_error = EIO; 1589 req->cu_done = false; 1590 req->cu_cancel = false; 1591 1592 /* Allocate a buffer for the RNG output. */ 1593 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1594 if (error) 1595 goto out0; 1596 1597 /* Allocate a task. */ 1598 task = req->cu_task = sun8i_crypto_task_get(sc, 1599 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 1600 if (task == NULL) { 1601 error = ENOMEM; 1602 goto out1; 1603 } 1604 1605 /* Set the task up for TRNG to our buffer. */ 1606 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1607 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1608 if (error) 1609 goto out2; 1610 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1611 if (error) 1612 goto out3; 1613 1614 /* Submit! */ 1615 error = sun8i_crypto_submit(sc, task); 1616 if (error) { 1617 /* Make sure we don't restart the syscall -- just fail. */ 1618 if (error == ERESTART) 1619 error = EBUSY; 1620 goto out3; 1621 } 1622 1623 /* Wait for the request to complete. */ 1624 mutex_enter(&req->cu_lock); 1625 while (!req->cu_done) { 1626 error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1627 if (error) { 1628 /* 1629 * If we finished while waiting to acquire the 1630 * lock, ignore the error and just return now. 1631 * Otherwise, notify the callback that it has 1632 * to clean up after us. 1633 */ 1634 if (req->cu_done) 1635 error = 0; 1636 else 1637 req->cu_cancel = true; 1638 break; 1639 } 1640 } 1641 mutex_exit(&req->cu_lock); 1642 1643 /* 1644 * Return early on error from cv_wait_sig, which means 1645 * interruption; the callback will clean up instead. 1646 */ 1647 if (error) 1648 return error; 1649 1650 /* Check for error from the device. */ 1651 error = req->cu_error; 1652 if (error) 1653 goto out3; 1654 1655 /* Copy out the data. */ 1656 node.sysctl_data = req->cu_buf.cb_kva; 1657 node.sysctl_size = size; 1658 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1659 1660 /* Clear the buffer. */ 1661 explicit_memset(req->cu_buf.cb_kva, 0, size); 1662 1663 /* Clean up. */ 1664 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1665 out2: sun8i_crypto_task_put(sc, task); 1666 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1667 out0: cv_destroy(&req->cu_cv); 1668 mutex_destroy(&req->cu_lock); 1669 kmem_free(req, sizeof(*req)); 1670 return error; 1671 } 1672 1673 static void 1674 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1675 struct sun8i_crypto_task *task, void *cookie, int error) 1676 { 1677 struct sun8i_crypto_userreq *req = cookie; 1678 bool cancel; 1679 1680 /* 1681 * Notify the waiting thread of the error, and find out whether 1682 * that thread cancelled. 1683 */ 1684 mutex_enter(&req->cu_lock); 1685 cancel = req->cu_cancel; 1686 req->cu_error = error; 1687 req->cu_done = true; 1688 cv_broadcast(&req->cu_cv); 1689 mutex_exit(&req->cu_lock); 1690 1691 /* 1692 * If it wasn't cancelled, we're done -- the main thread will 1693 * clean up after itself. 1694 */ 1695 if (!cancel) 1696 return; 1697 1698 /* Clean up after the main thread cancelled. */ 1699 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1700 sun8i_crypto_task_put(sc, task); 1701 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1702 cv_destroy(&req->cu_cv); 1703 mutex_destroy(&req->cu_lock); 1704 kmem_free(req, sizeof(*req)); 1705 } 1706 1707 /* 1708 * sun8i_crypto_register(sc) 1709 * 1710 * Register opencrypto algorithms supported by the crypto engine. 1711 */ 1712 static void 1713 sun8i_crypto_register(struct sun8i_crypto_softc *sc) 1714 { 1715 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 1716 1717 co->co_driverid = crypto_get_driverid(0); 1718 if (co->co_driverid == (uint32_t)-1) { 1719 aprint_error_dev(sc->sc_dev, 1720 "failed to register crypto driver\n"); 1721 return; 1722 } 1723 1724 sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 1725 sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 1726 #ifdef CRYPTO_AES_ECB 1727 sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 1728 #endif 1729 #ifdef CRYPTO_AES_XTS 1730 sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 1731 #endif 1732 #ifdef CRYPTO_DES_CBC 1733 sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 1734 #endif 1735 #ifdef CRYPTO_DES_ECB 1736 sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 1737 #endif 1738 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 1739 #ifdef CRYPTO_3DES_ECB 1740 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 1741 #endif 1742 1743 sun8i_crypto_register1(sc, CRYPTO_MD5); 1744 sun8i_crypto_register1(sc, CRYPTO_SHA1); 1745 #ifdef CRYPTO_SHA224 1746 sun8i_crypto_register1(sc, CRYPTO_SHA224); 1747 #endif 1748 #ifdef CRYPTO_SHA256 1749 sun8i_crypto_register1(sc, CRYPTO_SHA256); 1750 #endif 1751 1752 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 1753 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 1754 1755 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 1756 } 1757 1758 /* 1759 * sun8i_crypto_register1(sc, alg) 1760 * 1761 * Register support for one algorithm alg using 1762 * sun8i_crypto_newsession/freesession/process. 1763 */ 1764 static void 1765 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 1766 { 1767 1768 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 1769 sun8i_crypto_newsession, 1770 sun8i_crypto_freesession, 1771 sun8i_crypto_process, 1772 sc); 1773 } 1774 1775 /* 1776 * sun8i_crypto_newsession(cookie, sidp, cri) 1777 * 1778 * Called by opencrypto to allocate a new session. We don't keep 1779 * track of sessions, since there are no persistent keys in the 1780 * hardware that we take advantage of, so this only validates the 1781 * crypto operations and returns a zero session id. 1782 */ 1783 static int 1784 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 1785 { 1786 1787 /* No composition of operations is supported here. */ 1788 if (cri->cri_next) 1789 return EINVAL; 1790 1791 /* 1792 * No variation of rounds is supported here. (XXX Unused and 1793 * unimplemented in opencrypto(9) altogether? 1794 */ 1795 if (cri->cri_rnd) 1796 return EINVAL; 1797 1798 /* 1799 * Validate per-algorithm key length. 1800 * 1801 * XXX Does opencrypto(9) do this internally? 1802 */ 1803 switch (cri->cri_alg) { 1804 case CRYPTO_MD5: 1805 case CRYPTO_SHA1: 1806 #ifdef CRYPTO_SHA224 1807 case CRYPTO_SHA224: 1808 #endif 1809 #ifdef CRYPTO_SHA256 1810 case CRYPTO_SHA256: 1811 #endif 1812 if (cri->cri_klen) 1813 return EINVAL; 1814 break; 1815 case CRYPTO_AES_CBC: 1816 #ifdef CRYPTO_AES_ECB 1817 case CRYPTO_AES_ECB: 1818 #endif 1819 switch (cri->cri_klen) { 1820 case 128: 1821 case 192: 1822 case 256: 1823 break; 1824 default: 1825 return EINVAL; 1826 } 1827 break; 1828 case CRYPTO_AES_CTR: 1829 /* 1830 * opencrypto `AES-CTR' takes four bytes of the input 1831 * block as the last four bytes of the key, for reasons 1832 * that are not entirely clear. 1833 */ 1834 switch (cri->cri_klen) { 1835 case 128 + 32: 1836 case 192 + 32: 1837 case 256 + 32: 1838 break; 1839 default: 1840 return EINVAL; 1841 } 1842 break; 1843 #ifdef CRYPTO_AES_XTS 1844 case CRYPTO_AES_XTS: 1845 switch (cri->cri_klen) { 1846 case 256: 1847 case 384: 1848 case 512: 1849 break; 1850 default: 1851 return EINVAL; 1852 } 1853 break; 1854 #endif 1855 case CRYPTO_DES_CBC: 1856 #ifdef CRYPTO_DES_ECB 1857 case CRYPTO_DES_ECB: 1858 #endif 1859 switch (cri->cri_klen) { 1860 case 64: 1861 break; 1862 default: 1863 return EINVAL; 1864 } 1865 break; 1866 case CRYPTO_3DES_CBC: 1867 #ifdef CRYPTO_3DES_ECB 1868 case CRYPTO_3DES_ECB: 1869 #endif 1870 switch (cri->cri_klen) { 1871 case 192: 1872 break; 1873 default: 1874 return EINVAL; 1875 } 1876 break; 1877 case CRYPTO_SHA1_HMAC: 1878 /* 1879 * XXX Unclear what the length limit is, but since HMAC 1880 * behaves qualitatively different for a key of at 1881 * least the full block size -- and is generally best 1882 * to use with half the block size -- let's limit it to 1883 * one block. 1884 */ 1885 if (cri->cri_klen % 8) 1886 return EINVAL; 1887 if (cri->cri_klen > 512) 1888 return EINVAL; 1889 break; 1890 case CRYPTO_SHA2_256_HMAC: 1891 if (cri->cri_klen % 8) 1892 return EINVAL; 1893 if (cri->cri_klen > 512) 1894 return EINVAL; 1895 break; 1896 default: 1897 panic("unsupported algorithm %d", cri->cri_alg); 1898 } 1899 1900 KASSERT(cri->cri_klen % 8 == 0); 1901 1902 /* Success! */ 1903 *sidp = 1; 1904 return 0; 1905 } 1906 1907 /* 1908 * sun8i_crypto_freesession(cookie, dsid) 1909 * 1910 * Called by opencrypto to free a session. We don't keep track of 1911 * sessions, since there are no persistent keys in the hardware 1912 * that we take advantage of, so this is a no-op. 1913 * 1914 * Note: dsid is actually a 64-bit quantity containing both the 1915 * driver id in the high half and the session id in the low half. 1916 */ 1917 static int 1918 sun8i_crypto_freesession(void *cookie, uint64_t dsid) 1919 { 1920 1921 KASSERT((dsid & 0xffffffff) == 1); 1922 1923 /* Success! */ 1924 return 0; 1925 } 1926 1927 /* 1928 * sun8i_crypto_ivlen(crd) 1929 * 1930 * Return the crypto engine's notion of `IV length', in bytes, for 1931 * an opencrypto operation. 1932 */ 1933 static u_int 1934 sun8i_crypto_ivlen(const struct cryptodesc *crd) 1935 { 1936 1937 switch (crd->crd_alg) { 1938 case CRYPTO_AES_CBC: 1939 return 16; 1940 #ifdef CRYPTO_AES_XTS 1941 case CRYPTO_AES_XTS: 1942 return 16; 1943 #endif 1944 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 1945 return 8; 1946 #ifdef CRYPTO_DES_CBC 1947 case CRYPTO_DES_CBC: 1948 return 8; 1949 #endif 1950 case CRYPTO_3DES_CBC: 1951 return 8; 1952 case CRYPTO_MD5: 1953 return 16; 1954 #ifdef CRYPTO_SHA224 1955 case CRYPTO_SHA224: 1956 return 32; 1957 #endif 1958 #ifdef CRYPTO_SHA256 1959 case CRYPTO_SHA256: 1960 return 32; 1961 #endif 1962 case CRYPTO_SHA1_HMAC: 1963 return 20; 1964 case CRYPTO_SHA2_256_HMAC: 1965 return 32; 1966 default: 1967 return 0; 1968 } 1969 } 1970 1971 /* 1972 * sun8i_crypto_process(cookie, crp, hint) 1973 * 1974 * Main opencrypto processing dispatch. 1975 */ 1976 static int 1977 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 1978 { 1979 struct sun8i_crypto_softc *sc = cookie; 1980 struct sun8i_crypto_task *task; 1981 struct cryptodesc *crd = crp->crp_desc; 1982 unsigned klen, ivlen; 1983 uint32_t tdqc = 0, tdqs = 0; 1984 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 1985 const uint32_t tdqa = 0; 1986 int error; 1987 1988 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 1989 1990 /* Reject compositions -- we do not handle them. */ 1991 if (crd->crd_next != NULL) { 1992 error = EOPNOTSUPP; 1993 goto fail0; 1994 } 1995 1996 /* Reject transfers with nonsense skip. */ 1997 if (crd->crd_skip < 0) { 1998 error = EINVAL; 1999 goto fail0; 2000 } 2001 2002 /* 2003 * Actually just reject any nonzero skip, because it requires 2004 * DMA segment bookkeeping that we don't do yet. 2005 */ 2006 if (crd->crd_skip) { 2007 error = EOPNOTSUPP; 2008 goto fail0; 2009 } 2010 2011 /* Reject large transfers. */ 2012 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 2013 error = EFBIG; 2014 goto fail0; 2015 } 2016 2017 /* Reject nonsense, unaligned, or mismatched lengths. */ 2018 if (crd->crd_len < 0 || 2019 crd->crd_len % 4 || 2020 crd->crd_len != crp->crp_ilen) { 2021 error = EINVAL; 2022 goto fail0; 2023 } 2024 2025 /* Reject mismatched buffer lengths. */ 2026 /* XXX Handle crd_skip. */ 2027 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2028 struct mbuf *m = crp->crp_buf; 2029 uint32_t nbytes = 0; 2030 while (m != NULL) { 2031 KASSERT(m->m_len >= 0); 2032 if (m->m_len > crd->crd_len || 2033 nbytes > crd->crd_len - m->m_len) { 2034 error = EINVAL; 2035 goto fail0; 2036 } 2037 nbytes += m->m_len; 2038 m = m->m_next; 2039 } 2040 if (nbytes != crd->crd_len) { 2041 error = EINVAL; 2042 goto fail0; 2043 } 2044 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2045 struct uio *uio = crp->crp_buf; 2046 if (uio->uio_resid != crd->crd_len) { 2047 error = EINVAL; 2048 goto fail0; 2049 } 2050 } 2051 2052 /* Get a task, or fail with ERESTART if we can't. */ 2053 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 2054 PR_NOWAIT); 2055 if (task == NULL) { 2056 /* 2057 * Don't invoke crypto_done -- we are asking the 2058 * opencrypto(9) machinery to queue the request and get 2059 * back to us. 2060 */ 2061 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 2062 return ERESTART; 2063 } 2064 2065 /* Load key in, if relevant. */ 2066 klen = crd->crd_klen; 2067 if (klen) { 2068 if (crd->crd_alg == CRYPTO_AES_CTR) 2069 /* AES-CTR is special -- see IV processing below. */ 2070 klen -= 32; 2071 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 2072 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 2073 if (error) 2074 goto fail1; 2075 task->ct_flags |= TASK_KEY; 2076 } 2077 2078 /* Handle the IV, if relevant. */ 2079 ivlen = sun8i_crypto_ivlen(crd); 2080 if (ivlen) { 2081 void *iv; 2082 2083 /* 2084 * If there's an explicit IV, use it; otherwise 2085 * randomly generate one. 2086 */ 2087 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2088 iv = crd->crd_iv; 2089 } else { 2090 cprng_fast(task->ct_iv, ivlen); 2091 iv = task->ct_iv; 2092 } 2093 2094 /* 2095 * If the IV is not already present in the user's 2096 * buffer, copy it over. 2097 */ 2098 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 2099 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2100 m_copyback(crp->crp_buf, crd->crd_inject, 2101 ivlen, iv); 2102 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2103 cuio_copyback(crp->crp_buf, crd->crd_inject, 2104 ivlen, iv); 2105 } else { 2106 panic("invalid buffer type %x", 2107 crp->crp_flags); 2108 } 2109 } 2110 2111 /* 2112 * opencrypto's idea of `AES-CTR' is special. 2113 * 2114 * - The low 4 bytes of the input block are drawn from 2115 * an extra 4 bytes at the end of the key. 2116 * 2117 * - The next 8 bytes of the input block are drawn from 2118 * the opencrypto iv. 2119 * 2120 * - The high 4 bytes are the big-endian block counter, 2121 * which starts at 1 because why not. 2122 */ 2123 if (crd->crd_alg == CRYPTO_AES_CTR) { 2124 uint8_t block[16]; 2125 uint32_t blkno = 1; 2126 2127 /* Format the initial input block. */ 2128 memcpy(block, crd->crd_key + klen/8, 4); 2129 memcpy(block + 4, iv, 8); 2130 be32enc(block + 12, blkno); 2131 2132 /* Copy it into the DMA buffer. */ 2133 memcpy(task->ct_iv, block, 16); 2134 iv = task->ct_iv; 2135 ivlen = 16; 2136 } 2137 2138 /* Load the IV. */ 2139 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 2140 NULL, BUS_DMA_NOWAIT); 2141 if (error) 2142 goto fail1; 2143 task->ct_flags |= TASK_IV; 2144 } 2145 2146 /* Load the src and dst. */ 2147 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2148 struct mbuf *m = crp->crp_buf; 2149 2150 /* XXX Handle crd_skip. */ 2151 KASSERT(crd->crd_skip == 0); 2152 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 2153 BUS_DMA_NOWAIT); 2154 if (error) 2155 goto fail1; 2156 task->ct_flags |= TASK_SRC; 2157 2158 /* XXX Handle crd_skip. */ 2159 KASSERT(crd->crd_skip == 0); 2160 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 2161 BUS_DMA_NOWAIT); 2162 if (error) 2163 goto fail1; 2164 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2165 struct uio *uio = crp->crp_buf; 2166 2167 /* XXX Handle crd_skip. */ 2168 KASSERT(crd->crd_skip == 0); 2169 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 2170 BUS_DMA_NOWAIT); 2171 if (error) 2172 goto fail1; 2173 task->ct_flags |= TASK_SRC; 2174 2175 /* XXX Handle crd_skip. */ 2176 KASSERT(crd->crd_skip == 0); 2177 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 2178 BUS_DMA_NOWAIT); 2179 if (error) 2180 goto fail1; 2181 } else { 2182 panic("invalid buffer type %x", crp->crp_flags); 2183 } 2184 2185 /* Set the encryption direction. */ 2186 if (crd->crd_flags & CRD_F_ENCRYPT) 2187 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 2188 else 2189 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 2190 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 2191 2192 /* Set the method. */ 2193 switch (crd->crd_alg) { 2194 case CRYPTO_AES_CBC: 2195 case CRYPTO_AES_CTR: 2196 #ifdef CRYPTO_AES_ECB 2197 case CRYPTO_AES_ECB: 2198 #endif 2199 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2200 break; 2201 #ifdef CRYPTO_AES_XTS 2202 case CRYPTO_AES_XTS: 2203 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2204 break; 2205 #endif 2206 case CRYPTO_DES_CBC: 2207 #ifdef CRYPTO_DES_ECB 2208 case CRYPTO_DES_ECB: 2209 #endif 2210 method = SUN8I_CRYPTO_TDQC_METHOD_DES; 2211 break; 2212 case CRYPTO_3DES_CBC: 2213 #ifdef CRYPTO_3DES_ECB 2214 case CRYPTO_3DES_ECB: 2215 #endif 2216 method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 2217 break; 2218 case CRYPTO_MD5: 2219 method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 2220 break; 2221 case CRYPTO_SHA1: 2222 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 2223 break; 2224 #ifdef CRYPTO_SHA224 2225 case CRYPTO_SHA224: 2226 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 2227 break; 2228 #endif 2229 #ifdef CRYPTO_SHA256 2230 case CRYPTO_SHA256: 2231 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 2232 break; 2233 #endif 2234 case CRYPTO_SHA1_HMAC: 2235 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 2236 break; 2237 case CRYPTO_SHA2_256_HMAC: 2238 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 2239 break; 2240 default: 2241 panic("unknown algorithm %d", crd->crd_alg); 2242 } 2243 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 2244 2245 /* Set the key selector. No idea how to use the internal keys. */ 2246 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 2247 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 2248 2249 /* XXX Deal with AES_CTS_Last_Block_Flag. */ 2250 2251 /* Set the mode. */ 2252 switch (crd->crd_alg) { 2253 #ifdef CRYPTO_AES_ECB 2254 case CRYPTO_AES_ECB: 2255 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2256 break; 2257 #endif 2258 #ifdef CRYPTO_DES_ECB 2259 case CRYPTO_DES_ECB: 2260 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2261 break; 2262 #endif 2263 #ifdef CRYPTO_3DES_ECB 2264 case CRYPTO_3DES_ECB: 2265 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2266 break; 2267 #endif 2268 case CRYPTO_AES_CBC: 2269 case CRYPTO_DES_CBC: 2270 case CRYPTO_3DES_CBC: 2271 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 2272 break; 2273 case CRYPTO_AES_CTR: 2274 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 2275 break; 2276 #ifdef CRYPTO_AES_XTS 2277 case CRYPTO_AES_XTS: 2278 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 2279 break; 2280 #endif 2281 default: 2282 panic("unknown algorithm %d", crd->crd_alg); 2283 } 2284 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 2285 2286 /* Set the CTR width. */ 2287 switch (crd->crd_alg) { 2288 case CRYPTO_AES_CTR: 2289 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 2290 break; 2291 } 2292 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 2293 2294 /* Set the AES key size. */ 2295 switch (crd->crd_alg) { 2296 case CRYPTO_AES_CBC: 2297 #ifdef CRYPTO_AES_ECB 2298 case CRYPTO_AES_ECB: 2299 #endif 2300 switch (crd->crd_klen) { 2301 case 128: 2302 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2303 break; 2304 case 192: 2305 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2306 break; 2307 case 256: 2308 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2309 break; 2310 default: 2311 panic("invalid AES key size in bits: %u", 2312 crd->crd_klen); 2313 } 2314 break; 2315 case CRYPTO_AES_CTR: 2316 switch (crd->crd_klen) { 2317 case 128 + 32: 2318 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2319 break; 2320 case 192 + 32: 2321 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2322 break; 2323 case 256 + 32: 2324 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2325 break; 2326 default: 2327 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 2328 crd->crd_klen); 2329 } 2330 break; 2331 #ifdef CRYPTO_AES_XTS 2332 case CRYPTO_AES_XTS: 2333 switch (crd->crd_klen) { 2334 case 256: 2335 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2336 break; 2337 case 384: 2338 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2339 break; 2340 case 512: 2341 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2342 break; 2343 default: 2344 panic("invalid AES-XTS key size in bits: %u", 2345 crd->crd_klen); 2346 } 2347 break; 2348 #endif 2349 } 2350 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 2351 2352 /* Set up the task descriptor. */ 2353 error = sun8i_crypto_task_load(sc, task, crd->crd_len, 2354 tdqc, tdqs, tdqa); 2355 if (error) 2356 goto fail2; 2357 2358 /* Submit! */ 2359 error = sun8i_crypto_submit(sc, task); 2360 if (error) 2361 goto fail2; 2362 2363 /* Success! */ 2364 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 2365 return 0; 2366 2367 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2368 fail1: if (task->ct_flags & TASK_SRC) 2369 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2370 if (task->ct_flags & TASK_CTR) 2371 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2372 if (task->ct_flags & TASK_IV) 2373 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2374 if (task->ct_flags & TASK_KEY) 2375 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2376 sun8i_crypto_task_put(sc, task); 2377 fail0: KASSERT(error); 2378 KASSERT(error != ERESTART); 2379 crp->crp_etype = error; 2380 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2381 crypto_done(crp); 2382 return 0; 2383 } 2384 2385 /* 2386 * sun8i_crypto_callback(sc, task, cookie, error) 2387 * 2388 * Completion callback for a task submitted via opencrypto. 2389 * Release the task and pass the error on to opencrypto with 2390 * crypto_done. 2391 */ 2392 static void 2393 sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 2394 struct sun8i_crypto_task *task, void *cookie, int error) 2395 { 2396 struct cryptop *crp = cookie; 2397 struct cryptodesc *crd __diagused = crp->crp_desc; 2398 2399 KASSERT(error != ERESTART); 2400 KASSERT(crd != NULL); 2401 KASSERT(crd->crd_next == NULL); 2402 2403 /* Return the number of bytes processed. */ 2404 crp->crp_olen = error ? 0 : crp->crp_ilen; 2405 2406 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2407 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2408 if (task->ct_flags & TASK_CTR) 2409 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2410 if (task->ct_flags & TASK_IV) 2411 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2412 if (task->ct_flags & TASK_KEY) 2413 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2414 sun8i_crypto_task_put(sc, task); 2415 KASSERT(error != ERESTART); 2416 crp->crp_etype = error; 2417 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2418 crypto_done(crp); 2419 } 2420
Indexes created Fri Sep 26 08:10:20 GMT 2025