sun8i_crypto.c revision 1.17
1 /* $NetBSD: sun8i_crypto.c,v 1.17 2020/06/13 18:58:26 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * sun8i_crypto -- Allwinner Crypto Engine driver 34 * 35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64 36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at 37 * the moment, but in principle it could be wired up with opencrypto(9) 38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1, 39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support 40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?). 41 * 42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf 43 */ 44 45 #include <sys/cdefs.h> 46 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.17 2020/06/13 18:58:26 riastradh Exp $"); 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/atomic.h> 51 #include <sys/bus.h> 52 #include <sys/callout.h> 53 #include <sys/conf.h> 54 #include <sys/cprng.h> 55 #include <sys/device.h> 56 #include <sys/kernel.h> 57 #include <sys/kmem.h> 58 #include <sys/mbuf.h> 59 #include <sys/mutex.h> 60 #include <sys/rndsource.h> 61 #include <sys/sdt.h> 62 #include <sys/sysctl.h> 63 #include <sys/workqueue.h> 64 65 #include <dev/fdt/fdtvar.h> 66 67 #include <opencrypto/cryptodev.h> 68 69 #include <arm/sunxi/sun8i_crypto.h> 70 71 #define SUN8I_CRYPTO_TIMEOUT hz 72 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */ 73 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE 74 75 struct sun8i_crypto_task; 76 77 struct sun8i_crypto_buf { 78 bus_dma_segment_t cb_seg[1]; 79 int cb_nsegs; 80 void *cb_kva; 81 }; 82 83 struct sun8i_crypto_softc { 84 device_t sc_dev; 85 bus_space_tag_t sc_bst; 86 bus_space_handle_t sc_bsh; 87 bus_dma_tag_t sc_dmat; 88 struct pool_cache *sc_taskpool; 89 kmutex_t sc_lock; 90 struct sun8i_crypto_chan { 91 struct sun8i_crypto_task *cc_task; 92 unsigned cc_starttime; 93 } sc_chan[SUN8I_CRYPTO_NCHAN]; 94 struct callout sc_timeout; 95 struct workqueue *sc_wq; 96 struct work sc_work; 97 void *sc_ih; 98 uint32_t sc_done; 99 uint32_t sc_esr; 100 bool sc_work_pending; 101 struct sun8i_crypto_rng { 102 struct sun8i_crypto_buf cr_buf; 103 struct sun8i_crypto_task *cr_task; 104 struct krndsource cr_rndsource; 105 bool cr_pending; 106 } sc_rng; 107 struct sun8i_crypto_selftest { 108 struct sun8i_crypto_buf cs_in; 109 struct sun8i_crypto_buf cs_key; 110 struct sun8i_crypto_buf cs_out; 111 struct sun8i_crypto_task *cs_task; 112 } sc_selftest; 113 struct sun8i_crypto_sysctl { 114 struct sysctllog *cy_log; 115 const struct sysctlnode *cy_root_node; 116 const struct sysctlnode *cy_trng_node; 117 } sc_sysctl; 118 struct sun8i_crypto_opencrypto { 119 uint32_t co_driverid; 120 } sc_opencrypto; 121 }; 122 123 struct sun8i_crypto_task { 124 struct sun8i_crypto_buf ct_descbuf; 125 struct sun8i_crypto_taskdesc *ct_desc; 126 struct sun8i_crypto_buf ct_ivbuf; 127 void *ct_iv; 128 struct sun8i_crypto_buf ct_ctrbuf; 129 void *ct_ctr; 130 bus_dmamap_t ct_descmap; 131 bus_dmamap_t ct_keymap; 132 bus_dmamap_t ct_ivmap; /* IV input */ 133 bus_dmamap_t ct_ctrmap; /* updated IV output */ 134 bus_dmamap_t ct_srcmap; 135 bus_dmamap_t ct_dstmap; 136 uint32_t ct_nbytes; 137 int ct_flags; 138 #define TASK_KEY __BIT(0) 139 #define TASK_IV __BIT(1) 140 #define TASK_CTR __BIT(2) 141 #define TASK_SRC __BIT(3) 142 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */ 143 void (*ct_callback)(struct sun8i_crypto_softc *, 144 struct sun8i_crypto_task *, void *, int); 145 void *ct_cookie; 146 }; 147 148 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE 149 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE 150 151 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN); 152 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN); 153 154 /* 155 * Forward declarations 156 */ 157 158 static int sun8i_crypto_match(device_t, cfdata_t, void *); 159 static void sun8i_crypto_attach(device_t, device_t, void *); 160 161 static int sun8i_crypto_task_ctor(void *, void *, int); 162 static void sun8i_crypto_task_dtor(void *, void *); 163 static struct sun8i_crypto_task * 164 sun8i_crypto_task_get(struct sun8i_crypto_softc *, 165 void (*)(struct sun8i_crypto_softc *, 166 struct sun8i_crypto_task *, void *, int), 167 void *, int); 168 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *, 169 struct sun8i_crypto_task *); 170 171 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *, 172 struct sun8i_crypto_task *, uint32_t, 173 uint32_t, uint32_t, uint32_t); 174 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *, 175 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t); 176 177 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *, 178 struct sun8i_crypto_task *, uint32_t); 179 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *, 180 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t); 181 182 static int sun8i_crypto_submit(struct sun8i_crypto_softc *, 183 struct sun8i_crypto_task *); 184 185 static void sun8i_crypto_timeout(void *); 186 static int sun8i_crypto_intr(void *); 187 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *); 188 static void sun8i_crypto_worker(struct work *, void *); 189 static void sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned, 190 int); 191 192 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t, 193 struct sun8i_crypto_buf *, int); 194 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t, 195 struct sun8i_crypto_buf *); 196 197 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *); 198 static void sun8i_crypto_rng_get(size_t, void *); 199 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *, 200 struct sun8i_crypto_task *, void *, int); 201 202 static void sun8i_crypto_selftest(device_t); 203 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *, 204 struct sun8i_crypto_task *, void *, int); 205 206 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *); 207 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS); 208 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *, 209 struct sun8i_crypto_task *, void *, int); 210 211 static void sun8i_crypto_register(struct sun8i_crypto_softc *); 212 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t); 213 static int sun8i_crypto_newsession(void *, uint32_t *, 214 struct cryptoini *); 215 static int sun8i_crypto_freesession(void *, uint64_t); 216 static u_int sun8i_crypto_ivlen(const struct cryptodesc *); 217 static int sun8i_crypto_process(void *, struct cryptop *, int); 218 static void sun8i_crypto_callback(struct sun8i_crypto_softc *, 219 struct sun8i_crypto_task *, void *, int); 220 221 /* 222 * Probes 223 */ 224 225 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read, 226 "bus_size_t"/*reg*/, 227 "uint32_t"/*value*/); 228 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write, 229 "bus_size_t"/*reg*/, 230 "uint32_t"/*write*/); 231 232 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success, 233 "struct sun8i_crypto_task *"/*task*/); 234 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure, 235 "int"/*error*/); 236 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor, 237 "struct sun8i_crypto_task *"/*task*/); 238 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get, 239 "struct sun8i_crypto_task *"/*task*/); 240 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put, 241 "struct sun8i_crypto_task *"/*task*/); 242 243 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load, 244 "struct sun8i_crypto_task *"/*task*/, 245 "uint32_t"/*tdqc*/, 246 "uint32_t"/*tdqs*/, 247 "uint32_t"/*tdqa*/, 248 "struct sun8i_crypto_taskdesc *"/*desc*/, 249 "int"/*error*/); 250 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned, 251 "struct sun8i_crypto_task *"/*task*/, 252 "bus_addr_t"/*ds_addr*/, 253 "bus_size_t"/*ds_len*/); 254 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done, 255 "struct sun8i_crypto_task *"/*task*/, 256 "int"/*error*/); 257 258 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure, 259 "struct sun8i_crypto_softc *"/*sc*/, 260 "struct sun8i_crypto_task *"/*task*/, 261 "int"/*error*/); 262 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success, 263 "struct sun8i_crypto_softc *"/*sc*/, 264 "struct sun8i_crypto_task *"/*task*/, 265 "unsigned"/*chan*/); 266 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr, 267 "struct sun8i_crypto_softc *"/*sc*/, 268 "uint32_t"/*isr*/, 269 "uint32_t"/*esr*/); 270 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done, 271 "struct sun8i_crypto_softc *"/*sc*/, 272 "unsigned"/*chan*/, 273 "int"/*error*/); 274 275 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry, 276 "struct sun8i_crypto_softc *"/*sc*/, 277 "struct cryptop *"/*crp*/, 278 "int"/*hint*/); 279 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy, 280 "struct sun8i_crypto_softc *"/*sc*/, 281 "struct cryptop *"/*crp*/, 282 "int"/*hint*/); 283 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued, 284 "struct sun8i_crypto_softc *"/*sc*/, 285 "struct cryptop *"/*crp*/, 286 "int"/*hint*/, 287 "struct sun8i_crypto_task *"/*task*/); 288 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done, 289 "struct sun8i_crypto_softc *"/*sc*/, 290 "struct cryptop *"/*crp*/, 291 "int"/*error*/); 292 293 /* 294 * Register access 295 */ 296 297 static uint32_t 298 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg) 299 { 300 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg); 301 302 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v); 303 return v; 304 } 305 306 static void 307 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v) 308 { 309 310 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v); 311 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v); 312 } 313 314 /* 315 * Autoconf goo 316 */ 317 318 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc), 319 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL); 320 321 static const struct of_compat_data compat_data[] = { 322 {"allwinner,sun50i-a64-crypto", 0}, 323 {NULL} 324 }; 325 326 static int 327 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux) 328 { 329 const struct fdt_attach_args *const faa = aux; 330 331 return of_match_compat_data(faa->faa_phandle, compat_data); 332 } 333 334 static void 335 sun8i_crypto_attach(device_t parent, device_t self, void *aux) 336 { 337 struct sun8i_crypto_softc *const sc = device_private(self); 338 const struct fdt_attach_args *const faa = aux; 339 bus_addr_t addr; 340 bus_size_t size; 341 const int phandle = faa->faa_phandle; 342 char intrstr[128]; 343 struct clk *clk; 344 struct fdtbus_reset *rst; 345 346 sc->sc_dev = self; 347 sc->sc_dmat = faa->faa_dmat; 348 sc->sc_bst = faa->faa_bst; 349 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task), 350 0, 0, 0, "sun8icry", NULL, IPL_VM, 351 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc); 352 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM); 353 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE); 354 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc); 355 if (workqueue_create(&sc->sc_wq, device_xname(self), 356 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) { 357 aprint_error(": couldn't create workqueue\n"); 358 return; 359 } 360 361 /* 362 * Prime the pool with enough tasks that each channel can be 363 * busy with a task as we prepare another task for when it's 364 * done. 365 */ 366 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN); 367 368 /* Get and map device registers. */ 369 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) { 370 aprint_error(": couldn't get registers\n"); 371 return; 372 } 373 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) { 374 aprint_error(": couldn't map registers\n"); 375 return; 376 } 377 378 /* Get an interrupt handle. */ 379 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) { 380 aprint_error(": failed to decode interrupt\n"); 381 return; 382 } 383 384 /* Enable the bus clock. */ 385 if (fdtbus_clock_enable(phandle, "bus", true) != 0) { 386 aprint_error(": couldn't enable bus clock\n"); 387 return; 388 } 389 390 /* Get the module clock and set it to 300 MHz. */ 391 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) { 392 if (clk_enable(clk) != 0) { 393 aprint_error(": couldn't enable CE clock\n"); 394 return; 395 } 396 if (clk_set_rate(clk, 300*1000*1000) != 0) { 397 aprint_error(": couldn't set CE clock to 300MHz\n"); 398 return; 399 } 400 } 401 402 /* Get a reset handle if we need and try to deassert it. */ 403 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) { 404 if (fdtbus_reset_deassert(rst) != 0) { 405 aprint_error(": couldn't de-assert reset\n"); 406 return; 407 } 408 } 409 410 aprint_naive("\n"); 411 aprint_normal(": Crypto Engine\n"); 412 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk)); 413 414 /* Disable and clear interrupts. */ 415 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0); 416 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0); 417 418 /* Establish an interrupt handler. */ 419 sc->sc_ih = fdtbus_intr_establish(phandle, 0, IPL_VM, FDT_INTR_MPSAFE, 420 &sun8i_crypto_intr, sc); 421 if (sc->sc_ih == NULL) { 422 aprint_error_dev(self, "failed to establish interrupt on %s\n", 423 intrstr); 424 return; 425 } 426 aprint_normal_dev(self, "interrupting on %s\n", intrstr); 427 428 /* Set up the RNG. */ 429 sun8i_crypto_rng_attach(sc); 430 431 /* Attach the sysctl. */ 432 sun8i_crypto_sysctl_attach(sc); 433 434 /* Perform self-tests. */ 435 config_interrupts(self, sun8i_crypto_selftest); 436 437 /* Register opencrypto handlers. */ 438 sun8i_crypto_register(sc); 439 } 440 441 static int 442 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags) 443 { 444 struct sun8i_crypto_softc *sc = cookie; 445 struct sun8i_crypto_task *task = vtask; 446 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT; 447 int error; 448 449 /* Create a DMA buffer for the task descriptor. */ 450 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc), 451 &task->ct_descbuf, dmaflags); 452 if (error) 453 goto fail0; 454 task->ct_desc = task->ct_descbuf.cb_kva; 455 456 /* Create DMA buffers for the IV and CTR. */ 457 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES, 458 &task->ct_ivbuf, dmaflags); 459 if (error) 460 goto fail1; 461 task->ct_iv = task->ct_ivbuf.cb_kva; 462 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, 463 &task->ct_ctrbuf, dmaflags); 464 if (error) 465 goto fail2; 466 task->ct_ctr = task->ct_ctrbuf.cb_kva; 467 468 /* Create a DMA map for the task descriptor and preload it. */ 469 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1, 470 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap); 471 if (error) 472 goto fail3; 473 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc, 474 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK); 475 if (error) 476 goto fail4; 477 478 /* Create DMA maps for the key, IV, and CTR. */ 479 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1, 480 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap); 481 if (error) 482 goto fail5; 483 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1, 484 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap); 485 if (error) 486 goto fail6; 487 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1, 488 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap); 489 if (error) 490 goto fail7; 491 492 /* Create DMA maps for the src and dst scatter/gather vectors. */ 493 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 494 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 495 &task->ct_srcmap); 496 if (error) 497 goto fail8; 498 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE, 499 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags, 500 &task->ct_dstmap); 501 if (error) 502 goto fail9; 503 504 /* Success! */ 505 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task); 506 return 0; 507 508 fail10: __unused 509 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 510 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 511 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 512 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 513 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 514 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 515 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 516 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 517 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 518 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 519 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error); 520 return error; 521 } 522 523 static void 524 sun8i_crypto_task_dtor(void *cookie, void *vtask) 525 { 526 struct sun8i_crypto_softc *sc = cookie; 527 struct sun8i_crypto_task *task = vtask; 528 529 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task); 530 531 /* XXX Zero the bounce buffers if there are any. */ 532 533 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap); 534 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap); 535 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap); 536 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap); 537 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap); 538 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap); 539 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap); 540 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf); 541 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf); 542 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf); 543 } 544 545 /* 546 * sun8i_crypto_task_get(sc, callback, cookie, pflags) 547 * 548 * Allocate a task that will call callback(sc, task, cookie, 549 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if 550 * PR_NOWAIT, may fail and return NULL. No further allocation is 551 * needed to submit the task if this succeeds (although task 552 * submission may still fail if all channels are busy). 553 */ 554 static struct sun8i_crypto_task * 555 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc, 556 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *, 557 void *, int), 558 void *cookie, int pflags) 559 { 560 struct sun8i_crypto_task *task; 561 562 /* Allocate a task, or fail if we can't. */ 563 task = pool_cache_get(sc->sc_taskpool, pflags); 564 if (task == NULL) 565 goto out; 566 567 /* Set up flags and the callback. */ 568 task->ct_flags = 0; 569 task->ct_callback = callback; 570 task->ct_cookie = cookie; 571 572 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task); 573 return task; 574 } 575 576 /* 577 * sun8i_crypto_task_invalid(sc, task, cookie, error) 578 * 579 * Callback for a task not currently in use, to detect errors. 580 */ 581 static void 582 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc, 583 struct sun8i_crypto_task *task, void *cookie, int error) 584 { 585 void (*callback)(struct sun8i_crypto_softc *, 586 struct sun8i_crypto_task *, void *, int) = cookie; 587 588 panic("task for callback %p used after free", callback); 589 } 590 591 /* 592 * sun8i_crypto_task_put(sc, task) 593 * 594 * Free a task obtained with sun8i_crypto_task_get. 595 */ 596 static void 597 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc, 598 struct sun8i_crypto_task *task) 599 { 600 601 SDT_PROBE1(sdt, sun8i_crypto, task, put, task); 602 603 task->ct_cookie = task->ct_callback; 604 task->ct_callback = &sun8i_crypto_task_invalid; 605 pool_cache_put(sc->sc_taskpool, task); 606 } 607 608 /* 609 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa) 610 * 611 * Set up the task descriptor after the relevant DMA maps have 612 * been loaded for a transfer of nbytes. bus_dmamap_sync matches 613 * sun8i_crypto_chan_done. May fail if input is inadequately 614 * aligned. 615 * 616 * XXX Teach this to support task chains. 617 */ 618 static int 619 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc, 620 struct sun8i_crypto_task *task, uint32_t nbytes, 621 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa) 622 { 623 struct sun8i_crypto_taskdesc *desc = task->ct_desc; 624 int error; 625 626 KASSERT(tdqs == 0 || tdqa == 0); 627 KASSERT(nbytes % 4 == 0); 628 629 memset(desc, 0, sizeof(*desc)); 630 631 /* Always enable interrupt for the task. */ 632 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN; 633 634 desc->td_tdqc = htole32(tdqc); 635 desc->td_tdqs = htole32(tdqs); 636 desc->td_tdqa = htole32(tdqa); 637 638 if (task->ct_flags & TASK_KEY) { 639 bus_dmamap_t keymap = task->ct_keymap; 640 KASSERT(keymap->dm_nsegs == 1); 641 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr); 642 bus_dmamap_sync(sc->sc_dmat, keymap, 0, 643 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 644 } 645 if (task->ct_flags & TASK_IV) { 646 bus_dmamap_t ivmap = task->ct_ivmap; 647 KASSERT(ivmap->dm_nsegs == 1); 648 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr); 649 bus_dmamap_sync(sc->sc_dmat, ivmap, 0, 650 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE); 651 } 652 if (task->ct_flags & TASK_CTR) { 653 bus_dmamap_t ctrmap = task->ct_ctrmap; 654 KASSERT(ctrmap->dm_nsegs == 1); 655 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr); 656 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0, 657 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD); 658 } 659 660 if (task->ct_flags & TASK_BYTES) 661 desc->td_datalen = htole32(nbytes); 662 else 663 desc->td_datalen = htole32(nbytes/4); 664 665 if (task->ct_flags & TASK_SRC) { 666 bus_dmamap_t srcmap = task->ct_srcmap; 667 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize); 668 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap, 669 nbytes); 670 if (error) 671 return error; 672 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes, 673 BUS_DMASYNC_PREWRITE); 674 } 675 676 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap, 677 nbytes); 678 if (error) 679 goto out; 680 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 681 BUS_DMASYNC_PREREAD); 682 683 task->ct_nbytes = nbytes; 684 685 /* Success! */ 686 error = 0; 687 688 out: SDT_PROBE6(sdt, sun8i_crypto, task, load, 689 task, tdqc, tdqs, tdqa, desc, error); 690 return error; 691 } 692 693 /* 694 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes) 695 * 696 * Set up a task's scatter/gather vector -- src or dst -- with the 697 * given DMA map for a transfer of nbytes. May fail if input is 698 * inadequately aligned. 699 */ 700 static int 701 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task, 702 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map, 703 uint32_t nbytes __diagused) 704 { 705 uint32_t total __diagused = 0; 706 unsigned i; 707 708 /* 709 * Verify that the alignment is correct and initialize the 710 * scatter/gather vector. 711 */ 712 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS); 713 for (i = 0; i < map->dm_nsegs; i++) { 714 if ((map->dm_segs[i].ds_addr % 4) | 715 (map->dm_segs[i].ds_len % 4)) { 716 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned, 717 task, 718 map->dm_segs[i].ds_addr, 719 map->dm_segs[i].ds_len); 720 return EINVAL; 721 } 722 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX); 723 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total); 724 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr); 725 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4); 726 total += map->dm_segs[i].ds_len; 727 } 728 729 /* Set the remainder to zero. */ 730 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) { 731 adrlen[i].adr = 0; 732 adrlen[i].len = 0; 733 } 734 735 /* Verify the total size matches the transfer length. */ 736 KASSERT(total == nbytes); 737 738 /* Success! */ 739 return 0; 740 } 741 742 /* 743 * sun8i_crypto_task_load_trng(task, nbytes) 744 * 745 * Set up the task descriptor for a transfer of nbytes from the 746 * TRNG. 747 */ 748 static int 749 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc, 750 struct sun8i_crypto_task *task, uint32_t nbytes) 751 { 752 uint32_t tdqc = 0; 753 754 /* Caller must provide dst only. */ 755 KASSERT((task->ct_flags & TASK_KEY) == 0); 756 KASSERT((task->ct_flags & TASK_IV) == 0); 757 KASSERT((task->ct_flags & TASK_CTR) == 0); 758 KASSERT((task->ct_flags & TASK_SRC) == 0); 759 760 /* Set up the task descriptor queue control words. */ 761 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG, 762 SUN8I_CRYPTO_TDQC_METHOD); 763 764 /* Fill in the descriptor. */ 765 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0); 766 } 767 768 static int 769 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc, 770 struct sun8i_crypto_task *task, 771 uint32_t nbytes, uint32_t keysize, uint32_t dir) 772 { 773 uint32_t tdqc = 0, tdqs = 0; 774 775 /* Caller must provide key, src, and dst only. */ 776 KASSERT(task->ct_flags & TASK_KEY); 777 KASSERT((task->ct_flags & TASK_IV) == 0); 778 KASSERT((task->ct_flags & TASK_CTR) == 0); 779 KASSERT(task->ct_flags & TASK_SRC); 780 781 /* Set up the task descriptor queue control word. */ 782 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES, 783 SUN8I_CRYPTO_TDQC_METHOD); 784 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 785 786 #ifdef DIAGNOSTIC 787 switch (keysize) { 788 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128: 789 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16); 790 break; 791 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192: 792 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24); 793 break; 794 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256: 795 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32); 796 break; 797 } 798 #endif 799 800 /* Set up the symmetric control word. */ 801 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 802 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 803 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB, 804 SUN8I_CRYPTO_TDQS_OP_MODE); 805 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 806 807 /* Fill in the descriptor. */ 808 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0); 809 } 810 811 /* 812 * sun8i_crypto_submit(sc, task) 813 * 814 * Submit a task to the crypto engine after it has been loaded 815 * with sun8i_crypto_task_load. On success, guarantees to 816 * eventually call the task's callback. 817 */ 818 static int 819 sun8i_crypto_submit(struct sun8i_crypto_softc *sc, 820 struct sun8i_crypto_task *task) 821 { 822 unsigned i, retries = 0; 823 uint32_t icr; 824 int error = 0; 825 826 /* One at a time at the device registers, please. */ 827 mutex_enter(&sc->sc_lock); 828 829 /* Find a channel. */ 830 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 831 if (sc->sc_chan[i].cc_task == NULL) 832 break; 833 } 834 if (i == SUN8I_CRYPTO_NCHAN) { 835 device_printf(sc->sc_dev, "no free channels\n"); 836 error = ERESTART; 837 goto out; 838 } 839 840 /* 841 * Set the channel id. Caller is responsible for setting up 842 * all other parts of the descriptor. 843 */ 844 task->ct_desc->td_cid = htole32(i); 845 846 /* 847 * Prepare to send the descriptor to the device by DMA. 848 * Matches POSTWRITE in sun8i_crypto_chan_done. 849 */ 850 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 851 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE); 852 853 /* Confirm we're ready to go. */ 854 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) { 855 device_printf(sc->sc_dev, "TLR not clear\n"); 856 error = EIO; 857 goto out; 858 } 859 860 /* Enable interrupts for this channel. */ 861 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 862 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 863 SUN8I_CRYPTO_ICR_INTR_EN); 864 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 865 866 /* Set the task descriptor queue address. */ 867 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ, 868 task->ct_descmap->dm_segs[0].ds_addr); 869 870 /* Notify the engine to load it, and wait for acknowledgement. */ 871 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD); 872 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) 873 { 874 /* 875 * XXX Timeout pulled from arse. Is it even important 876 * to wait here? 877 */ 878 if (++retries == 1000) { 879 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n", 880 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR)); 881 /* 882 * Hope it clears eventually; if not, we'll 883 * time out. 884 */ 885 break; 886 } 887 DELAY(1); 888 } 889 890 /* 891 * Loaded up and ready to go. Start a timer ticking if it's 892 * not already. 893 */ 894 sc->sc_chan[i].cc_task = task; 895 sc->sc_chan[i].cc_starttime = getticks(); 896 if (!callout_pending(&sc->sc_timeout)) 897 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 898 899 /* XXX Consider polling if cold to get entropy earlier. */ 900 901 out: /* Done! */ 902 if (error) 903 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure, 904 sc, task, error); 905 else 906 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success, 907 sc, task, i); 908 mutex_exit(&sc->sc_lock); 909 return error; 910 } 911 912 /* 913 * sun8i_crypto_timeout(cookie) 914 * 915 * Timeout handler. Schedules work in a thread to cancel all 916 * pending tasks that were started long enough ago we're bored of 917 * waiting for them, and reschedules another timeout unless the 918 * channels are all idle. 919 */ 920 static void 921 sun8i_crypto_timeout(void *cookie) 922 { 923 struct sun8i_crypto_softc *sc = cookie; 924 unsigned i; 925 926 mutex_enter(&sc->sc_lock); 927 928 /* Check whether there are any tasks pending. */ 929 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 930 if (sc->sc_chan[i].cc_task) 931 break; 932 } 933 if (i == SUN8I_CRYPTO_NCHAN) 934 /* None pending, so nothing to do. */ 935 goto out; 936 937 /* 938 * Schedule the worker to check for timeouts, and schedule 939 * another timeout in case we need it. 940 */ 941 sun8i_crypto_schedule_worker(sc); 942 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT); 943 944 out: mutex_exit(&sc->sc_lock); 945 } 946 947 /* 948 * sun8i_crypto_intr(cookie) 949 * 950 * Device interrupt handler. Find what channels have completed, 951 * whether with success or with failure, and schedule work in 952 * thread context to invoke the appropriate callbacks. 953 */ 954 static int 955 sun8i_crypto_intr(void *cookie) 956 { 957 struct sun8i_crypto_softc *sc = cookie; 958 uint32_t isr, esr; 959 960 mutex_enter(&sc->sc_lock); 961 962 /* 963 * Get and acknowledge the interrupts and error status. 964 * 965 * XXX Data sheet says the error status register is read-only, 966 * but then advises writing 1 to bit x1xx (keysram access error 967 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do? 968 */ 969 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR); 970 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR); 971 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr); 972 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr); 973 974 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr); 975 976 /* Start the worker if necessary. */ 977 sun8i_crypto_schedule_worker(sc); 978 979 /* Tell the worker what to do. */ 980 sc->sc_done |= __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE); 981 sc->sc_esr |= esr; 982 983 mutex_exit(&sc->sc_lock); 984 985 return __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE) != 0; 986 } 987 988 /* 989 * sun8i_crypto_schedule_worker(sc) 990 * 991 * Ensure that crypto engine thread context work to invoke task 992 * callbacks will run promptly. Idempotent. 993 */ 994 static void 995 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc) 996 { 997 998 KASSERT(mutex_owned(&sc->sc_lock)); 999 1000 /* Start the worker if necessary. */ 1001 if (!sc->sc_work_pending) { 1002 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL); 1003 sc->sc_work_pending = true; 1004 } 1005 } 1006 1007 /* 1008 * sun8i_crypto_worker(wk, cookie) 1009 * 1010 * Thread-context worker: Invoke all task callbacks for which the 1011 * device has notified us of completion or for which we gave up 1012 * waiting. 1013 */ 1014 static void 1015 sun8i_crypto_worker(struct work *wk, void *cookie) 1016 { 1017 struct sun8i_crypto_softc *sc = cookie; 1018 uint32_t done, esr, esr_chan; 1019 unsigned i, now; 1020 int error; 1021 1022 /* 1023 * Acquire the lock. Note: We will be releasing and 1024 * reacquiring it throughout the loop. 1025 */ 1026 mutex_enter(&sc->sc_lock); 1027 1028 /* Acknowledge the work. */ 1029 KASSERT(sc->sc_work_pending); 1030 sc->sc_work_pending = false; 1031 1032 /* 1033 * Claim the done mask and error status once; we will be 1034 * releasing and reacquiring the lock for the callbacks, so 1035 * they may change. 1036 */ 1037 done = sc->sc_done; 1038 esr = sc->sc_esr; 1039 sc->sc_done = 0; 1040 sc->sc_esr = 0; 1041 1042 /* Check the time to determine what's timed out. */ 1043 now = getticks(); 1044 1045 /* Process the channels. */ 1046 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) { 1047 /* Check whether the channel is done. */ 1048 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) { 1049 /* Nope. Do we have a task to time out? */ 1050 if ((sc->sc_chan[i].cc_task != NULL) && 1051 ((now - sc->sc_chan[i].cc_starttime) >= 1052 SUN8I_CRYPTO_TIMEOUT)) 1053 sun8i_crypto_chan_done(sc, i, ETIMEDOUT); 1054 continue; 1055 } 1056 1057 /* Channel is done. Interpret the error if any. */ 1058 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i)); 1059 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) { 1060 device_printf(sc->sc_dev, "channel %u:" 1061 " alg not supported\n", i); 1062 error = ENODEV; 1063 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) { 1064 device_printf(sc->sc_dev, "channel %u:" 1065 " data length error\n", i); 1066 error = EIO; /* XXX */ 1067 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) { 1068 device_printf(sc->sc_dev, "channel %u:" 1069 " key sram error\n", i); 1070 error = EIO; /* XXX */ 1071 } else if (esr_chan != 0) { 1072 error = EIO; /* generic I/O error */ 1073 } else { 1074 error = 0; 1075 } 1076 1077 /* 1078 * Notify the task of completion. May release the lock 1079 * to invoke a callback. 1080 */ 1081 sun8i_crypto_chan_done(sc, i, error); 1082 } 1083 1084 /* All one; release the lock one last time. */ 1085 mutex_exit(&sc->sc_lock); 1086 } 1087 1088 /* 1089 * sun8i_crypto_chan_done(sc, i, error) 1090 * 1091 * Notify the callback for the task on channel i, if there is one, 1092 * of the specified error, or 0 for success. 1093 */ 1094 static void 1095 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error) 1096 { 1097 struct sun8i_crypto_task *task; 1098 uint32_t nbytes; 1099 uint32_t icr; 1100 1101 KASSERT(mutex_owned(&sc->sc_lock)); 1102 1103 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error); 1104 1105 /* Claim the task if there is one; bail if not. */ 1106 if ((task = sc->sc_chan[i].cc_task) == NULL) { 1107 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n", 1108 i, error); 1109 return; 1110 } 1111 sc->sc_chan[i].cc_task = NULL; 1112 1113 /* Disable interrupts on this channel. */ 1114 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR); 1115 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i), 1116 SUN8I_CRYPTO_ICR_INTR_EN); 1117 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr); 1118 1119 /* 1120 * Finished sending the descriptor to the device by DMA. 1121 * Matches PREWRITE in sun8i_crypto_task_submit. 1122 */ 1123 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0, 1124 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE); 1125 1126 /* 1127 * Finished with all the other bits of DMA too. Matches 1128 * sun8i_crypto_task_load. 1129 */ 1130 nbytes = task->ct_nbytes; 1131 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes, 1132 BUS_DMASYNC_POSTREAD); 1133 if (task->ct_flags & TASK_SRC) 1134 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes, 1135 BUS_DMASYNC_POSTWRITE); 1136 if (task->ct_flags & TASK_CTR) 1137 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0, 1138 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD); 1139 if (task->ct_flags & TASK_IV) 1140 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0, 1141 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1142 if (task->ct_flags & TASK_KEY) 1143 /* XXX Can we zero the bounce buffer if there is one? */ 1144 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0, 1145 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE); 1146 1147 /* Temporarily release the lock to invoke the callback. */ 1148 mutex_exit(&sc->sc_lock); 1149 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error); 1150 (*task->ct_callback)(sc, task, task->ct_cookie, error); 1151 mutex_enter(&sc->sc_lock); 1152 } 1153 1154 /* 1155 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags) 1156 * 1157 * Allocate a single-segment DMA-safe buffer and map it into KVA. 1158 * May fail if dmaflags is BUS_DMA_NOWAIT. 1159 */ 1160 static int 1161 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size, 1162 struct sun8i_crypto_buf *buf, int dmaflags) 1163 { 1164 int error; 1165 1166 /* Allocate a DMA-safe buffer. */ 1167 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0, 1168 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags); 1169 if (error) 1170 goto fail0; 1171 1172 /* Map the buffer into kernel virtual address space. */ 1173 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs, 1174 size, &buf->cb_kva, dmaflags); 1175 if (error) 1176 goto fail1; 1177 1178 /* Success! */ 1179 return 0; 1180 1181 fail2: __unused 1182 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1183 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1184 fail0: return error; 1185 } 1186 1187 /* 1188 * sun8i_crypto_freebuf(sc, buf) 1189 * 1190 * Unmap buf and free it. 1191 */ 1192 static void 1193 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size, 1194 struct sun8i_crypto_buf *buf) 1195 { 1196 1197 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size); 1198 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs); 1199 } 1200 1201 /* 1202 * sun8i_crypto_rng_attach(sc) 1203 * 1204 * Attach an rndsource for the crypto engine's TRNG. 1205 */ 1206 static void 1207 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc) 1208 { 1209 device_t self = sc->sc_dev; 1210 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1211 struct sun8i_crypto_task *task; 1212 int error; 1213 1214 /* Preallocate a buffer to reuse. */ 1215 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf, 1216 BUS_DMA_WAITOK); 1217 if (error) { 1218 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n", 1219 error); 1220 goto fail0; 1221 } 1222 1223 /* Create a task to reuse. */ 1224 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done, 1225 rng, PR_WAITOK); 1226 if (rng->cr_task == NULL) { 1227 aprint_error_dev(self, "failed to allocate RNG task\n"); 1228 error = ENOMEM; 1229 goto fail1; 1230 } 1231 1232 /* Preload the destination map. */ 1233 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1234 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1235 if (error) { 1236 aprint_error_dev(self, "failed to load RNG buffer: %d\n", 1237 error); 1238 goto fail2; 1239 } 1240 1241 /* 1242 * Attach the rndsource. This is _not_ marked as RND_TYPE_RNG 1243 * because the output is not uniformly distributed. The bits 1244 * are heavily weighted toward 0 or 1, at different times, and 1245 * I haven't scienced a satisfactory story out of it yet. 1246 */ 1247 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc); 1248 rnd_attach_source(&rng->cr_rndsource, device_xname(self), 1249 RND_TYPE_UNKNOWN, 1250 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB); 1251 1252 /* Success! */ 1253 return; 1254 1255 fail3: __unused 1256 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1257 fail2: sun8i_crypto_task_put(sc, task); 1258 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf); 1259 fail0: return; 1260 } 1261 1262 /* 1263 * sun8i_crypto_rng_get(nbytes, cookie) 1264 * 1265 * On-demand rndsource callback: try to gather nbytes of entropy 1266 * and enter them into the pool ASAP. 1267 */ 1268 static void 1269 sun8i_crypto_rng_get(size_t nbytes, void *cookie) 1270 { 1271 struct sun8i_crypto_softc *sc = cookie; 1272 struct sun8i_crypto_rng *rng = &sc->sc_rng; 1273 struct sun8i_crypto_task *task = rng->cr_task; 1274 bool pending; 1275 int error; 1276 1277 /* 1278 * Test and set the RNG-pending flag. If it's already in 1279 * progress, nothing to do here. 1280 */ 1281 mutex_enter(&sc->sc_lock); 1282 pending = rng->cr_pending; 1283 rng->cr_pending = true; 1284 mutex_exit(&sc->sc_lock); 1285 if (pending) 1286 return; 1287 1288 /* Load the task descriptor. */ 1289 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1290 if (error) 1291 goto fail; 1292 1293 /* Submit! */ 1294 error = sun8i_crypto_submit(sc, task); 1295 if (error) 1296 goto fail; 1297 1298 /* All done! */ 1299 return; 1300 1301 fail: mutex_enter(&sc->sc_lock); 1302 rng->cr_pending = false; 1303 mutex_exit(&sc->sc_lock); 1304 } 1305 1306 static void 1307 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc, 1308 struct sun8i_crypto_task *task, void *cookie, int error) 1309 { 1310 struct sun8i_crypto_rng *rng = cookie; 1311 uint8_t *buf = rng->cr_buf.cb_kva; 1312 uint32_t entropybits; 1313 1314 KASSERT(rng == &sc->sc_rng); 1315 1316 /* If anything went wrong, forget about it. */ 1317 if (error) 1318 goto out; 1319 1320 /* 1321 * This TRNG has quite low entropy at best. But if it fails a 1322 * repeated output test, then assume it's busted. 1323 */ 1324 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY); 1325 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY; 1326 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2, 1327 SUN8I_CRYPTO_RNGBYTES/2)) { 1328 device_printf(sc->sc_dev, "failed repeated output test\n"); 1329 entropybits = 0; 1330 } 1331 1332 /* 1333 * Actually we don't believe in any of the entropy until this 1334 * device has had more scrutiny. 1335 */ 1336 entropybits = 0; 1337 1338 /* Success! Enter and erase the data. */ 1339 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES, 1340 entropybits); 1341 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES); 1342 1343 out: /* Done -- clear the RNG-pending flag. */ 1344 mutex_enter(&sc->sc_lock); 1345 rng->cr_pending = false; 1346 mutex_exit(&sc->sc_lock); 1347 } 1348 1349 /* 1350 * Self-test 1351 */ 1352 1353 static const uint8_t selftest_input[16]; 1354 static const uint8_t selftest_key[16]; 1355 static const uint8_t selftest_output[16] = { 1356 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b, 1357 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e, 1358 }; 1359 1360 static void 1361 sun8i_crypto_selftest(device_t self) 1362 { 1363 const size_t keybytes = sizeof selftest_key; 1364 const size_t nbytes = sizeof selftest_input; 1365 struct sun8i_crypto_softc *sc = device_private(self); 1366 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest; 1367 struct sun8i_crypto_task *task; 1368 int error; 1369 1370 CTASSERT(sizeof selftest_input == sizeof selftest_output); 1371 1372 /* Allocate an input buffer. */ 1373 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in, 1374 BUS_DMA_WAITOK); 1375 if (error) 1376 goto fail0; 1377 1378 /* Allocate a key buffer. */ 1379 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key, 1380 BUS_DMA_WAITOK); 1381 if (error) 1382 goto fail1; 1383 1384 /* Allocate an output buffer. */ 1385 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out, 1386 BUS_DMA_WAITOK); 1387 if (error) 1388 goto fail2; 1389 1390 /* Allocate a task descriptor. */ 1391 task = selftest->cs_task = sun8i_crypto_task_get(sc, 1392 sun8i_crypto_selftest_done, selftest, PR_WAITOK); 1393 if (selftest->cs_task == NULL) { 1394 error = ENOMEM; 1395 goto fail3; 1396 } 1397 1398 /* Copy the input and key into their buffers. */ 1399 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes); 1400 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes); 1401 1402 /* Load the key, src, and dst for DMA transfers. */ 1403 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 1404 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK); 1405 if (error) 1406 goto fail4; 1407 task->ct_flags |= TASK_KEY; 1408 1409 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap, 1410 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1411 if (error) 1412 goto fail5; 1413 task->ct_flags |= TASK_SRC; 1414 1415 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1416 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK); 1417 if (error) 1418 goto fail6; 1419 1420 /* Set up the task descriptor. */ 1421 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes, 1422 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC); 1423 if (error) 1424 goto fail7; 1425 1426 /* Submit! */ 1427 error = sun8i_crypto_submit(sc, task); 1428 if (error) 1429 goto fail7; 1430 1431 device_printf(sc->sc_dev, "AES-128 self-test initiated\n"); 1432 1433 /* Success! */ 1434 return; 1435 1436 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1437 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1438 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1439 fail4: sun8i_crypto_task_put(sc, task); 1440 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1441 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1442 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1443 fail0: aprint_error_dev(self, "failed to run self-test, error=%d\n", error); 1444 } 1445 1446 static bool 1447 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title, 1448 size_t n, const void *expected, const void *actual) 1449 { 1450 const uint8_t *e = expected; 1451 const uint8_t *a = actual; 1452 size_t i; 1453 1454 if (memcmp(e, a, n) == 0) 1455 return true; 1456 1457 device_printf(sc->sc_dev, "self-test: %s\n", title); 1458 printf("expected: "); 1459 for (i = 0; i < n; i++) 1460 printf("%02hhx", e[i]); 1461 printf("\n"); 1462 printf("actual: "); 1463 for (i = 0; i < n; i++) 1464 printf("%02hhx", a[i]); 1465 printf("\n"); 1466 return false; 1467 } 1468 1469 static void 1470 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc, 1471 struct sun8i_crypto_task *task, void *cookie, int error) 1472 { 1473 const size_t keybytes = sizeof selftest_key; 1474 const size_t nbytes = sizeof selftest_input; 1475 struct sun8i_crypto_selftest *selftest = cookie; 1476 bool ok = true; 1477 1478 KASSERT(selftest == &sc->sc_selftest); 1479 1480 /* If anything went wrong, fail now. */ 1481 if (error) { 1482 device_printf(sc->sc_dev, "self-test error=%d\n", error); 1483 goto out; 1484 } 1485 1486 /* 1487 * Verify the input and key weren't clobbered, and verify the 1488 * output matches what we expect. 1489 */ 1490 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes, 1491 selftest_input, selftest->cs_in.cb_kva); 1492 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes, 1493 selftest_key, selftest->cs_key.cb_kva); 1494 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes, 1495 selftest_output, selftest->cs_out.cb_kva); 1496 1497 /* XXX Disable the RNG and other stuff if this fails... */ 1498 if (ok) 1499 device_printf(sc->sc_dev, "AES-128 self-test passed\n"); 1500 1501 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1502 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 1503 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 1504 sun8i_crypto_task_put(sc, task); 1505 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out); 1506 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key); 1507 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in); 1508 } 1509 1510 /* 1511 * Sysctl for testing 1512 */ 1513 1514 struct sun8i_crypto_userreq { 1515 kmutex_t cu_lock; 1516 kcondvar_t cu_cv; 1517 size_t cu_size; 1518 struct sun8i_crypto_buf cu_buf; 1519 struct sun8i_crypto_task *cu_task; 1520 int cu_error; 1521 bool cu_done; 1522 bool cu_cancel; 1523 }; 1524 1525 static void 1526 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc) 1527 { 1528 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl; 1529 int error; 1530 1531 /* hw.sun8icryptoN (node) */ 1532 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node, 1533 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev), 1534 SYSCTL_DESCR("sun8i crypto engine knobs"), 1535 NULL, 0, NULL, 0, 1536 CTL_HW, CTL_CREATE, CTL_EOL); 1537 if (error) { 1538 aprint_error_dev(sc->sc_dev, 1539 "failed to set up sysctl hw.%s: %d\n", 1540 device_xname(sc->sc_dev), error); 1541 return; 1542 } 1543 1544 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */ 1545 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node, 1546 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT, 1547 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"), 1548 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL); 1549 if (error) { 1550 aprint_error_dev(sc->sc_dev, 1551 "failed to set up sysctl hw.%s.rng: %d\n", 1552 device_xname(sc->sc_dev), error); 1553 return; 1554 } 1555 } 1556 1557 static int 1558 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS) 1559 { 1560 struct sysctlnode node = *rnode; 1561 struct sun8i_crypto_softc *sc = node.sysctl_data; 1562 struct sun8i_crypto_userreq *req; 1563 struct sun8i_crypto_task *task; 1564 size_t size; 1565 int error; 1566 1567 /* If oldp == NULL, the caller wants to learn the size. */ 1568 if (oldp == NULL) { 1569 *oldlenp = 4096; 1570 return 0; 1571 } 1572 1573 /* Truncate to 4096 bytes. */ 1574 size = MIN(4096, *oldlenp); 1575 if (size == 0) 1576 return 0; /* nothing to do */ 1577 1578 /* Allocate a request context. */ 1579 req = kmem_alloc(sizeof(*req), KM_NOSLEEP); 1580 if (req == NULL) 1581 return ENOMEM; 1582 1583 /* Initialize the request context. */ 1584 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE); 1585 cv_init(&req->cu_cv, "sun8isy"); 1586 req->cu_size = size; 1587 req->cu_error = EIO; 1588 req->cu_done = false; 1589 req->cu_cancel = false; 1590 1591 /* Allocate a buffer for the RNG output. */ 1592 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT); 1593 if (error) 1594 goto out0; 1595 1596 /* Allocate a task. */ 1597 task = req->cu_task = sun8i_crypto_task_get(sc, 1598 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT); 1599 if (task == NULL) { 1600 error = ENOMEM; 1601 goto out1; 1602 } 1603 1604 /* Set the task up for TRNG to our buffer. */ 1605 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap, 1606 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT); 1607 if (error) 1608 goto out2; 1609 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES); 1610 if (error) 1611 goto out3; 1612 1613 /* Submit! */ 1614 error = sun8i_crypto_submit(sc, task); 1615 if (error) { 1616 /* Make sure we don't restart the syscall -- just fail. */ 1617 if (error == ERESTART) 1618 error = EBUSY; 1619 goto out3; 1620 } 1621 1622 /* Wait for the request to complete. */ 1623 mutex_enter(&req->cu_lock); 1624 while (!req->cu_done) { 1625 error = cv_wait_sig(&req->cu_cv, &req->cu_lock); 1626 if (error) { 1627 /* 1628 * If we finished while waiting to acquire the 1629 * lock, ignore the error and just return now. 1630 * Otherwise, notify the callback that it has 1631 * to clean up after us. 1632 */ 1633 if (req->cu_done) 1634 error = 0; 1635 else 1636 req->cu_cancel = true; 1637 break; 1638 } 1639 } 1640 mutex_exit(&req->cu_lock); 1641 1642 /* 1643 * Return early on error from cv_wait_sig, which means 1644 * interruption; the callback will clean up instead. 1645 */ 1646 if (error) 1647 return error; 1648 1649 /* Check for error from the device. */ 1650 error = req->cu_error; 1651 if (error) 1652 goto out3; 1653 1654 /* Copy out the data. */ 1655 node.sysctl_data = req->cu_buf.cb_kva; 1656 node.sysctl_size = size; 1657 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1658 1659 /* Clear the buffer. */ 1660 explicit_memset(req->cu_buf.cb_kva, 0, size); 1661 1662 /* Clean up. */ 1663 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1664 out2: sun8i_crypto_task_put(sc, task); 1665 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1666 out0: cv_destroy(&req->cu_cv); 1667 mutex_destroy(&req->cu_lock); 1668 kmem_free(req, sizeof(*req)); 1669 return error; 1670 } 1671 1672 static void 1673 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc, 1674 struct sun8i_crypto_task *task, void *cookie, int error) 1675 { 1676 struct sun8i_crypto_userreq *req = cookie; 1677 bool cancel; 1678 1679 /* 1680 * Notify the waiting thread of the error, and find out whether 1681 * that thread cancelled. 1682 */ 1683 mutex_enter(&req->cu_lock); 1684 cancel = req->cu_cancel; 1685 req->cu_error = error; 1686 req->cu_done = true; 1687 cv_broadcast(&req->cu_cv); 1688 mutex_exit(&req->cu_lock); 1689 1690 /* 1691 * If it wasn't cancelled, we're done -- the main thread will 1692 * clean up after itself. 1693 */ 1694 if (!cancel) 1695 return; 1696 1697 /* Clean up after the main thread cancelled. */ 1698 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 1699 sun8i_crypto_task_put(sc, task); 1700 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf); 1701 cv_destroy(&req->cu_cv); 1702 mutex_destroy(&req->cu_lock); 1703 kmem_free(req, sizeof(*req)); 1704 } 1705 1706 /* 1707 * sun8i_crypto_register(sc) 1708 * 1709 * Register opencrypto algorithms supported by the crypto engine. 1710 */ 1711 static void 1712 sun8i_crypto_register(struct sun8i_crypto_softc *sc) 1713 { 1714 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto; 1715 1716 co->co_driverid = crypto_get_driverid(0); 1717 if (co->co_driverid == (uint32_t)-1) { 1718 aprint_error_dev(sc->sc_dev, 1719 "failed to register crypto driver\n"); 1720 return; 1721 } 1722 1723 sun8i_crypto_register1(sc, CRYPTO_AES_CBC); 1724 sun8i_crypto_register1(sc, CRYPTO_AES_CTR); 1725 #ifdef CRYPTO_AES_ECB 1726 sun8i_crypto_register1(sc, CRYPTO_AES_ECB); 1727 #endif 1728 #ifdef CRYPTO_AES_XTS 1729 sun8i_crypto_register1(sc, CRYPTO_AES_XTS); 1730 #endif 1731 #ifdef CRYPTO_DES_CBC 1732 sun8i_crypto_register1(sc, CRYPTO_DES_CBC); 1733 #endif 1734 #ifdef CRYPTO_DES_ECB 1735 sun8i_crypto_register1(sc, CRYPTO_DES_ECB); 1736 #endif 1737 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC); 1738 #ifdef CRYPTO_3DES_ECB 1739 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB); 1740 #endif 1741 1742 sun8i_crypto_register1(sc, CRYPTO_MD5); 1743 sun8i_crypto_register1(sc, CRYPTO_SHA1); 1744 #ifdef CRYPTO_SHA224 1745 sun8i_crypto_register1(sc, CRYPTO_SHA224); 1746 #endif 1747 #ifdef CRYPTO_SHA256 1748 sun8i_crypto_register1(sc, CRYPTO_SHA256); 1749 #endif 1750 1751 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC); 1752 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC); 1753 1754 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */ 1755 } 1756 1757 /* 1758 * sun8i_crypto_register1(sc, alg) 1759 * 1760 * Register support for one algorithm alg using 1761 * sun8i_crypto_newsession/freesession/process. 1762 */ 1763 static void 1764 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg) 1765 { 1766 1767 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0, 1768 sun8i_crypto_newsession, 1769 sun8i_crypto_freesession, 1770 sun8i_crypto_process, 1771 sc); 1772 } 1773 1774 /* 1775 * sun8i_crypto_newsession(cookie, sidp, cri) 1776 * 1777 * Called by opencrypto to allocate a new session. We don't keep 1778 * track of sessions, since there are no persistent keys in the 1779 * hardware that we take advantage of, so this only validates the 1780 * crypto operations and returns a zero session id. 1781 */ 1782 static int 1783 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri) 1784 { 1785 1786 /* No composition of operations is supported here. */ 1787 if (cri->cri_next) 1788 return EINVAL; 1789 1790 /* 1791 * No variation of rounds is supported here. (XXX Unused and 1792 * unimplemented in opencrypto(9) altogether? 1793 */ 1794 if (cri->cri_rnd) 1795 return EINVAL; 1796 1797 /* 1798 * Validate per-algorithm key length. 1799 * 1800 * XXX Does opencrypto(9) do this internally? 1801 */ 1802 switch (cri->cri_alg) { 1803 case CRYPTO_MD5: 1804 case CRYPTO_SHA1: 1805 #ifdef CRYPTO_SHA224 1806 case CRYPTO_SHA224: 1807 #endif 1808 #ifdef CRYPTO_SHA256 1809 case CRYPTO_SHA256: 1810 #endif 1811 if (cri->cri_klen) 1812 return EINVAL; 1813 break; 1814 case CRYPTO_AES_CBC: 1815 #ifdef CRYPTO_AES_ECB 1816 case CRYPTO_AES_ECB: 1817 #endif 1818 switch (cri->cri_klen) { 1819 case 128: 1820 case 192: 1821 case 256: 1822 break; 1823 default: 1824 return EINVAL; 1825 } 1826 break; 1827 case CRYPTO_AES_CTR: 1828 /* 1829 * opencrypto `AES-CTR' takes four bytes of the input 1830 * block as the last four bytes of the key, for reasons 1831 * that are not entirely clear. 1832 */ 1833 switch (cri->cri_klen) { 1834 case 128 + 32: 1835 case 192 + 32: 1836 case 256 + 32: 1837 break; 1838 default: 1839 return EINVAL; 1840 } 1841 break; 1842 #ifdef CRYPTO_AES_XTS 1843 case CRYPTO_AES_XTS: 1844 switch (cri->cri_klen) { 1845 case 256: 1846 case 384: 1847 case 512: 1848 break; 1849 default: 1850 return EINVAL; 1851 } 1852 break; 1853 #endif 1854 case CRYPTO_DES_CBC: 1855 #ifdef CRYPTO_DES_ECB 1856 case CRYPTO_DES_ECB: 1857 #endif 1858 switch (cri->cri_klen) { 1859 case 64: 1860 break; 1861 default: 1862 return EINVAL; 1863 } 1864 break; 1865 case CRYPTO_3DES_CBC: 1866 #ifdef CRYPTO_3DES_ECB 1867 case CRYPTO_3DES_ECB: 1868 #endif 1869 switch (cri->cri_klen) { 1870 case 192: 1871 break; 1872 default: 1873 return EINVAL; 1874 } 1875 break; 1876 case CRYPTO_SHA1_HMAC: 1877 /* 1878 * XXX Unclear what the length limit is, but since HMAC 1879 * behaves qualitatively different for a key of at 1880 * least the full block size -- and is generally best 1881 * to use with half the block size -- let's limit it to 1882 * one block. 1883 */ 1884 if (cri->cri_klen % 8) 1885 return EINVAL; 1886 if (cri->cri_klen > 512) 1887 return EINVAL; 1888 break; 1889 case CRYPTO_SHA2_256_HMAC: 1890 if (cri->cri_klen % 8) 1891 return EINVAL; 1892 if (cri->cri_klen > 512) 1893 return EINVAL; 1894 break; 1895 default: 1896 panic("unsupported algorithm %d", cri->cri_alg); 1897 } 1898 1899 KASSERT(cri->cri_klen % 8 == 0); 1900 1901 /* Success! */ 1902 *sidp = 1; 1903 return 0; 1904 } 1905 1906 /* 1907 * sun8i_crypto_freesession(cookie, dsid) 1908 * 1909 * Called by opencrypto to free a session. We don't keep track of 1910 * sessions, since there are no persistent keys in the hardware 1911 * that we take advantage of, so this is a no-op. 1912 * 1913 * Note: dsid is actually a 64-bit quantity containing both the 1914 * driver id in the high half and the session id in the low half. 1915 */ 1916 static int 1917 sun8i_crypto_freesession(void *cookie, uint64_t dsid) 1918 { 1919 1920 KASSERT((dsid & 0xffffffff) == 1); 1921 1922 /* Success! */ 1923 return 0; 1924 } 1925 1926 /* 1927 * sun8i_crypto_ivlen(crd) 1928 * 1929 * Return the crypto engine's notion of `IV length', in bytes, for 1930 * an opencrypto operation. 1931 */ 1932 static u_int 1933 sun8i_crypto_ivlen(const struct cryptodesc *crd) 1934 { 1935 1936 switch (crd->crd_alg) { 1937 case CRYPTO_AES_CBC: 1938 return 16; 1939 #ifdef CRYPTO_AES_XTS 1940 case CRYPTO_AES_XTS: 1941 return 16; 1942 #endif 1943 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */ 1944 return 8; 1945 #ifdef CRYPTO_DES_CBC 1946 case CRYPTO_DES_CBC: 1947 return 8; 1948 #endif 1949 case CRYPTO_3DES_CBC: 1950 return 8; 1951 case CRYPTO_MD5: 1952 return 16; 1953 #ifdef CRYPTO_SHA224 1954 case CRYPTO_SHA224: 1955 return 32; 1956 #endif 1957 #ifdef CRYPTO_SHA256 1958 case CRYPTO_SHA256: 1959 return 32; 1960 #endif 1961 case CRYPTO_SHA1_HMAC: 1962 return 20; 1963 case CRYPTO_SHA2_256_HMAC: 1964 return 32; 1965 default: 1966 return 0; 1967 } 1968 } 1969 1970 /* 1971 * sun8i_crypto_process(cookie, crp, hint) 1972 * 1973 * Main opencrypto processing dispatch. 1974 */ 1975 static int 1976 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint) 1977 { 1978 struct sun8i_crypto_softc *sc = cookie; 1979 struct sun8i_crypto_task *task; 1980 struct cryptodesc *crd = crp->crp_desc; 1981 unsigned klen, ivlen; 1982 uint32_t tdqc = 0, tdqs = 0; 1983 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0; 1984 const uint32_t tdqa = 0; 1985 int error; 1986 1987 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint); 1988 1989 /* Reject compositions -- we do not handle them. */ 1990 if (crd->crd_next != NULL) { 1991 error = EOPNOTSUPP; 1992 goto fail0; 1993 } 1994 1995 /* Reject transfers with nonsense skip. */ 1996 if (crd->crd_skip < 0) { 1997 error = EINVAL; 1998 goto fail0; 1999 } 2000 2001 /* 2002 * Actually just reject any nonzero skip, because it requires 2003 * DMA segment bookkeeping that we don't do yet. 2004 */ 2005 if (crd->crd_skip) { 2006 error = EOPNOTSUPP; 2007 goto fail0; 2008 } 2009 2010 /* Reject large transfers. */ 2011 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) { 2012 error = EFBIG; 2013 goto fail0; 2014 } 2015 2016 /* Reject nonsense, unaligned, or mismatched lengths. */ 2017 if (crd->crd_len < 0 || 2018 crd->crd_len % 4 || 2019 crd->crd_len != crp->crp_ilen) { 2020 error = EINVAL; 2021 goto fail0; 2022 } 2023 2024 /* Reject mismatched buffer lengths. */ 2025 /* XXX Handle crd_skip. */ 2026 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2027 struct mbuf *m = crp->crp_buf; 2028 uint32_t nbytes = 0; 2029 while (m != NULL) { 2030 KASSERT(m->m_len >= 0); 2031 if (m->m_len > crd->crd_len || 2032 nbytes > crd->crd_len - m->m_len) { 2033 error = EINVAL; 2034 goto fail0; 2035 } 2036 nbytes += m->m_len; 2037 m = m->m_next; 2038 } 2039 if (nbytes != crd->crd_len) { 2040 error = EINVAL; 2041 goto fail0; 2042 } 2043 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2044 struct uio *uio = crp->crp_buf; 2045 if (uio->uio_resid != crd->crd_len) { 2046 error = EINVAL; 2047 goto fail0; 2048 } 2049 } 2050 2051 /* Get a task, or fail with ERESTART if we can't. */ 2052 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp, 2053 PR_NOWAIT); 2054 if (task == NULL) { 2055 /* 2056 * Don't invoke crypto_done -- we are asking the 2057 * opencrypto(9) machinery to queue the request and get 2058 * back to us. 2059 */ 2060 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint); 2061 return ERESTART; 2062 } 2063 2064 /* Load key in, if relevant. */ 2065 klen = crd->crd_klen; 2066 if (klen) { 2067 if (crd->crd_alg == CRYPTO_AES_CTR) 2068 /* AES-CTR is special -- see IV processing below. */ 2069 klen -= 32; 2070 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap, 2071 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT); 2072 if (error) 2073 goto fail1; 2074 task->ct_flags |= TASK_KEY; 2075 } 2076 2077 /* Handle the IV, if relevant. */ 2078 ivlen = sun8i_crypto_ivlen(crd); 2079 if (ivlen) { 2080 void *iv; 2081 2082 /* 2083 * If there's an explicit IV, use it; otherwise 2084 * randomly generate one. 2085 */ 2086 if (crd->crd_flags & CRD_F_IV_EXPLICIT) { 2087 iv = crd->crd_iv; 2088 } else { 2089 cprng_fast(task->ct_iv, ivlen); 2090 iv = task->ct_iv; 2091 } 2092 2093 /* 2094 * If the IV is not already present in the user's 2095 * buffer, copy it over. 2096 */ 2097 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) { 2098 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2099 m_copyback(crp->crp_buf, crd->crd_inject, 2100 ivlen, iv); 2101 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2102 cuio_copyback(crp->crp_buf, crd->crd_inject, 2103 ivlen, iv); 2104 } else { 2105 panic("invalid buffer type %x", 2106 crp->crp_flags); 2107 } 2108 } 2109 2110 /* 2111 * opencrypto's idea of `AES-CTR' is special. 2112 * 2113 * - The low 4 bytes of the input block are drawn from 2114 * an extra 4 bytes at the end of the key. 2115 * 2116 * - The next 8 bytes of the input block are drawn from 2117 * the opencrypto iv. 2118 * 2119 * - The high 4 bytes are the big-endian block counter, 2120 * which starts at 1 because why not. 2121 */ 2122 if (crd->crd_alg == CRYPTO_AES_CTR) { 2123 uint8_t block[16]; 2124 uint32_t blkno = 1; 2125 2126 /* Format the initial input block. */ 2127 memcpy(block, crd->crd_key + klen/8, 4); 2128 memcpy(block + 4, iv, 8); 2129 be32enc(block + 12, blkno); 2130 2131 /* Copy it into the DMA buffer. */ 2132 memcpy(task->ct_iv, block, 16); 2133 iv = task->ct_iv; 2134 ivlen = 16; 2135 } 2136 2137 /* Load the IV. */ 2138 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen, 2139 NULL, BUS_DMA_NOWAIT); 2140 if (error) 2141 goto fail1; 2142 task->ct_flags |= TASK_IV; 2143 } 2144 2145 /* Load the src and dst. */ 2146 if (crp->crp_flags & CRYPTO_F_IMBUF) { 2147 struct mbuf *m = crp->crp_buf; 2148 2149 /* XXX Handle crd_skip. */ 2150 KASSERT(crd->crd_skip == 0); 2151 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m, 2152 BUS_DMA_NOWAIT); 2153 if (error) 2154 goto fail1; 2155 task->ct_flags |= TASK_SRC; 2156 2157 /* XXX Handle crd_skip. */ 2158 KASSERT(crd->crd_skip == 0); 2159 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m, 2160 BUS_DMA_NOWAIT); 2161 if (error) 2162 goto fail1; 2163 } else if (crp->crp_flags & CRYPTO_F_IOV) { 2164 struct uio *uio = crp->crp_buf; 2165 2166 /* XXX Handle crd_skip. */ 2167 KASSERT(crd->crd_skip == 0); 2168 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio, 2169 BUS_DMA_NOWAIT); 2170 if (error) 2171 goto fail1; 2172 task->ct_flags |= TASK_SRC; 2173 2174 /* XXX Handle crd_skip. */ 2175 KASSERT(crd->crd_skip == 0); 2176 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio, 2177 BUS_DMA_NOWAIT); 2178 if (error) 2179 goto fail1; 2180 } else { 2181 panic("invalid buffer type %x", crp->crp_flags); 2182 } 2183 2184 /* Set the encryption direction. */ 2185 if (crd->crd_flags & CRD_F_ENCRYPT) 2186 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC; 2187 else 2188 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC; 2189 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR); 2190 2191 /* Set the method. */ 2192 switch (crd->crd_alg) { 2193 case CRYPTO_AES_CBC: 2194 case CRYPTO_AES_CTR: 2195 #ifdef CRYPTO_AES_ECB 2196 case CRYPTO_AES_ECB: 2197 #endif 2198 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2199 break; 2200 #ifdef CRYPTO_AES_XTS 2201 case CRYPTO_AES_XTS: 2202 method = SUN8I_CRYPTO_TDQC_METHOD_AES; 2203 break; 2204 #endif 2205 case CRYPTO_DES_CBC: 2206 #ifdef CRYPTO_DES_ECB 2207 case CRYPTO_DES_ECB: 2208 #endif 2209 method = SUN8I_CRYPTO_TDQC_METHOD_DES; 2210 break; 2211 case CRYPTO_3DES_CBC: 2212 #ifdef CRYPTO_3DES_ECB 2213 case CRYPTO_3DES_ECB: 2214 #endif 2215 method = SUN8I_CRYPTO_TDQC_METHOD_3DES; 2216 break; 2217 case CRYPTO_MD5: 2218 method = SUN8I_CRYPTO_TDQC_METHOD_MD5; 2219 break; 2220 case CRYPTO_SHA1: 2221 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1; 2222 break; 2223 #ifdef CRYPTO_SHA224 2224 case CRYPTO_SHA224: 2225 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224; 2226 break; 2227 #endif 2228 #ifdef CRYPTO_SHA256 2229 case CRYPTO_SHA256: 2230 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256; 2231 break; 2232 #endif 2233 case CRYPTO_SHA1_HMAC: 2234 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1; 2235 break; 2236 case CRYPTO_SHA2_256_HMAC: 2237 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256; 2238 break; 2239 default: 2240 panic("unknown algorithm %d", crd->crd_alg); 2241 } 2242 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD); 2243 2244 /* Set the key selector. No idea how to use the internal keys. */ 2245 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx, 2246 SUN8I_CRYPTO_TDQS_SKEY_SELECT); 2247 2248 /* XXX Deal with AES_CTS_Last_Block_Flag. */ 2249 2250 /* Set the mode. */ 2251 switch (crd->crd_alg) { 2252 #ifdef CRYPTO_AES_ECB 2253 case CRYPTO_AES_ECB: 2254 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2255 break; 2256 #endif 2257 #ifdef CRYPTO_DES_ECB 2258 case CRYPTO_DES_ECB: 2259 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2260 break; 2261 #endif 2262 #ifdef CRYPTO_3DES_ECB 2263 case CRYPTO_3DES_ECB: 2264 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB; 2265 break; 2266 #endif 2267 case CRYPTO_AES_CBC: 2268 case CRYPTO_DES_CBC: 2269 case CRYPTO_3DES_CBC: 2270 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC; 2271 break; 2272 case CRYPTO_AES_CTR: 2273 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR; 2274 break; 2275 #ifdef CRYPTO_AES_XTS 2276 case CRYPTO_AES_XTS: 2277 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS; 2278 break; 2279 #endif 2280 default: 2281 panic("unknown algorithm %d", crd->crd_alg); 2282 } 2283 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE); 2284 2285 /* Set the CTR width. */ 2286 switch (crd->crd_alg) { 2287 case CRYPTO_AES_CTR: 2288 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32; 2289 break; 2290 } 2291 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH); 2292 2293 /* Set the AES key size. */ 2294 switch (crd->crd_alg) { 2295 case CRYPTO_AES_CBC: 2296 #ifdef CRYPTO_AES_ECB 2297 case CRYPTO_AES_ECB: 2298 #endif 2299 switch (crd->crd_klen) { 2300 case 128: 2301 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2302 break; 2303 case 192: 2304 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2305 break; 2306 case 256: 2307 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2308 break; 2309 default: 2310 panic("invalid AES key size in bits: %u", 2311 crd->crd_klen); 2312 } 2313 break; 2314 case CRYPTO_AES_CTR: 2315 switch (crd->crd_klen) { 2316 case 128 + 32: 2317 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2318 break; 2319 case 192 + 32: 2320 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2321 break; 2322 case 256 + 32: 2323 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2324 break; 2325 default: 2326 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u", 2327 crd->crd_klen); 2328 } 2329 break; 2330 #ifdef CRYPTO_AES_XTS 2331 case CRYPTO_AES_XTS: 2332 switch (crd->crd_klen) { 2333 case 256: 2334 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128; 2335 break; 2336 case 384: 2337 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192; 2338 break; 2339 case 512: 2340 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256; 2341 break; 2342 default: 2343 panic("invalid AES-XTS key size in bits: %u", 2344 crd->crd_klen); 2345 } 2346 break; 2347 #endif 2348 } 2349 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE); 2350 2351 /* Set up the task descriptor. */ 2352 error = sun8i_crypto_task_load(sc, task, crd->crd_len, 2353 tdqc, tdqs, tdqa); 2354 if (error) 2355 goto fail2; 2356 2357 /* Submit! */ 2358 error = sun8i_crypto_submit(sc, task); 2359 if (error) 2360 goto fail2; 2361 2362 /* Success! */ 2363 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task); 2364 return 0; 2365 2366 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2367 fail1: if (task->ct_flags & TASK_SRC) 2368 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2369 if (task->ct_flags & TASK_CTR) 2370 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2371 if (task->ct_flags & TASK_IV) 2372 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2373 if (task->ct_flags & TASK_KEY) 2374 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2375 sun8i_crypto_task_put(sc, task); 2376 fail0: KASSERT(error); 2377 KASSERT(error != ERESTART); 2378 crp->crp_etype = error; 2379 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2380 crypto_done(crp); 2381 return 0; 2382 } 2383 2384 /* 2385 * sun8i_crypto_callback(sc, task, cookie, error) 2386 * 2387 * Completion callback for a task submitted via opencrypto. 2388 * Release the task and pass the error on to opencrypto with 2389 * crypto_done. 2390 */ 2391 static void 2392 sun8i_crypto_callback(struct sun8i_crypto_softc *sc, 2393 struct sun8i_crypto_task *task, void *cookie, int error) 2394 { 2395 struct cryptop *crp = cookie; 2396 struct cryptodesc *crd = crp->crp_desc; 2397 2398 KASSERT(error != ERESTART); 2399 KASSERT(crd != NULL); 2400 KASSERT(crd->crd_next == NULL); 2401 2402 /* Return the number of bytes processed. */ 2403 crp->crp_olen = error ? 0 : crp->crp_ilen; 2404 2405 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap); 2406 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap); 2407 if (task->ct_flags & TASK_CTR) 2408 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap); 2409 if (task->ct_flags & TASK_IV) 2410 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap); 2411 if (task->ct_flags & TASK_KEY) 2412 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap); 2413 sun8i_crypto_task_put(sc, task); 2414 KASSERT(error != ERESTART); 2415 crp->crp_etype = error; 2416 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error); 2417 crypto_done(crp); 2418 } 2419
Indexes created Fri Sep 26 08:10:20 GMT 2025