Home | History | Annotate | Line # | Download | only in marvell
      1 /*	$NetBSD: mvcesa.c,v 1.6 2022/05/22 11:39:27 riastradh Exp $	*/
      2 /*
      3  * Copyright (c) 2008 KIYOHARA Takashi
      4  * All rights reserved.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  *
     15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     18  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
     19  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     20  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     21  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
     23  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
     24  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     25  * POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #include <sys/cdefs.h>
     29 __KERNEL_RCSID(0, "$NetBSD: mvcesa.c,v 1.6 2022/05/22 11:39:27 riastradh Exp $");
     30 
     31 #include <sys/param.h>
     32 #include <sys/bus.h>
     33 #include <sys/cprng.h>
     34 #include <sys/device.h>
     35 #include <sys/endian.h>
     36 #include <sys/errno.h>
     37 #include <sys/mbuf.h>
     38 #include <sys/md5.h>
     39 #include <sys/uio.h>
     40 #include <sys/sha1.h>
     41 
     42 #include <opencrypto/cryptodev.h>
     43 #include <opencrypto/xform.h>
     44 
     45 #include <dev/marvell/marvellreg.h>
     46 #include <dev/marvell/marvellvar.h>
     47 #include <dev/marvell/mvcesareg.h>
     48 
     49 #include "locators.h"
     50 
     51 #define MVCESA_SESSION(sid)		((sid) & 0x0fffffff)
     52 #define MVCESA_SID(crd, sesn)		(((crd) << 28) | ((sesn) & 0x0fffffff))
     53 
     54 
     55 struct mvcesa_session {
     56 	int ses_used;
     57 
     58 	int ses_klen;
     59 	uint32_t ses_key[8];
     60 
     61 	uint32_t ses_hminner[5];	/* HMAC inner state */
     62 	uint32_t ses_hmouter[5];	/* HMAC outer state */
     63 };
     64 
     65 struct mvcesa_softc {
     66 	device_t sc_dev;
     67 
     68 	bus_space_tag_t sc_iot;
     69 	bus_space_handle_t sc_ioh;
     70 	bus_dma_tag_t sc_dmat;
     71 
     72 	int sc_cid;
     73 	int sc_nsessions;
     74 	struct mvcesa_session *sc_sessions;
     75 };
     76 
     77 static int mvcesa_match(device_t, cfdata_t, void *);
     78 static void mvcesa_attach(device_t, device_t, void *);
     79 
     80 static int mvcesa_intr(void *);
     81 
     82 static int mvcesa_newsession(void *, u_int32_t *, struct cryptoini *);
     83 static void mvcesa_freesession(void *, u_int64_t);
     84 static int mvcesa_process(void *, struct cryptop *, int);
     85 
     86 static int mvcesa_authentication(struct mvcesa_softc *, struct mvcesa_session *,
     87 				 uint32_t, uint32_t *, uint32_t *, uint64_t,
     88 				 int, int, char *, struct mbuf *, struct uio *);
     89 static int mvcesa_des_encdec(struct mvcesa_softc *, struct mvcesa_session *,
     90 			     uint32_t, uint32_t, uint32_t, uint32_t *, int, int,
     91 			     char *, struct mbuf *, struct uio *);
     92 
     93 
     94 CFATTACH_DECL_NEW(mvcesa_gt, sizeof(struct mvcesa_softc),
     95     mvcesa_match, mvcesa_attach, NULL, NULL);
     96 CFATTACH_DECL_NEW(mvcesa_mbus, sizeof(struct mvcesa_softc),
     97     mvcesa_match, mvcesa_attach, NULL, NULL);
     98 
     99 
    100 /* ARGSUSED */
    101 static int
    102 mvcesa_match(device_t parent, cfdata_t match, void *aux)
    103 {
    104 	struct marvell_attach_args *mva = aux;
    105 
    106 	if (strcmp(mva->mva_name, match->cf_name) != 0)
    107 		return 0;
    108 	if (mva->mva_offset == MVA_OFFSET_DEFAULT ||
    109 	    mva->mva_irq == MVA_IRQ_DEFAULT)
    110 		return 0;
    111 
    112 	mva->mva_size = MVCESA_SIZE;
    113 	return 1;
    114 }
    115 
    116 /* ARGSUSED */
    117 static void
    118 mvcesa_attach(device_t parent, device_t self, void *aux)
    119 {
    120 	struct mvcesa_softc *sc = device_private(self);
    121 	struct marvell_attach_args *mva = aux;
    122 
    123 	aprint_normal(
    124 	    ": Marvell Cryptographic Engines and Security Accelerator\n");
    125 	aprint_naive("\n");
    126 
    127 	sc->sc_dev = self;
    128 	sc->sc_iot = mva->mva_iot;
    129         /* Map I/O registers */
    130 	if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, mva->mva_offset,
    131 	    mva->mva_size, &sc->sc_ioh)) {
    132 		aprint_error_dev(self, "can't map registers\n");
    133 		return;
    134 	}
    135 	sc->sc_dmat = mva->mva_dmat;
    136 
    137 	sc->sc_nsessions = 0;
    138 
    139 	/* Setup Opencrypto stuff */
    140 	sc->sc_cid = crypto_get_driverid(0);
    141 	if (sc->sc_cid < 0) {
    142 		aprint_error_dev(self, "couldn't get crypto driver id\n");
    143 		return;
    144 	}
    145 	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
    146 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    147 	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
    148 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    149 #if __DMA_notyet__
    150 /*
    151  * Don't know how to process to AES CBC in PIO-mode.
    152  * I havn't found IV registers.
    153  */
    154 	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
    155 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    156 #endif
    157 	crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0,
    158 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    159 	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
    160 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    161 	crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0,
    162 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    163 	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
    164 	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
    165 
    166 	/* Clear and establish interrupt */
    167 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_IC, 0);
    168 	marvell_intr_establish(mva->mva_irq, IPL_NET, mvcesa_intr, sc);
    169 
    170 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_IM, 0);
    171 }
    172 
    173 
    174 static int
    175 mvcesa_intr(void *arg)
    176 {
    177 #if 0
    178 	struct mvcesa_softc *sc = (struct mvcesa_softc *)arg;
    179 #endif
    180 	int handled = 0;
    181 
    182 	return handled;
    183 }
    184 
    185 
    186 /*
    187  * Opencrypto functions
    188  */
    189 /*
    190  * Allocate a new 'session' and return an encoded session id.  'sidp'
    191  * contains our registration id, and should contain an encoded session
    192  * id on successful allocation.
    193  */
    194 static int
    195 mvcesa_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
    196 {
    197 	struct mvcesa_softc *sc = (struct mvcesa_softc *)arg;
    198 	struct cryptoini *c;
    199 	struct mvcesa_session *ses = NULL;
    200 	int sesn, count, enc, mac, i;
    201 
    202 	KASSERT(sc != NULL /*, ("mvcesa_newsession: null softc")*/);
    203 	if (sidp == NULL || cri == NULL || sc == NULL)
    204 		return EINVAL;
    205 
    206 	for (sesn = 0; sesn < sc->sc_nsessions; sesn++)
    207 		if (sc->sc_sessions[sesn].ses_used == 0) {
    208 			ses = sc->sc_sessions + sesn;
    209 			break;
    210 		}
    211 
    212 	if (ses == NULL) {
    213 		sesn = sc->sc_nsessions;
    214 		ses = malloc((sesn + 1) * sizeof(*ses), M_DEVBUF, M_NOWAIT);
    215 		if (ses == NULL)
    216 			return ENOMEM;
    217 		if (sesn != 0) {
    218 			memcpy(ses, sc->sc_sessions, sesn * sizeof(*ses));
    219 			memset(sc->sc_sessions, 0, sesn * sizeof(*ses));
    220 			free(sc->sc_sessions, M_DEVBUF);
    221 		}
    222 		sc->sc_sessions = ses;
    223 		ses = sc->sc_sessions + sesn;
    224 		sc->sc_nsessions++;
    225 	}
    226 	memset(ses, 0, sizeof(*ses));
    227 
    228 	count = 0;
    229 	enc = mac = 0;
    230 	for (c = cri; c != NULL; c = c->cri_next) {
    231 		switch (c->cri_alg) {
    232 		case CRYPTO_DES_CBC:
    233 		case CRYPTO_3DES_CBC:
    234 			if (enc)
    235 				return EINVAL;
    236 			enc = 1;
    237 
    238 			/* Go ahead and compute key in CESA's byte order */
    239 			ses->ses_klen = c->cri_klen;
    240 			memcpy(ses->ses_key, c->cri_key, c->cri_klen / 8);
    241 			switch (c->cri_alg) {
    242 			case CRYPTO_3DES_CBC:
    243 				ses->ses_key[5] = htobe32(ses->ses_key[5]);
    244 				ses->ses_key[4] = htobe32(ses->ses_key[4]);
    245 				ses->ses_key[3] = htobe32(ses->ses_key[3]);
    246 				ses->ses_key[2] = htobe32(ses->ses_key[2]);
    247 
    248 				/* FALLTHROUGH */
    249 			case CRYPTO_DES_CBC:
    250 				ses->ses_key[1] = htobe32(ses->ses_key[1]);
    251 				ses->ses_key[0] = htobe32(ses->ses_key[0]);
    252 			}
    253 			break;
    254 
    255 		case CRYPTO_SHA1_HMAC:
    256 		case CRYPTO_MD5_HMAC:
    257 		{
    258 			MD5_CTX md5ctx;
    259 			SHA1_CTX sha1ctx;
    260 			int klen_bytes = c->cri_klen / 8;
    261 
    262 			KASSERT(c->cri_klen == 512);
    263 
    264 			for (i = 0; i < klen_bytes; i++)
    265 				c->cri_key[i] ^= HMAC_IPAD_VAL;
    266 			if (c->cri_alg == CRYPTO_MD5_HMAC_96) {
    267 				MD5Init(&md5ctx);
    268 				MD5Update(&md5ctx, c->cri_key, klen_bytes);
    269 				MD5Update(&md5ctx, hmac_ipad_buffer,
    270 				    HMAC_BLOCK_LEN - klen_bytes);
    271 				memcpy(ses->ses_hminner, md5ctx.state,
    272 				    sizeof(md5ctx.state));
    273 			} else {
    274 				SHA1Init(&sha1ctx);
    275 				SHA1Update(&sha1ctx, c->cri_key, klen_bytes);
    276 				SHA1Update(&sha1ctx, hmac_ipad_buffer,
    277 				    HMAC_BLOCK_LEN - klen_bytes);
    278 				memcpy(ses->ses_hminner, sha1ctx.state,
    279 				    sizeof(sha1ctx.state));
    280 			}
    281 
    282 			for (i = 0; i < klen_bytes; i++)
    283 				c->cri_key[i] ^=
    284 				    (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
    285 			if (c->cri_alg == CRYPTO_MD5_HMAC_96) {
    286 				MD5Init(&md5ctx);
    287 				MD5Update(&md5ctx, c->cri_key, klen_bytes);
    288 				MD5Update(&md5ctx, hmac_opad_buffer,
    289 				    HMAC_BLOCK_LEN - klen_bytes);
    290 				memcpy(ses->ses_hmouter, md5ctx.state,
    291 				    sizeof(md5ctx.state));
    292 			} else {
    293 				SHA1Init(&sha1ctx);
    294 				SHA1Update(&sha1ctx, c->cri_key, klen_bytes);
    295 				SHA1Update(&sha1ctx, hmac_opad_buffer,
    296 				    HMAC_BLOCK_LEN - klen_bytes);
    297 				memcpy(ses->ses_hmouter, sha1ctx.state,
    298 				    sizeof(sha1ctx.state));
    299 			}
    300 
    301 			for (i = 0; i < klen_bytes; i++)
    302 				c->cri_key[i] ^= HMAC_OPAD_VAL;
    303 		}
    304 			/* FALLTHROUGH */
    305 
    306 		case CRYPTO_SHA1:
    307 		case CRYPTO_MD5:
    308 			if (mac)
    309 				return EINVAL;
    310 			mac = 1;
    311 		}
    312 		count++;
    313 	}
    314 	if (count > 2) {
    315 		mvcesa_freesession(sc, sesn);
    316 		return EINVAL;
    317 	}
    318 
    319 	*sidp = MVCESA_SID(device_unit(sc->sc_dev), sesn);
    320 	ses->ses_used = 1;
    321 
    322 	return 0;
    323 }
    324 
    325 /*
    326  * Deallocate a session.
    327  */
    328 static void
    329 mvcesa_freesession(void *arg, u_int64_t tid)
    330 {
    331 	struct mvcesa_softc *sc = (struct mvcesa_softc *)arg;
    332 	int session;
    333 	uint32_t sid = ((uint32_t)tid) & 0xffffffff;
    334 
    335 	session = MVCESA_SESSION(sid);
    336 	KASSERTMSG(session >= 0, "session=%d", session);
    337 	KASSERTMSG(session < sc->sc_nsessions, "session=%d nsessions=%d",
    338 	    session, sc->sc_nsessions);
    339 
    340 	memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session]));
    341 }
    342 
    343 static int
    344 mvcesa_process(void *arg, struct cryptop *crp, int hint)
    345 {
    346 	struct mvcesa_softc *sc = arg;
    347 	struct mvcesa_session *ses;
    348 	struct cryptodesc *crd;
    349 	struct mbuf *m = NULL;
    350 	struct uio *uio = NULL;
    351 	int session;
    352 	char *buf = NULL;
    353 
    354 	session = MVCESA_SESSION(crp->crp_sid);
    355 	KASSERTMSG(session < sc->sc_nsessions, "session=%d nsessions=%d",
    356 	    session, sc->sc_nsessions);
    357 	ses = &sc->sc_sessions[session];
    358 
    359 	if (crp->crp_flags & CRYPTO_F_IMBUF)
    360 		m = (struct mbuf *)crp->crp_buf;
    361 	else if (crp->crp_flags & CRYPTO_F_IOV)
    362 		uio = (struct uio *)crp->crp_buf;
    363 	else
    364 		buf = (char *)crp->crp_buf;
    365 
    366 	if (0 /* DMA support */) {
    367 		/* not yet... */
    368 
    369 		goto done;
    370 	}
    371 
    372 	/* PIO operation */
    373 
    374 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
    375 		switch (crd->crd_alg) {
    376 		case CRYPTO_DES_CBC:
    377 		case CRYPTO_3DES_CBC:
    378 		{
    379 			uint32_t alg, mode, dir, *iv, ivbuf[2];
    380 
    381 			mode = MVCESA_DESE_C_DESMODE_CBC;
    382 			if (crd->crd_alg == CRYPTO_DES_CBC)
    383 				alg = MVCESA_DESE_C_ALGORITHM_DES;
    384 			else {	/* CRYPTO_3DES_CBC */
    385 				alg = MVCESA_DESE_C_ALGORITHM_3DES;
    386 				mode |= MVCESA_DESE_C_3DESMODE_EDE;
    387 			}
    388 			if (crd->crd_flags & CRD_F_ENCRYPT) {
    389 				dir = MVCESA_DESE_C_DIRECTION_ENC;
    390 				if (crd->crd_flags & CRD_F_IV_EXPLICIT)
    391 					iv = (uint32_t *)crd->crd_iv;
    392 				else {
    393 					cprng_fast(ivbuf, sizeof(ivbuf));
    394 					iv = ivbuf;
    395 				}
    396 				if (!(crd->crd_flags & CRD_F_IV_PRESENT)) {
    397 					if (m != NULL)
    398 						m_copyback(m, crd->crd_inject,
    399 						    8, iv);
    400 					else if (uio != NULL)
    401 						cuio_copyback(uio,
    402 						    crd->crd_inject, 8, iv);
    403 				}
    404 			} else {
    405 				dir = MVCESA_DESE_C_DIRECTION_DEC;
    406 				if (crd->crd_flags & CRD_F_IV_EXPLICIT)
    407 					iv = (uint32_t *)crd->crd_iv;
    408 				else {
    409 					if (m != NULL)
    410 						m_copydata(m, crd->crd_inject,
    411 						    8, ivbuf);
    412 					else if (uio != NULL)
    413 						cuio_copydata(uio,
    414 						    crd->crd_inject, 8, ivbuf);
    415 					iv = ivbuf;
    416 				}
    417 			}
    418 
    419 			crp->crp_etype = mvcesa_des_encdec(sc, ses,
    420 			    alg, mode, dir, iv, crd->crd_skip, crd->crd_len,
    421 			    buf, m, uio);
    422 			break;
    423 		}
    424 
    425 		case CRYPTO_SHA1:
    426 		case CRYPTO_SHA1_HMAC:
    427 		case CRYPTO_MD5:
    428 		case CRYPTO_MD5_HMAC:
    429 		{
    430 			uint64_t bits;
    431 			uint32_t alg, *iv = NULL, digest[512 / 8 / 4], dlen;
    432 
    433 			if (crd->crd_alg == CRYPTO_SHA1 ||
    434 			    crd->crd_alg == CRYPTO_SHA1_HMAC) {
    435 				alg = MVCESA_SHA1MD5I_AC_ALGORITHM_SHA1;
    436 				dlen = 160;
    437 			} else {	/* CRYPTO_MD5 || CRYPTO_MD5_HMAC */
    438 				alg = MVCESA_SHA1MD5I_AC_ALGORITHM_MD5;
    439 				dlen = 128;
    440 			}
    441 			bits = crd->crd_len << 3;
    442 			if (crd->crd_alg == CRYPTO_SHA1_HMAC ||
    443 			    crd->crd_alg == CRYPTO_MD5_HMAC) {
    444 				iv = ses->ses_hminner;
    445 				bits += 512;
    446 			}
    447 
    448 			crp->crp_etype = mvcesa_authentication(sc, ses,
    449 			    alg, iv, digest, bits, crd->crd_skip, crd->crd_len,
    450 			    buf, m, uio);
    451 			if (crp->crp_etype != 0)
    452 				break;
    453 
    454 			if (crd->crd_alg == CRYPTO_SHA1_HMAC ||
    455 			    crd->crd_alg == CRYPTO_MD5_HMAC)
    456 				crp->crp_etype = mvcesa_authentication(sc,
    457 				    ses, alg, ses->ses_hmouter, digest,
    458 				    512 + dlen, 0, dlen, (char *)digest, NULL,
    459 				    NULL);
    460 			if (crp->crp_etype != 0)
    461 				break;
    462 
    463 			/* Inject the authentication data */
    464 			if (buf != NULL)
    465 				memcpy(buf + crd->crd_inject, digest, dlen / 8);
    466 			else if (m != NULL)
    467 				m_copyback(m, crd->crd_inject, dlen / 8,
    468 				    digest);
    469 			else if (uio != NULL)
    470 				memcpy(crp->crp_mac, digest, dlen / 8);
    471 		}
    472 		}
    473 		if (crp->crp_etype != 0)
    474 			break;
    475 	}
    476 
    477 done:
    478 	DPRINTF(("request %08x done\n", (uint32_t)crp));
    479 	crypto_done(crp);
    480 	return 0;
    481 }
    482 
    483 
    484 static int
    485 mvcesa_authentication(struct mvcesa_softc *sc, struct mvcesa_session *ses,
    486 		      uint32_t alg, uint32_t *iv, uint32_t *digest,
    487 		      uint64_t bits, int skip, int len, char *buf,
    488 		      struct mbuf *m, struct uio *uio)
    489 {
    490 	uint32_t cmd, bswp, data = 0;
    491 	int dlen, off, i, s;
    492 
    493 	/*
    494 	 * SHA/MD5 algorithms work in 512-bit chunks, equal to 16 words.
    495 	 */
    496 
    497 	KASSERT(!(len & (512 - 1)) || bits != 0);
    498 	KASSERT(buf != NULL || m != NULL || uio != NULL);
    499 
    500 	cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_AC);
    501 	if (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION))
    502 		return ERESTART;
    503 
    504 	bswp = 0;
    505 	if (alg == MVCESA_SHA1MD5I_AC_ALGORITHM_SHA1) {
    506 		dlen = 160;
    507 		bits = htobe64(bits);
    508 #if BYTE_ORDER == LITTLE_ENDIAN
    509 		bswp = MVCESA_SHA1MD5I_AC_DATABYTESWAP |
    510 		    MVCESA_SHA1MD5I_AC_IVBYTESWAP;
    511 #endif
    512 	} else {	/* MVCESA_SHA1MD5I_AC_ALGORITHM_MD5 */
    513 		dlen = 128;
    514 		bits = htole64(bits);
    515 #if BYTE_ORDER == BIG_ENDIAN
    516 		bswp = MVCESA_SHA1MD5I_AC_DATABYTESWAP |
    517 		    MVCESA_SHA1MD5I_AC_IVBYTESWAP;
    518 #endif
    519 	}
    520 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_AC,
    521 	    alg | bswp | MVCESA_SHA1MD5I_AC_MODE_USEIV);
    522 
    523 	if (iv != NULL)
    524 		bus_space_write_region_4(sc->sc_iot, sc->sc_ioh,
    525 		    MVCESA_SHA1MD5I_IVDA, iv, dlen / 4);
    526 
    527 	off = i = 0;
    528 	while (1 /* CONSTCOND */) {
    529 		data = 0;
    530 		if (buf != NULL)
    531 			for (i = 0; i < 512 / 8 && off + i < len; i += s) {
    532 				s = uimin(sizeof(data), len - off - i);
    533 				memcpy(&data, buf + skip + off + i, s);
    534 				if (s == sizeof(data))
    535 					bus_space_write_4(sc->sc_iot,
    536 					    sc->sc_ioh, MVCESA_SHA1MD5I_DI,
    537 					    data);
    538 			}
    539 		else if (m != NULL)
    540 			for (i = 0; i < 512 / 8 && off + i < len; i += s) {
    541 				s = uimin(sizeof(data), len - off - i);
    542 				m_copydata(m, skip + off + i, s, &data);
    543 				if (s == sizeof(data))
    544 					bus_space_write_4(sc->sc_iot,
    545 					    sc->sc_ioh, MVCESA_SHA1MD5I_DI,
    546 					    data);
    547 			}
    548 		else if (uio != NULL)
    549 			for (i = 0; i < 512 / 8 && off + i < len; i += s) {
    550 				s = uimin(sizeof(data), len - off - i);
    551 				cuio_copydata(uio, skip + off + i, s, &data);
    552 				if (s == sizeof(data))
    553 					bus_space_write_4(sc->sc_iot,
    554 					    sc->sc_ioh, MVCESA_SHA1MD5I_DI,
    555 					    data);
    556 			}
    557 
    558 		off += i;
    559 		if (i < 512 / 8)
    560 			break;
    561 
    562 		do {
    563 			cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    564 			    MVCESA_SHA1MD5I_AC);
    565 		} while (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION));
    566 
    567 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_AC,
    568 		    alg | bswp | MVCESA_SHA1MD5I_AC_MODE_CONTINUE);
    569 	}
    570 
    571 	if (i < 512 / 8) {
    572 		*((char *)&data + (i % 4)) = 0x80;
    573 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_DI,
    574 		    data);
    575 		i = (i & ~3) + 4;
    576 
    577 		/* Do pad to 512 bits, if chunk size is more than 448 bits. */
    578 		if (i > 448 / 8) {
    579 			for (; i < 512 / 8; i += 4)
    580 				bus_space_write_4(sc->sc_iot, sc->sc_ioh,
    581 				    MVCESA_SHA1MD5I_DI, 0);
    582 			do {
    583 				cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    584 				    MVCESA_SHA1MD5I_AC);
    585 			} while (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION));
    586 			i = 0;
    587 		}
    588 		for (; i < 448 / 8; i += 4)
    589 			bus_space_write_4(sc->sc_iot, sc->sc_ioh,
    590 			    MVCESA_SHA1MD5I_DI, 0);
    591 
    592 		/* Set total bits */
    593 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_BCL,
    594 		    bits & 0xffffffff);
    595 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_BCH,
    596 		    bits >> 32);
    597 		do {
    598 			cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    599 			    MVCESA_SHA1MD5I_AC);
    600 		} while (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION));
    601 	}
    602 
    603 	if (digest != NULL) {
    604 		/* Read digest */
    605 		bus_space_read_region_4(sc->sc_iot, sc->sc_ioh,
    606 		    MVCESA_SHA1MD5I_IVDA, digest, dlen / 8 / 4);
    607 #if BYTE_ORDER == LITTLE_ENDIAN
    608 		if (alg == MVCESA_SHA1MD5I_AC_ALGORITHM_SHA1)
    609 			for (i = 0; i < dlen / 8 / 4; i++)
    610 				digest[i] = be32toh(digest[i]);
    611 #else
    612 		if (alg == MVCESA_SHA1MD5I_AC_ALGORITHM_MD5)
    613 			for (i = 0; i < dlen / 8 / 4; i++)
    614 				digest[i] = le32toh(digest[i]);
    615 #endif
    616 	}
    617 	return 0;
    618 }
    619 
    620 static int
    621 mvcesa_des_encdec(struct mvcesa_softc *sc, struct mvcesa_session *ses,
    622 		  uint32_t alg, uint32_t mode, uint32_t dir, uint32_t *iv,
    623 		  int skip, int len, char *buf, struct mbuf *m, struct uio *uio)
    624 {
    625 	uint64_t iblk, oblk;
    626 	uint32_t cmd, bswp = 0;
    627 	int i, o, s;
    628 
    629 	KASSERT(buf != NULL || m != NULL || uio != NULL);
    630 
    631 	cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_C);
    632 	if (!(cmd & MVCESA_DESE_C_TERMINATION))
    633 		return ERESTART;
    634 
    635 #if BYTE_ORDER == LITTLE_ENDIAN
    636 	bswp = MVCESA_DESE_C_DATABYTESWAP | MVCESA_DESE_C_IVBYTESWAP   |
    637 	    MVCESA_DESE_C_OUTBYTESWAP;
    638 #endif
    639 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_C,
    640 	    dir | alg | mode | bswp | MVCESA_DESE_C_ALLTERMINATION);
    641 
    642 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K0L,
    643 	    ses->ses_key[1]);
    644 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K0H,
    645 	    ses->ses_key[0]);
    646 	if (alg == MVCESA_DESE_C_ALGORITHM_3DES) {
    647 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K1L,
    648 		    ses->ses_key[3]);
    649 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K1H,
    650 		    ses->ses_key[2]);
    651 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K2L,
    652 		    ses->ses_key[5]);
    653 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K2H,
    654 		    ses->ses_key[4]);
    655 	}
    656 
    657 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_IVL, iv[1]);
    658 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_IVH, iv[0]);
    659 
    660 	i = o = 0;
    661 	while (i < len) {
    662 		s = uimin(sizeof(iblk), len - i);
    663 		iblk = 0;
    664 
    665 		if (buf != NULL)
    666 			memcpy(&iblk, buf + skip + i, s);
    667 		else if (m != NULL)
    668 			m_copydata(m, skip + i, s, &iblk);
    669 		else if (uio != NULL)
    670 			cuio_copydata(uio, skip + i, s, &iblk);
    671 
    672 		/*
    673 		 * We have the pipeline that two data enters.
    674 		 */
    675 
    676 		while (1 /* CONSTCOND */) {
    677 			cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    678 			    MVCESA_DESE_C);
    679 			if (cmd & MVCESA_DESE_C_ALLTERMINATION)
    680 				/* Engine is ready.  Can write two data. */
    681 				break;
    682 			if (cmd & MVCESA_DESE_C_READALLOW) {
    683 				oblk = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    684 				    MVCESA_DESE_DOH);
    685 				/* XXXX: needs barrier? */
    686 				oblk |= (uint64_t)bus_space_read_4(sc->sc_iot,
    687 				    sc->sc_ioh, MVCESA_DESE_DOL) << 32;
    688 
    689 				if (buf != NULL)
    690 					memcpy(buf + skip + o, &oblk,
    691 					    sizeof(oblk));
    692 				else if (m != NULL)
    693 					m_copydata(m, skip + o, sizeof(oblk),
    694 					    &oblk);
    695 				else if (uio != NULL)
    696 					cuio_copyback(uio, skip + o,
    697 					    sizeof(oblk), &oblk);
    698 				o += sizeof(oblk);
    699 
    700 				/* Can write one data */
    701 				break;
    702 			}
    703 		}
    704 
    705 		/*
    706 		 * Encryption/Decryption calculation time is 9 cycles in DES
    707 		 * mode and 25 cycles in 3DES mode.
    708 		 */
    709 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_DBL,
    710 		    iblk >> 32);
    711 		/* XXXX: needs barrier? */
    712 		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_DBH,
    713 		    iblk & 0xffffffff);
    714 		i += s;
    715 	}
    716 
    717 	while (1 /* CONSTCOND */) {
    718 		cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    719 		    MVCESA_DESE_C);
    720 		if (cmd & (MVCESA_DESE_C_READALLOW |
    721 					MVCESA_DESE_C_ALLTERMINATION)) {
    722 			oblk = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
    723 			    MVCESA_DESE_DOH);
    724 			/* XXXX: needs barrier? */
    725 			oblk |= (uint64_t)bus_space_read_4(sc->sc_iot,
    726 			    sc->sc_ioh, MVCESA_DESE_DOL) << 32;
    727 
    728 			if (cmd & MVCESA_DESE_C_ALLTERMINATION) {
    729 				/* We can read IV from Data Out Registers. */
    730 				if (dir == MVCESA_DESE_C_DIRECTION_ENC)
    731 					o -= sizeof(oblk);
    732 				else
    733 					break;
    734 			}
    735 			if (buf != NULL)
    736 				memcpy(buf + skip + o, &oblk, sizeof(oblk));
    737 			else if (m != NULL)
    738 				m_copydata(m, skip + o, sizeof(oblk), &oblk);
    739 			else if (uio != NULL)
    740 				cuio_copyback(uio, skip + o, sizeof(oblk),
    741 				    &oblk);
    742 			o += sizeof(oblk);
    743 			if (cmd & MVCESA_DESE_C_ALLTERMINATION)
    744 				break;
    745 		}
    746 	}
    747 
    748 	return 0;
    749 }
    750