xref: /src/sys/dev/marvell/mvcesa.c

/*	$NetBSD: mvcesa.c,v 1.6 2022/05/22 11:39:27 riastradh Exp $	*/
/*
 * Copyright (c) 2008 KIYOHARA Takashi
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: mvcesa.c,v 1.6 2022/05/22 11:39:27 riastradh Exp $");

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/cprng.h>
#include <sys/device.h>
#include <sys/endian.h>
#include <sys/errno.h>
#include <sys/mbuf.h>
#include <sys/md5.h>
#include <sys/uio.h>
#include <sys/sha1.h>

#include <opencrypto/cryptodev.h>
#include <opencrypto/xform.h>

#include <dev/marvell/marvellreg.h>
#include <dev/marvell/marvellvar.h>
#include <dev/marvell/mvcesareg.h>

#include "locators.h"

#define MVCESA_SESSION(sid)		((sid) & 0x0fffffff)
#define MVCESA_SID(crd, sesn)		(((crd) << 28) | ((sesn) & 0x0fffffff))


struct mvcesa_session {
	int ses_used;

	int ses_klen;
	uint32_t ses_key[8];

	uint32_t ses_hminner[5];	/* HMAC inner state */
	uint32_t ses_hmouter[5];	/* HMAC outer state */
};

struct mvcesa_softc {
	device_t sc_dev;

	bus_space_tag_t sc_iot;
	bus_space_handle_t sc_ioh;
	bus_dma_tag_t sc_dmat;

	int sc_cid;
	int sc_nsessions;
	struct mvcesa_session *sc_sessions;
};

static int mvcesa_match(device_t, cfdata_t, void *);
static void mvcesa_attach(device_t, device_t, void *);

static int mvcesa_intr(void *);

static int mvcesa_newsession(void *, u_int32_t *, struct cryptoini *);
static void mvcesa_freesession(void *, u_int64_t);
static int mvcesa_process(void *, struct cryptop *, int);

static int mvcesa_authentication(struct mvcesa_softc *, struct mvcesa_session *,
				 uint32_t, uint32_t *, uint32_t *, uint64_t,
				 int, int, char *, struct mbuf *, struct uio *);
static int mvcesa_des_encdec(struct mvcesa_softc *, struct mvcesa_session *,
			     uint32_t, uint32_t, uint32_t, uint32_t *, int, int,
			     char *, struct mbuf *, struct uio *);


CFATTACH_DECL_NEW(mvcesa_gt, sizeof(struct mvcesa_softc),
    mvcesa_match, mvcesa_attach, NULL, NULL);
CFATTACH_DECL_NEW(mvcesa_mbus, sizeof(struct mvcesa_softc),
    mvcesa_match, mvcesa_attach, NULL, NULL);


/* ARGSUSED */
static int
mvcesa_match(device_t parent, cfdata_t match, void *aux)
{
	struct marvell_attach_args *mva = aux;

	if (strcmp(mva->mva_name, match->cf_name) != 0)
		return 0;
	if (mva->mva_offset == MVA_OFFSET_DEFAULT ||
	    mva->mva_irq == MVA_IRQ_DEFAULT)
		return 0;

	mva->mva_size = MVCESA_SIZE;
	return 1;
}

/* ARGSUSED */
static void
mvcesa_attach(device_t parent, device_t self, void *aux)
{
	struct mvcesa_softc *sc = device_private(self);
	struct marvell_attach_args *mva = aux;

	aprint_normal(
	    ": Marvell Cryptographic Engines and Security Accelerator\n");
	aprint_naive("\n");

	sc->sc_dev = self;
	sc->sc_iot = mva->mva_iot;
        /* Map I/O registers */
	if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, mva->mva_offset,
	    mva->mva_size, &sc->sc_ioh)) {
		aprint_error_dev(self, "can't map registers\n");
		return;
	}
	sc->sc_dmat = mva->mva_dmat;

	sc->sc_nsessions = 0;

	/* Setup Opencrypto stuff */
	sc->sc_cid = crypto_get_driverid(0);
	if (sc->sc_cid < 0) {
		aprint_error_dev(self, "couldn't get crypto driver id\n");
		return;
	}
	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
#if __DMA_notyet__
/*
 * Don't know how to process to AES CBC in PIO-mode.
 * I havn't found IV registers.
 */
	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
#endif
	crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
	crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);
	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
	    mvcesa_newsession, mvcesa_freesession, mvcesa_process, sc);

	/* Clear and establish interrupt */
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_IC, 0);
	marvell_intr_establish(mva->mva_irq, IPL_NET, mvcesa_intr, sc);

	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_IM, 0);
}


static int
mvcesa_intr(void *arg)
{
#if 0
	struct mvcesa_softc *sc = (struct mvcesa_softc *)arg;
#endif
	int handled = 0;

	return handled;
}


/*
 * Opencrypto functions
 */
/*
 * Allocate a new 'session' and return an encoded session id.  'sidp'
 * contains our registration id, and should contain an encoded session
 * id on successful allocation.
 */
static int
mvcesa_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
{
	struct mvcesa_softc *sc = (struct mvcesa_softc *)arg;
	struct cryptoini *c;
	struct mvcesa_session *ses = NULL;
	int sesn, count, enc, mac, i;

	KASSERT(sc != NULL /*, ("mvcesa_newsession: null softc")*/);
	if (sidp == NULL || cri == NULL || sc == NULL)
		return EINVAL;

	for (sesn = 0; sesn < sc->sc_nsessions; sesn++)
		if (sc->sc_sessions[sesn].ses_used == 0) {
			ses = sc->sc_sessions + sesn;
			break;
		}

	if (ses == NULL) {
		sesn = sc->sc_nsessions;
		ses = malloc((sesn + 1) * sizeof(*ses), M_DEVBUF, M_NOWAIT);
		if (ses == NULL)
			return ENOMEM;
		if (sesn != 0) {
			memcpy(ses, sc->sc_sessions, sesn * sizeof(*ses));
			memset(sc->sc_sessions, 0, sesn * sizeof(*ses));
			free(sc->sc_sessions, M_DEVBUF);
		}
		sc->sc_sessions = ses;
		ses = sc->sc_sessions + sesn;
		sc->sc_nsessions++;
	}
	memset(ses, 0, sizeof(*ses));

	count = 0;
	enc = mac = 0;
	for (c = cri; c != NULL; c = c->cri_next) {
		switch (c->cri_alg) {
		case CRYPTO_DES_CBC:
		case CRYPTO_3DES_CBC:
			if (enc)
				return EINVAL;
			enc = 1;

			/* Go ahead and compute key in CESA's byte order */
			ses->ses_klen = c->cri_klen;
			memcpy(ses->ses_key, c->cri_key, c->cri_klen / 8);
			switch (c->cri_alg) {
			case CRYPTO_3DES_CBC:
				ses->ses_key[5] = htobe32(ses->ses_key[5]);
				ses->ses_key[4] = htobe32(ses->ses_key[4]);
				ses->ses_key[3] = htobe32(ses->ses_key[3]);
				ses->ses_key[2] = htobe32(ses->ses_key[2]);

				/* FALLTHROUGH */
			case CRYPTO_DES_CBC:
				ses->ses_key[1] = htobe32(ses->ses_key[1]);
				ses->ses_key[0] = htobe32(ses->ses_key[0]);
			}
			break;

		case CRYPTO_SHA1_HMAC:
		case CRYPTO_MD5_HMAC:
		{
			MD5_CTX md5ctx;
			SHA1_CTX sha1ctx;
			int klen_bytes = c->cri_klen / 8;

			KASSERT(c->cri_klen == 512);

			for (i = 0; i < klen_bytes; i++)
				c->cri_key[i] ^= HMAC_IPAD_VAL;
			if (c->cri_alg == CRYPTO_MD5_HMAC_96) {
				MD5Init(&md5ctx);
				MD5Update(&md5ctx, c->cri_key, klen_bytes);
				MD5Update(&md5ctx, hmac_ipad_buffer,
				    HMAC_BLOCK_LEN - klen_bytes);
				memcpy(ses->ses_hminner, md5ctx.state,
				    sizeof(md5ctx.state));
			} else {
				SHA1Init(&sha1ctx);
				SHA1Update(&sha1ctx, c->cri_key, klen_bytes);
				SHA1Update(&sha1ctx, hmac_ipad_buffer,
				    HMAC_BLOCK_LEN - klen_bytes);
				memcpy(ses->ses_hminner, sha1ctx.state,
				    sizeof(sha1ctx.state));
			}

			for (i = 0; i < klen_bytes; i++)
				c->cri_key[i] ^=
				    (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
			if (c->cri_alg == CRYPTO_MD5_HMAC_96) {
				MD5Init(&md5ctx);
				MD5Update(&md5ctx, c->cri_key, klen_bytes);
				MD5Update(&md5ctx, hmac_opad_buffer,
				    HMAC_BLOCK_LEN - klen_bytes);
				memcpy(ses->ses_hmouter, md5ctx.state,
				    sizeof(md5ctx.state));
			} else {
				SHA1Init(&sha1ctx);
				SHA1Update(&sha1ctx, c->cri_key, klen_bytes);
				SHA1Update(&sha1ctx, hmac_opad_buffer,
				    HMAC_BLOCK_LEN - klen_bytes);
				memcpy(ses->ses_hmouter, sha1ctx.state,
				    sizeof(sha1ctx.state));
			}

			for (i = 0; i < klen_bytes; i++)
				c->cri_key[i] ^= HMAC_OPAD_VAL;
		}
			/* FALLTHROUGH */

		case CRYPTO_SHA1:
		case CRYPTO_MD5:
			if (mac)
				return EINVAL;
			mac = 1;
		}
		count++;
	}
	if (count > 2) {
		mvcesa_freesession(sc, sesn);
		return EINVAL;
	}

	*sidp = MVCESA_SID(device_unit(sc->sc_dev), sesn);
	ses->ses_used = 1;

	return 0;
}

/*
 * Deallocate a session.
 */
static void
mvcesa_freesession(void *arg, u_int64_t tid)
{
	struct mvcesa_softc *sc = (struct mvcesa_softc *)arg;
	int session;
	uint32_t sid = ((uint32_t)tid) & 0xffffffff;

	session = MVCESA_SESSION(sid);
	KASSERTMSG(session >= 0, "session=%d", session);
	KASSERTMSG(session < sc->sc_nsessions, "session=%d nsessions=%d",
	    session, sc->sc_nsessions);

	memset(&sc->sc_sessions[session], 0, sizeof(sc->sc_sessions[session]));
}

static int
mvcesa_process(void *arg, struct cryptop *crp, int hint)
{
	struct mvcesa_softc *sc = arg;
	struct mvcesa_session *ses;
	struct cryptodesc *crd;
	struct mbuf *m = NULL;
	struct uio *uio = NULL;
	int session;
	char *buf = NULL;

	session = MVCESA_SESSION(crp->crp_sid);
	KASSERTMSG(session < sc->sc_nsessions, "session=%d nsessions=%d",
	    session, sc->sc_nsessions);
	ses = &sc->sc_sessions[session];

	if (crp->crp_flags & CRYPTO_F_IMBUF)
		m = (struct mbuf *)crp->crp_buf;
	else if (crp->crp_flags & CRYPTO_F_IOV)
		uio = (struct uio *)crp->crp_buf;
	else
		buf = (char *)crp->crp_buf;

	if (0 /* DMA support */) {
		/* not yet... */

		goto done;
	}

	/* PIO operation */

	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
		switch (crd->crd_alg) {
		case CRYPTO_DES_CBC:
		case CRYPTO_3DES_CBC:
		{
			uint32_t alg, mode, dir, *iv, ivbuf[2];

			mode = MVCESA_DESE_C_DESMODE_CBC;
			if (crd->crd_alg == CRYPTO_DES_CBC)
				alg = MVCESA_DESE_C_ALGORITHM_DES;
			else {	/* CRYPTO_3DES_CBC */
				alg = MVCESA_DESE_C_ALGORITHM_3DES;
				mode |= MVCESA_DESE_C_3DESMODE_EDE;
			}
			if (crd->crd_flags & CRD_F_ENCRYPT) {
				dir = MVCESA_DESE_C_DIRECTION_ENC;
				if (crd->crd_flags & CRD_F_IV_EXPLICIT)
					iv = (uint32_t *)crd->crd_iv;
				else {
					cprng_fast(ivbuf, sizeof(ivbuf));
					iv = ivbuf;
				}
				if (!(crd->crd_flags & CRD_F_IV_PRESENT)) {
					if (m != NULL)
						m_copyback(m, crd->crd_inject,
						    8, iv);
					else if (uio != NULL)
						cuio_copyback(uio,
						    crd->crd_inject, 8, iv);
				}
			} else {
				dir = MVCESA_DESE_C_DIRECTION_DEC;
				if (crd->crd_flags & CRD_F_IV_EXPLICIT)
					iv = (uint32_t *)crd->crd_iv;
				else {
					if (m != NULL)
						m_copydata(m, crd->crd_inject,
						    8, ivbuf);
					else if (uio != NULL)
						cuio_copydata(uio,
						    crd->crd_inject, 8, ivbuf);
					iv = ivbuf;
				}
			}

			crp->crp_etype = mvcesa_des_encdec(sc, ses,
			    alg, mode, dir, iv, crd->crd_skip, crd->crd_len,
			    buf, m, uio);
			break;
		}

		case CRYPTO_SHA1:
		case CRYPTO_SHA1_HMAC:
		case CRYPTO_MD5:
		case CRYPTO_MD5_HMAC:
		{
			uint64_t bits;
			uint32_t alg, *iv = NULL, digest[512 / 8 / 4], dlen;

			if (crd->crd_alg == CRYPTO_SHA1 ||
			    crd->crd_alg == CRYPTO_SHA1_HMAC) {
				alg = MVCESA_SHA1MD5I_AC_ALGORITHM_SHA1;
				dlen = 160;
			} else {	/* CRYPTO_MD5 || CRYPTO_MD5_HMAC */
				alg = MVCESA_SHA1MD5I_AC_ALGORITHM_MD5;
				dlen = 128;
			}
			bits = crd->crd_len << 3;
			if (crd->crd_alg == CRYPTO_SHA1_HMAC ||
			    crd->crd_alg == CRYPTO_MD5_HMAC) {
				iv = ses->ses_hminner;
				bits += 512;
			}

			crp->crp_etype = mvcesa_authentication(sc, ses,
			    alg, iv, digest, bits, crd->crd_skip, crd->crd_len,
			    buf, m, uio);
			if (crp->crp_etype != 0)
				break;

			if (crd->crd_alg == CRYPTO_SHA1_HMAC ||
			    crd->crd_alg == CRYPTO_MD5_HMAC)
				crp->crp_etype = mvcesa_authentication(sc,
				    ses, alg, ses->ses_hmouter, digest,
				    512 + dlen, 0, dlen, (char *)digest, NULL,
				    NULL);
			if (crp->crp_etype != 0)
				break;

			/* Inject the authentication data */
			if (buf != NULL)
				memcpy(buf + crd->crd_inject, digest, dlen / 8);
			else if (m != NULL)
				m_copyback(m, crd->crd_inject, dlen / 8,
				    digest);
			else if (uio != NULL)
				memcpy(crp->crp_mac, digest, dlen / 8);
		}
		}
		if (crp->crp_etype != 0)
			break;
	}

done:
	DPRINTF(("request %08x done\n", (uint32_t)crp));
	crypto_done(crp);
	return 0;
}


static int
mvcesa_authentication(struct mvcesa_softc *sc, struct mvcesa_session *ses,
		      uint32_t alg, uint32_t *iv, uint32_t *digest,
		      uint64_t bits, int skip, int len, char *buf,
		      struct mbuf *m, struct uio *uio)
{
	uint32_t cmd, bswp, data = 0;
	int dlen, off, i, s;

	/*
	 * SHA/MD5 algorithms work in 512-bit chunks, equal to 16 words.
	 */

	KASSERT(!(len & (512 - 1)) || bits != 0);
	KASSERT(buf != NULL || m != NULL || uio != NULL);

	cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_AC);
	if (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION))
		return ERESTART;

	bswp = 0;
	if (alg == MVCESA_SHA1MD5I_AC_ALGORITHM_SHA1) {
		dlen = 160;
		bits = htobe64(bits);
#if BYTE_ORDER == LITTLE_ENDIAN
		bswp = MVCESA_SHA1MD5I_AC_DATABYTESWAP |
		    MVCESA_SHA1MD5I_AC_IVBYTESWAP;
#endif
	} else {	/* MVCESA_SHA1MD5I_AC_ALGORITHM_MD5 */
		dlen = 128;
		bits = htole64(bits);
#if BYTE_ORDER == BIG_ENDIAN
		bswp = MVCESA_SHA1MD5I_AC_DATABYTESWAP |
		    MVCESA_SHA1MD5I_AC_IVBYTESWAP;
#endif
	}
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_AC,
	    alg | bswp | MVCESA_SHA1MD5I_AC_MODE_USEIV);

	if (iv != NULL)
		bus_space_write_region_4(sc->sc_iot, sc->sc_ioh,
		    MVCESA_SHA1MD5I_IVDA, iv, dlen / 4);

	off = i = 0;
	while (1 /* CONSTCOND */) {
		data = 0;
		if (buf != NULL)
			for (i = 0; i < 512 / 8 && off + i < len; i += s) {
				s = uimin(sizeof(data), len - off - i);
				memcpy(&data, buf + skip + off + i, s);
				if (s == sizeof(data))
					bus_space_write_4(sc->sc_iot,
					    sc->sc_ioh, MVCESA_SHA1MD5I_DI,
					    data);
			}
		else if (m != NULL)
			for (i = 0; i < 512 / 8 && off + i < len; i += s) {
				s = uimin(sizeof(data), len - off - i);
				m_copydata(m, skip + off + i, s, &data);
				if (s == sizeof(data))
					bus_space_write_4(sc->sc_iot,
					    sc->sc_ioh, MVCESA_SHA1MD5I_DI,
					    data);
			}
		else if (uio != NULL)
			for (i = 0; i < 512 / 8 && off + i < len; i += s) {
				s = uimin(sizeof(data), len - off - i);
				cuio_copydata(uio, skip + off + i, s, &data);
				if (s == sizeof(data))
					bus_space_write_4(sc->sc_iot,
					    sc->sc_ioh, MVCESA_SHA1MD5I_DI,
					    data);
			}

		off += i;
		if (i < 512 / 8)
			break;

		do {
			cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
			    MVCESA_SHA1MD5I_AC);
		} while (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION));

		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_AC,
		    alg | bswp | MVCESA_SHA1MD5I_AC_MODE_CONTINUE);
	}

	if (i < 512 / 8) {
		*((char *)&data + (i % 4)) = 0x80;
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_DI,
		    data);
		i = (i & ~3) + 4;

		/* Do pad to 512 bits, if chunk size is more than 448 bits. */
		if (i > 448 / 8) {
			for (; i < 512 / 8; i += 4)
				bus_space_write_4(sc->sc_iot, sc->sc_ioh,
				    MVCESA_SHA1MD5I_DI, 0);
			do {
				cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
				    MVCESA_SHA1MD5I_AC);
			} while (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION));
			i = 0;
		}
		for (; i < 448 / 8; i += 4)
			bus_space_write_4(sc->sc_iot, sc->sc_ioh,
			    MVCESA_SHA1MD5I_DI, 0);

		/* Set total bits */
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_BCL,
		    bits & 0xffffffff);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_SHA1MD5I_BCH,
		    bits >> 32);
		do {
			cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
			    MVCESA_SHA1MD5I_AC);
		} while (!(cmd & MVCESA_SHA1MD5I_AC_TERMINATION));
	}

	if (digest != NULL) {
		/* Read digest */
		bus_space_read_region_4(sc->sc_iot, sc->sc_ioh,
		    MVCESA_SHA1MD5I_IVDA, digest, dlen / 8 / 4);
#if BYTE_ORDER == LITTLE_ENDIAN
		if (alg == MVCESA_SHA1MD5I_AC_ALGORITHM_SHA1)
			for (i = 0; i < dlen / 8 / 4; i++)
				digest[i] = be32toh(digest[i]);
#else
		if (alg == MVCESA_SHA1MD5I_AC_ALGORITHM_MD5)
			for (i = 0; i < dlen / 8 / 4; i++)
				digest[i] = le32toh(digest[i]);
#endif
	}
	return 0;
}

static int
mvcesa_des_encdec(struct mvcesa_softc *sc, struct mvcesa_session *ses,
		  uint32_t alg, uint32_t mode, uint32_t dir, uint32_t *iv,
		  int skip, int len, char *buf, struct mbuf *m, struct uio *uio)
{
	uint64_t iblk, oblk;
	uint32_t cmd, bswp = 0;
	int i, o, s;

	KASSERT(buf != NULL || m != NULL || uio != NULL);

	cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_C);
	if (!(cmd & MVCESA_DESE_C_TERMINATION))
		return ERESTART;

#if BYTE_ORDER == LITTLE_ENDIAN
	bswp = MVCESA_DESE_C_DATABYTESWAP | MVCESA_DESE_C_IVBYTESWAP   |
	    MVCESA_DESE_C_OUTBYTESWAP;
#endif
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_C,
	    dir | alg | mode | bswp | MVCESA_DESE_C_ALLTERMINATION);

	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K0L,
	    ses->ses_key[1]);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K0H,
	    ses->ses_key[0]);
	if (alg == MVCESA_DESE_C_ALGORITHM_3DES) {
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K1L,
		    ses->ses_key[3]);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K1H,
		    ses->ses_key[2]);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K2L,
		    ses->ses_key[5]);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_K2H,
		    ses->ses_key[4]);
	}

	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_IVL, iv[1]);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_IVH, iv[0]);

	i = o = 0;
	while (i < len) {
		s = uimin(sizeof(iblk), len - i);
		iblk = 0;

		if (buf != NULL)
			memcpy(&iblk, buf + skip + i, s);
		else if (m != NULL)
			m_copydata(m, skip + i, s, &iblk);
		else if (uio != NULL)
			cuio_copydata(uio, skip + i, s, &iblk);

		/*
		 * We have the pipeline that two data enters.
		 */

		while (1 /* CONSTCOND */) {
			cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
			    MVCESA_DESE_C);
			if (cmd & MVCESA_DESE_C_ALLTERMINATION)
				/* Engine is ready.  Can write two data. */
				break;
			if (cmd & MVCESA_DESE_C_READALLOW) {
				oblk = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
				    MVCESA_DESE_DOH);
				/* XXXX: needs barrier? */
				oblk |= (uint64_t)bus_space_read_4(sc->sc_iot,
				    sc->sc_ioh, MVCESA_DESE_DOL) << 32;

				if (buf != NULL)
					memcpy(buf + skip + o, &oblk,
					    sizeof(oblk));
				else if (m != NULL)
					m_copydata(m, skip + o, sizeof(oblk),
					    &oblk);
				else if (uio != NULL)
					cuio_copyback(uio, skip + o,
					    sizeof(oblk), &oblk);
				o += sizeof(oblk);

				/* Can write one data */
				break;
			}
		}

		/*
		 * Encryption/Decryption calculation time is 9 cycles in DES
		 * mode and 25 cycles in 3DES mode.
		 */
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_DBL,
		    iblk >> 32);
		/* XXXX: needs barrier? */
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVCESA_DESE_DBH,
		    iblk & 0xffffffff);
		i += s;
	}

	while (1 /* CONSTCOND */) {
		cmd = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
		    MVCESA_DESE_C);
		if (cmd & (MVCESA_DESE_C_READALLOW |
					MVCESA_DESE_C_ALLTERMINATION)) {
			oblk = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
			    MVCESA_DESE_DOH);
			/* XXXX: needs barrier? */
			oblk |= (uint64_t)bus_space_read_4(sc->sc_iot,
			    sc->sc_ioh, MVCESA_DESE_DOL) << 32;

			if (cmd & MVCESA_DESE_C_ALLTERMINATION) {
				/* We can read IV from Data Out Registers. */
				if (dir == MVCESA_DESE_C_DIRECTION_ENC)
					o -= sizeof(oblk);
				else
					break;
			}
			if (buf != NULL)
				memcpy(buf + skip + o, &oblk, sizeof(oblk));
			else if (m != NULL)
				m_copydata(m, skip + o, sizeof(oblk), &oblk);
			else if (uio != NULL)
				cuio_copyback(uio, skip + o, sizeof(oblk),
				    &oblk);
			o += sizeof(oblk);
			if (cmd & MVCESA_DESE_C_ALLTERMINATION)
				break;
		}
	}

	return 0;
}