Home | History | Annotate | Line # | Download | only in opencrypto
cryptodev.c revision 1.53
      1 /*	$NetBSD: cryptodev.c,v 1.53 2010/08/02 19:59:35 jakllsch Exp $ */
      2 /*	$FreeBSD: src/sys/opencrypto/cryptodev.c,v 1.4.2.4 2003/06/03 00:09:02 sam Exp $	*/
      3 /*	$OpenBSD: cryptodev.c,v 1.53 2002/07/10 22:21:30 mickey Exp $	*/
      4 
      5 /*-
      6  * Copyright (c) 2008 The NetBSD Foundation, Inc.
      7  * All rights reserved.
      8  *
      9  * This code is derived from software contributed to The NetBSD Foundation
     10  * by Coyote Point Systems, Inc.
     11  *
     12  * Redistribution and use in source and binary forms, with or without
     13  * modification, are permitted provided that the following conditions
     14  * are met:
     15  * 1. Redistributions of source code must retain the above copyright
     16  *    notice, this list of conditions and the following disclaimer.
     17  * 2. Redistributions in binary form must reproduce the above copyright
     18  *    notice, this list of conditions and the following disclaimer in the
     19  *    documentation and/or other materials provided with the distribution.
     20  *
     21  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     23  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     24  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     31  * POSSIBILITY OF SUCH DAMAGE.
     32  */
     33 
     34 /*
     35  * Copyright (c) 2001 Theo de Raadt
     36  *
     37  * Redistribution and use in source and binary forms, with or without
     38  * modification, are permitted provided that the following conditions
     39  * are met:
     40  *
     41  * 1. Redistributions of source code must retain the above copyright
     42  *   notice, this list of conditions and the following disclaimer.
     43  * 2. Redistributions in binary form must reproduce the above copyright
     44  *   notice, this list of conditions and the following disclaimer in the
     45  *   documentation and/or other materials provided with the distribution.
     46  * 3. The name of the author may not be used to endorse or promote products
     47  *   derived from this software without specific prior written permission.
     48  *
     49  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     50  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     51  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     52  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     53  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     54  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     55  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     56  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     57  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     58  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     59  *
     60  * Effort sponsored in part by the Defense Advanced Research Projects
     61  * Agency (DARPA) and Air Force Research Laboratory, Air Force
     62  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
     63  *
     64  */
     65 
     66 #include <sys/cdefs.h>
     67 __KERNEL_RCSID(0, "$NetBSD: cryptodev.c,v 1.53 2010/08/02 19:59:35 jakllsch Exp $");
     68 
     69 #include <sys/param.h>
     70 #include <sys/systm.h>
     71 #include <sys/kmem.h>
     72 #include <sys/malloc.h>
     73 #include <sys/mbuf.h>
     74 #include <sys/pool.h>
     75 #include <sys/sysctl.h>
     76 #include <sys/file.h>
     77 #include <sys/filedesc.h>
     78 #include <sys/errno.h>
     79 #include <sys/md5.h>
     80 #include <sys/sha1.h>
     81 #include <sys/conf.h>
     82 #include <sys/device.h>
     83 #include <sys/kauth.h>
     84 #include <sys/select.h>
     85 #include <sys/poll.h>
     86 #include <sys/atomic.h>
     87 #include <sys/stat.h>
     88 
     89 #include "opt_ocf.h"
     90 #include <opencrypto/cryptodev.h>
     91 #include <opencrypto/ocryptodev.h>
     92 #include <opencrypto/xform.h>
     93 
     94 struct csession {
     95 	TAILQ_ENTRY(csession) next;
     96 	u_int64_t	sid;
     97 	u_int32_t	ses;
     98 
     99 	u_int32_t	cipher;		/* note: shares name space in crd_alg */
    100 	struct enc_xform *txform;
    101 	u_int32_t	mac;		/* note: shares name space in crd_alg */
    102 	struct auth_hash *thash;
    103 	u_int32_t	comp_alg;	/* note: shares name space in crd_alg */
    104 	struct comp_algo *tcomp;
    105 
    106 	void *		key;
    107 	int		keylen;
    108 	u_char		tmp_iv[EALG_MAX_BLOCK_LEN];
    109 
    110 	void *		mackey;
    111 	int		mackeylen;
    112 	u_char		tmp_mac[CRYPTO_MAX_MAC_LEN];
    113 
    114 	struct iovec	iovec[1];	/* user requests never have more */
    115 	struct uio	uio;
    116 	int		error;
    117 };
    118 
    119 struct fcrypt {
    120 	TAILQ_HEAD(csessionlist, csession) csessions;
    121 	TAILQ_HEAD(crprethead, cryptop) crp_ret_mq;
    122 	TAILQ_HEAD(krprethead, cryptkop) crp_ret_mkq;
    123 	int		sesn;
    124 	struct selinfo	sinfo;
    125 	u_int32_t	requestid;
    126 	struct timespec atime;
    127 	struct timespec mtime;
    128 	struct timespec btime;
    129 };
    130 
    131 /* For our fixed-size allocations */
    132 static struct pool fcrpl;
    133 static struct pool csepl;
    134 
    135 /* Declaration of master device (fd-cloning/ctxt-allocating) entrypoints */
    136 static int	cryptoopen(dev_t dev, int flag, int mode, struct lwp *l);
    137 static int	cryptoread(dev_t dev, struct uio *uio, int ioflag);
    138 static int	cryptowrite(dev_t dev, struct uio *uio, int ioflag);
    139 static int	cryptoselect(dev_t dev, int rw, struct lwp *l);
    140 
    141 /* Declaration of cloned-device (per-ctxt) entrypoints */
    142 static int	cryptof_read(struct file *, off_t *, struct uio *,
    143     kauth_cred_t, int);
    144 static int	cryptof_write(struct file *, off_t *, struct uio *,
    145     kauth_cred_t, int);
    146 static int	cryptof_ioctl(struct file *, u_long, void *);
    147 static int	cryptof_close(struct file *);
    148 static int 	cryptof_poll(struct file *, int);
    149 static int 	cryptof_stat(struct file *, struct stat *);
    150 
    151 static const struct fileops cryptofops = {
    152 	.fo_read = cryptof_read,
    153 	.fo_write = cryptof_write,
    154 	.fo_ioctl = cryptof_ioctl,
    155 	.fo_fcntl = fnullop_fcntl,
    156 	.fo_poll = cryptof_poll,
    157 	.fo_stat = cryptof_stat,
    158 	.fo_close = cryptof_close,
    159 	.fo_kqfilter = fnullop_kqfilter,
    160 	.fo_restart = fnullop_restart,
    161 };
    162 
    163 struct csession *cryptodev_csefind(struct fcrypt *, u_int);
    164 static struct	csession *csefind(struct fcrypt *, u_int);
    165 static int	csedelete(struct fcrypt *, struct csession *);
    166 static struct	csession *cseadd(struct fcrypt *, struct csession *);
    167 static struct	csession *csecreate(struct fcrypt *, u_int64_t, void *,
    168     u_int64_t, void *, u_int64_t, u_int32_t, u_int32_t, u_int32_t,
    169     struct enc_xform *, struct auth_hash *, struct comp_algo *);
    170 static int	csefree(struct csession *);
    171 
    172 static int	cryptodev_key(struct crypt_kop *);
    173 static int	cryptodev_mkey(struct fcrypt *, struct crypt_n_kop *, int);
    174 static int	cryptodev_msessionfin(struct fcrypt *, int, u_int32_t *);
    175 
    176 static int	cryptodev_cb(void *);
    177 static int	cryptodevkey_cb(void *);
    178 
    179 static int	cryptodev_mcb(void *);
    180 static int	cryptodevkey_mcb(void *);
    181 
    182 static int 	cryptodev_getmstatus(struct fcrypt *, struct crypt_result *,
    183     int);
    184 static int	cryptodev_getstatus(struct fcrypt *, struct crypt_result *);
    185 
    186 extern int	ocryptof_ioctl(struct file *, u_long, void *);
    187 
    188 /*
    189  * sysctl-able control variables for /dev/crypto now defined in crypto.c:
    190  * crypto_usercrypto, crypto_userasmcrypto, crypto_devallowsoft.
    191  */
    192 
    193 /* ARGSUSED */
    194 int
    195 cryptof_read(file_t *fp, off_t *poff,
    196     struct uio *uio, kauth_cred_t cred, int flags)
    197 {
    198 	return EIO;
    199 }
    200 
    201 /* ARGSUSED */
    202 int
    203 cryptof_write(file_t *fp, off_t *poff,
    204     struct uio *uio, kauth_cred_t cred, int flags)
    205 {
    206 	return EIO;
    207 }
    208 
    209 /* ARGSUSED */
    210 int
    211 cryptof_ioctl(struct file *fp, u_long cmd, void *data)
    212 {
    213 	struct fcrypt *fcr = fp->f_data;
    214 	struct csession *cse;
    215 	struct session_op *sop;
    216 	struct session_n_op *snop;
    217 	struct crypt_op *cop;
    218 	struct crypt_mop *mop;
    219 	struct crypt_mkop *mkop;
    220 	struct crypt_n_op *cnop;
    221 	struct crypt_n_kop *knop;
    222 	struct crypt_sgop *sgop;
    223 	struct crypt_sfop *sfop;
    224 	struct cryptret *crypt_ret;
    225 	struct crypt_result *crypt_res;
    226 	u_int32_t ses;
    227 	u_int32_t *sesid;
    228 	int error = 0;
    229 	size_t count;
    230 
    231 	/* backwards compatibility */
    232         file_t *criofp;
    233 	struct fcrypt *criofcr;
    234 	int criofd;
    235 
    236 	mutex_spin_enter(&crypto_mtx);
    237 	getnanotime(&fcr->atime);
    238 	mutex_spin_exit(&crypto_mtx);
    239 
    240 	switch (cmd) {
    241         case CRIOGET:   /* XXX deprecated, remove after 5.0 */
    242 		if ((error = fd_allocfile(&criofp, &criofd)) != 0)
    243 			return error;
    244 		criofcr = pool_get(&fcrpl, PR_WAITOK);
    245 		mutex_spin_enter(&crypto_mtx);
    246 		TAILQ_INIT(&criofcr->csessions);
    247 		TAILQ_INIT(&criofcr->crp_ret_mq);
    248 		TAILQ_INIT(&criofcr->crp_ret_mkq);
    249 		selinit(&criofcr->sinfo);
    250 
    251                 /*
    252                  * Don't ever return session 0, to allow detection of
    253                  * failed creation attempts with multi-create ioctl.
    254                  */
    255 		criofcr->sesn = 1;
    256 		criofcr->requestid = 1;
    257 		mutex_spin_exit(&crypto_mtx);
    258 		(void)fd_clone(criofp, criofd, (FREAD|FWRITE),
    259 			      &cryptofops, criofcr);
    260 		*(u_int32_t *)data = criofd;
    261 		return error;
    262 		break;
    263 	case CIOCGSESSION:
    264 		sop = (struct session_op *)data;
    265 		error = cryptodev_session(fcr, sop);
    266 		break;
    267 	case CIOCNGSESSION:
    268 		sgop = (struct crypt_sgop *)data;
    269 		snop = kmem_alloc((sgop->count *
    270 				  sizeof(struct session_n_op)), KM_SLEEP);
    271 		error = copyin(sgop->sessions, snop, sgop->count *
    272 			       sizeof(struct session_n_op));
    273 		if (error) {
    274 			goto mbail;
    275 		}
    276 
    277 		mutex_spin_enter(&crypto_mtx);
    278 		fcr->mtime = fcr->atime;
    279 		mutex_spin_exit(&crypto_mtx);
    280 		error = cryptodev_msession(fcr, snop, sgop->count);
    281 		if (error) {
    282 			goto mbail;
    283 		}
    284 
    285 		error = copyout(snop, sgop->sessions, sgop->count *
    286 		    sizeof(struct session_n_op));
    287 mbail:
    288 		kmem_free(snop, sgop->count * sizeof(struct session_n_op));
    289 		break;
    290 	case CIOCFSESSION:
    291 		mutex_spin_enter(&crypto_mtx);
    292 		fcr->mtime = fcr->atime;
    293 		ses = *(u_int32_t *)data;
    294 		cse = csefind(fcr, ses);
    295 		if (cse == NULL)
    296 			return EINVAL;
    297 		csedelete(fcr, cse);
    298 		error = csefree(cse);
    299 		mutex_spin_exit(&crypto_mtx);
    300 		break;
    301 	case CIOCNFSESSION:
    302 		mutex_spin_enter(&crypto_mtx);
    303 		fcr->mtime = fcr->atime;
    304 		mutex_spin_exit(&crypto_mtx);
    305 		sfop = (struct crypt_sfop *)data;
    306 		sesid = kmem_alloc((sfop->count * sizeof(u_int32_t)),
    307 		    KM_SLEEP);
    308 		error = copyin(sfop->sesid, sesid,
    309 		    (sfop->count * sizeof(u_int32_t)));
    310 		if (!error) {
    311 			error = cryptodev_msessionfin(fcr, sfop->count, sesid);
    312 		}
    313 		kmem_free(sesid, (sfop->count * sizeof(u_int32_t)));
    314 		break;
    315 	case CIOCCRYPT:
    316 		mutex_spin_enter(&crypto_mtx);
    317 		fcr->mtime = fcr->atime;
    318 		cop = (struct crypt_op *)data;
    319 		cse = csefind(fcr, cop->ses);
    320 		mutex_spin_exit(&crypto_mtx);
    321 		if (cse == NULL) {
    322 			DPRINTF(("csefind failed\n"));
    323 			return EINVAL;
    324 		}
    325 		error = cryptodev_op(cse, cop, curlwp);
    326 		DPRINTF(("cryptodev_op error = %d\n", error));
    327 		break;
    328 	case CIOCNCRYPTM:
    329 		mutex_spin_enter(&crypto_mtx);
    330 		fcr->mtime = fcr->atime;
    331 		mutex_spin_exit(&crypto_mtx);
    332 		mop = (struct crypt_mop *)data;
    333 		cnop = kmem_alloc((mop->count * sizeof(struct crypt_n_op)),
    334 		    KM_SLEEP);
    335 		error = copyin(mop->reqs, cnop,
    336 		    (mop->count * sizeof(struct crypt_n_op)));
    337 		if(!error) {
    338 			error = cryptodev_mop(fcr, cnop, mop->count, curlwp);
    339 			if (!error) {
    340 				error = copyout(cnop, mop->reqs,
    341 				    (mop->count * sizeof(struct crypt_n_op)));
    342 			}
    343 		}
    344 		kmem_free(cnop, (mop->count * sizeof(struct crypt_n_op)));
    345 		break;
    346 	case CIOCKEY:
    347 		error = cryptodev_key((struct crypt_kop *)data);
    348 		DPRINTF(("cryptodev_key error = %d\n", error));
    349 		break;
    350 	case CIOCNFKEYM:
    351 		mutex_spin_enter(&crypto_mtx);
    352 		fcr->mtime = fcr->atime;
    353 		mutex_spin_exit(&crypto_mtx);
    354 		mkop = (struct crypt_mkop *)data;
    355 		knop = kmem_alloc((mkop->count * sizeof(struct crypt_n_kop)),
    356 		    KM_SLEEP);
    357 		error = copyin(mkop->reqs, knop,
    358 		    (mkop->count * sizeof(struct crypt_n_kop)));
    359 		if (!error) {
    360 			error = cryptodev_mkey(fcr, knop, mkop->count);
    361 			if (!error)
    362 				error = copyout(knop, mkop->reqs,
    363 				    (mkop->count * sizeof(struct crypt_n_kop)));
    364 		}
    365 		kmem_free(knop, (mkop->count * sizeof(struct crypt_n_kop)));
    366 		break;
    367 	case CIOCASYMFEAT:
    368 		error = crypto_getfeat((int *)data);
    369 		break;
    370 	case CIOCNCRYPTRETM:
    371 		mutex_spin_enter(&crypto_mtx);
    372 		fcr->mtime = fcr->atime;
    373 		mutex_spin_exit(&crypto_mtx);
    374 		crypt_ret = (struct cryptret *)data;
    375 		count = crypt_ret->count;
    376 		crypt_res = kmem_alloc((count * sizeof(struct crypt_result)),
    377 		    KM_SLEEP);
    378 		error = copyin(crypt_ret->results, crypt_res,
    379 		    (count * sizeof(struct crypt_result)));
    380 		if (error)
    381 			goto reterr;
    382 		crypt_ret->count = cryptodev_getmstatus(fcr, crypt_res,
    383 		    crypt_ret->count);
    384 		/* sanity check count */
    385 		if (crypt_ret->count > count) {
    386 			printf("%s.%d: error returned count %zd > original "
    387 			    " count %zd\n",
    388 			    __FILE__, __LINE__, crypt_ret->count, count);
    389 			crypt_ret->count = count;
    390 
    391 		}
    392 		error = copyout(crypt_res, crypt_ret->results,
    393 		    (crypt_ret->count * sizeof(struct crypt_result)));
    394 reterr:
    395 		kmem_free(crypt_res, (count * sizeof(struct crypt_result)));
    396 		break;
    397 	case CIOCNCRYPTRET:
    398 		error = cryptodev_getstatus(fcr, (struct crypt_result *)data);
    399 		break;
    400 	default:
    401 		/* Check for backward compatible commands */
    402 		error = ocryptof_ioctl(fp, cmd, data);
    403 	}
    404 	return error;
    405 }
    406 
    407 int
    408 cryptodev_op(struct csession *cse, struct crypt_op *cop, struct lwp *l)
    409 {
    410 	struct cryptop *crp = NULL;
    411 	struct cryptodesc *crde = NULL, *crda = NULL, *crdc = NULL;
    412 	int error;
    413 	int iov_len = cop->len;
    414 	int flags=0;
    415 	int dst_len;	/* copyout size */
    416 
    417 	if (cop->len > 256*1024-4)
    418 		return E2BIG;
    419 
    420 	if (cse->txform) {
    421 		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0)
    422 			return EINVAL;
    423 	}
    424 
    425 	DPRINTF(("cryptodev_op[%u]: iov_len %d\n",
    426 		CRYPTO_SESID2LID(cse->sid), iov_len));
    427 	if ((cse->tcomp) && cop->dst_len) {
    428 		if (iov_len < cop->dst_len) {
    429 			/* Need larger iov to deal with decompress */
    430 			iov_len = cop->dst_len;
    431 		}
    432 		DPRINTF(("cryptodev_op: iov_len -> %d for decompress\n", iov_len));
    433 	}
    434 
    435 	(void)memset(&cse->uio, 0, sizeof(cse->uio));
    436 	cse->uio.uio_iovcnt = 1;
    437 	cse->uio.uio_resid = 0;
    438 	cse->uio.uio_rw = UIO_WRITE;
    439 	cse->uio.uio_iov = cse->iovec;
    440 	UIO_SETUP_SYSSPACE(&cse->uio);
    441 	memset(&cse->iovec, 0, sizeof(cse->iovec));
    442 
    443 	/* the iov needs to be big enough to handle the uncompressed
    444 	 * data.... */
    445 	cse->uio.uio_iov[0].iov_len = iov_len;
    446 	cse->uio.uio_iov[0].iov_base = kmem_alloc(iov_len, KM_SLEEP);
    447 	cse->uio.uio_resid = cse->uio.uio_iov[0].iov_len;
    448 	DPRINTF(("cryptodev_op[%u]: uio.iov_base %p malloced %d bytes\n",
    449 		CRYPTO_SESID2LID(cse->sid),
    450 		cse->uio.uio_iov[0].iov_base, iov_len));
    451 
    452 	crp = crypto_getreq((cse->tcomp != NULL) + (cse->txform != NULL) + (cse->thash != NULL));
    453 	if (crp == NULL) {
    454 		error = ENOMEM;
    455 		goto bail;
    456 	}
    457 	DPRINTF(("cryptodev_op[%u]: crp %p\n",
    458 		CRYPTO_SESID2LID(cse->sid), crp));
    459 
    460 	/* crds are always ordered tcomp, thash, then txform */
    461 	/* with optional missing links */
    462 
    463 	/* XXX: If we're going to compress then hash or encrypt, we need
    464 	 * to be able to pass on the new size of the data.
    465 	 */
    466 
    467 	if (cse->tcomp) {
    468 		crdc = crp->crp_desc;
    469 	}
    470 
    471 	if (cse->thash) {
    472 		crda = crdc ? crdc->crd_next : crp->crp_desc;
    473 		if (cse->txform && crda)
    474 			crde = crda->crd_next;
    475 	} else {
    476 		if (cse->txform) {
    477 			crde = crdc ? crdc->crd_next : crp->crp_desc;
    478 		} else if (!cse->tcomp) {
    479 			error = EINVAL;
    480 			goto bail;
    481 		}
    482 	}
    483 
    484 	DPRINTF(("ocf[%u]: iov_len %zu, cop->len %u\n",
    485 			CRYPTO_SESID2LID(cse->sid),
    486 			cse->uio.uio_iov[0].iov_len,
    487 			cop->len));
    488 
    489 	if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, cop->len)))
    490 	{
    491 		printf("copyin failed %s %d \n", (char *)cop->src, error);
    492 		goto bail;
    493 	}
    494 
    495 	if (crdc) {
    496 		switch (cop->op) {
    497 		case COP_COMP:
    498 			crdc->crd_flags |= CRD_F_COMP;
    499 			break;
    500 		case COP_DECOMP:
    501 			crdc->crd_flags &= ~CRD_F_COMP;
    502 			break;
    503 		default:
    504 			break;
    505 		}
    506 		/* more data to follow? */
    507 		if (cop->flags & COP_F_MORE) {
    508 			flags |= CRYPTO_F_MORE;
    509 		}
    510 		crdc->crd_len = cop->len;
    511 		crdc->crd_inject = 0;
    512 
    513 		crdc->crd_alg = cse->comp_alg;
    514 		crdc->crd_key = NULL;
    515 		crdc->crd_klen = 0;
    516 		DPRINTF(("cryptodev_op[%u]: crdc setup for comp_alg %d.\n",
    517 			CRYPTO_SESID2LID(cse->sid), crdc->crd_alg));
    518 	}
    519 
    520 	if (crda) {
    521 		crda->crd_skip = 0;
    522 		crda->crd_len = cop->len;
    523 		crda->crd_inject = 0;	/* ??? */
    524 
    525 		crda->crd_alg = cse->mac;
    526 		crda->crd_key = cse->mackey;
    527 		crda->crd_klen = cse->mackeylen * 8;
    528 		DPRINTF(("cryptodev_op: crda setup for mac %d.\n", crda->crd_alg));
    529 	}
    530 
    531 	if (crde) {
    532 		switch (cop->op) {
    533 		case COP_ENCRYPT:
    534 			crde->crd_flags |= CRD_F_ENCRYPT;
    535 			break;
    536 		case COP_DECRYPT:
    537 			crde->crd_flags &= ~CRD_F_ENCRYPT;
    538 			break;
    539 		default:
    540 			break;
    541 		}
    542 		crde->crd_len = cop->len;
    543 		crde->crd_inject = 0;
    544 
    545 		crde->crd_alg = cse->cipher;
    546 		crde->crd_key = cse->key;
    547 		crde->crd_klen = cse->keylen * 8;
    548 		DPRINTF(("cryptodev_op: crde setup for cipher %d.\n", crde->crd_alg));
    549 	}
    550 
    551 
    552 	crp->crp_ilen = cop->len;
    553 	/* The reqest is flagged as CRYPTO_F_USER as long as it is running
    554 	 * in the user IOCTL thread.  This flag lets us skip using the retq for
    555 	 * the request if it completes immediately. If the request ends up being
    556 	 * delayed or is not completed immediately the flag is removed.
    557 	 */
    558 	crp->crp_flags = CRYPTO_F_IOV | (cop->flags & COP_F_BATCH) | CRYPTO_F_USER |
    559 			flags;
    560 	crp->crp_buf = (void *)&cse->uio;
    561 	crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb;
    562 	crp->crp_sid = cse->sid;
    563 	crp->crp_opaque = (void *)cse;
    564 
    565 	if (cop->iv) {
    566 		if (crde == NULL) {
    567 			error = EINVAL;
    568 			goto bail;
    569 		}
    570 		if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
    571 			error = EINVAL;
    572 			goto bail;
    573 		}
    574 		if ((error = copyin(cop->iv, cse->tmp_iv,
    575 		    cse->txform->blocksize)))
    576 			goto bail;
    577 		(void)memcpy(crde->crd_iv, cse->tmp_iv, cse->txform->blocksize);
    578 		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
    579 		crde->crd_skip = 0;
    580 	} else if (crde) {
    581 		if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
    582 			crde->crd_skip = 0;
    583 		} else {
    584 			crde->crd_flags |= CRD_F_IV_PRESENT;
    585 			crde->crd_skip = cse->txform->blocksize;
    586 			crde->crd_len -= cse->txform->blocksize;
    587 		}
    588 	}
    589 
    590 	if (cop->mac) {
    591 		if (crda == NULL) {
    592 			error = EINVAL;
    593 			goto bail;
    594 		}
    595 		crp->crp_mac=cse->tmp_mac;
    596 	}
    597 
    598 	/*
    599 	 * XXX there was a comment here which said that we went to
    600 	 * XXX splcrypto() but needed to only if CRYPTO_F_CBIMM,
    601 	 * XXX disabled on NetBSD since 1.6O due to a race condition.
    602 	 * XXX But crypto_dispatch went to splcrypto() itself!  (And
    603 	 * XXX now takes the crypto_mtx mutex itself).  We do, however,
    604 	 * XXX need to hold the mutex across the call to cv_wait().
    605 	 * XXX     (should we arrange for crypto_dispatch to return to
    606 	 * XXX      us with it held?  it seems quite ugly to do so.)
    607 	 */
    608 #ifdef notyet
    609 eagain:
    610 #endif
    611 	error = crypto_dispatch(crp);
    612 	mutex_spin_enter(&crypto_mtx);
    613 
    614 	/*
    615 	 * If the request was going to be completed by the
    616 	 * ioctl thread then it would have been done by now.
    617 	 * Remove the F_USER flag so crypto_done() is not confused
    618 	 * if the crypto device calls it after this point.
    619 	 */
    620 	crp->crp_flags &= ~(CRYPTO_F_USER);
    621 
    622 	switch (error) {
    623 #ifdef notyet	/* don't loop forever -- but EAGAIN not possible here yet */
    624 	case EAGAIN:
    625 		mutex_spin_exit(&crypto_mtx);
    626 		goto eagain;
    627 		break;
    628 #endif
    629 	case 0:
    630 		break;
    631 	default:
    632 		DPRINTF(("cryptodev_op: not waiting, error.\n"));
    633 		mutex_spin_exit(&crypto_mtx);
    634 		goto bail;
    635 	}
    636 
    637 	while (!(crp->crp_flags & CRYPTO_F_DONE)) {
    638 		DPRINTF(("cryptodev_op[%d]: sleeping on cv %p for crp %p\n",
    639 			(uint32_t)cse->sid, &crp->crp_cv, crp));
    640 		cv_wait(&crp->crp_cv, &crypto_mtx);	/* XXX cv_wait_sig? */
    641 	}
    642 	if (crp->crp_flags & CRYPTO_F_ONRETQ) {
    643 		/* XXX this should never happen now with the CRYPTO_F_USER flag
    644 		 * changes.
    645 		 */
    646 		DPRINTF(("cryptodev_op: DONE, not woken by cryptoret.\n"));
    647 		(void)crypto_ret_q_remove(crp);
    648 	}
    649 	mutex_spin_exit(&crypto_mtx);
    650 
    651 	if (crp->crp_etype != 0) {
    652 		DPRINTF(("cryptodev_op: crp_etype %d\n", crp->crp_etype));
    653 		error = crp->crp_etype;
    654 		goto bail;
    655 	}
    656 
    657 	if (cse->error) {
    658 		DPRINTF(("cryptodev_op: cse->error %d\n", cse->error));
    659 		error = cse->error;
    660 		goto bail;
    661 	}
    662 
    663 	dst_len = crp->crp_ilen;
    664 	/* let the user know how much data was returned */
    665 	if (crp->crp_olen) {
    666 		dst_len = cop->dst_len = crp->crp_olen;
    667 	}
    668 	crp->len = dst_len;
    669 
    670 	if (cop->dst) {
    671 		DPRINTF(("cryptodev_op: copyout %d bytes to %p\n", dst_len, cop->dst));
    672 	}
    673 	if (cop->dst &&
    674 	    (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, dst_len)))
    675 	{
    676 		DPRINTF(("cryptodev_op: copyout error %d\n", error));
    677 		goto bail;
    678 	}
    679 
    680 	if (cop->mac &&
    681 	    (error = copyout(crp->crp_mac, cop->mac, cse->thash->authsize))) {
    682 		DPRINTF(("cryptodev_op: mac copyout error %d\n", error));
    683 		goto bail;
    684 	}
    685 
    686 
    687 bail:
    688 	if (crp) {
    689 		crypto_freereq(crp);
    690 	}
    691 	if (cse->uio.uio_iov[0].iov_base) {
    692 		kmem_free(cse->uio.uio_iov[0].iov_base,iov_len);
    693 	}
    694 
    695 	return error;
    696 }
    697 
    698 static int
    699 cryptodev_cb(void *op)
    700 {
    701 	struct cryptop *crp = (struct cryptop *) op;
    702 	struct csession *cse = (struct csession *)crp->crp_opaque;
    703 	int error = 0;
    704 
    705 	mutex_spin_enter(&crypto_mtx);
    706 	cse->error = crp->crp_etype;
    707 	if (crp->crp_etype == EAGAIN) {
    708 		/* always drop mutex to call dispatch routine */
    709 		mutex_spin_exit(&crypto_mtx);
    710 		error = crypto_dispatch(crp);
    711 		mutex_spin_enter(&crypto_mtx);
    712 	}
    713 	if (error != 0 || (crp->crp_flags & CRYPTO_F_DONE)) {
    714 		cv_signal(&crp->crp_cv);
    715 	}
    716 	mutex_spin_exit(&crypto_mtx);
    717 	return 0;
    718 }
    719 
    720 static int
    721 cryptodev_mcb(void *op)
    722 {
    723 	struct cryptop *crp = (struct cryptop *) op;
    724 	struct csession *cse = (struct csession *)crp->crp_opaque;
    725 	int  error=0;
    726 
    727 	mutex_spin_enter(&crypto_mtx);
    728 	cse->error = crp->crp_etype;
    729 	if (crp->crp_etype == EAGAIN) {
    730 		mutex_spin_exit(&crypto_mtx);
    731 		error = crypto_dispatch(crp);
    732 		mutex_spin_enter(&crypto_mtx);
    733 	}
    734 	if (error != 0 || (crp->crp_flags & CRYPTO_F_DONE)) {
    735 		cv_signal(&crp->crp_cv);
    736 	}
    737 
    738 	TAILQ_INSERT_TAIL(&crp->fcrp->crp_ret_mq, crp, crp_next);
    739 	selnotify(&crp->fcrp->sinfo, 0, 0);
    740 	mutex_spin_exit(&crypto_mtx);
    741 	return 0;
    742 }
    743 
    744 static int
    745 cryptodevkey_cb(void *op)
    746 {
    747 	struct cryptkop *krp = op;
    748 
    749 	mutex_spin_enter(&crypto_mtx);
    750 	cv_signal(&krp->krp_cv);
    751 	mutex_spin_exit(&crypto_mtx);
    752 	return 0;
    753 }
    754 
    755 static int
    756 cryptodevkey_mcb(void *op)
    757 {
    758 	struct cryptkop *krp = op;
    759 
    760 	mutex_spin_enter(&crypto_mtx);
    761 	cv_signal(&krp->krp_cv);
    762 	TAILQ_INSERT_TAIL(&krp->fcrp->crp_ret_mkq, krp, krp_next);
    763 	selnotify(&krp->fcrp->sinfo, 0, 0);
    764 	mutex_spin_exit(&crypto_mtx);
    765 	return 0;
    766 }
    767 
    768 static int
    769 cryptodev_key(struct crypt_kop *kop)
    770 {
    771 	struct cryptkop *krp = NULL;
    772 	int error = EINVAL;
    773 	int in, out, size, i;
    774 
    775 	if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM)
    776 		return EFBIG;
    777 
    778 	in = kop->crk_iparams;
    779 	out = kop->crk_oparams;
    780 	switch (kop->crk_op) {
    781 	case CRK_MOD_EXP:
    782 		if (in == 3 && out == 1)
    783 			break;
    784 		return EINVAL;
    785 	case CRK_MOD_EXP_CRT:
    786 		if (in == 6 && out == 1)
    787 			break;
    788 		return EINVAL;
    789 	case CRK_DSA_SIGN:
    790 		if (in == 5 && out == 2)
    791 			break;
    792 		return EINVAL;
    793 	case CRK_DSA_VERIFY:
    794 		if (in == 7 && out == 0)
    795 			break;
    796 		return EINVAL;
    797 	case CRK_DH_COMPUTE_KEY:
    798 		if (in == 3 && out == 1)
    799 			break;
    800 		return EINVAL;
    801 	case CRK_MOD_ADD:
    802 		if (in == 3 && out == 1)
    803 			break;
    804 		return EINVAL;
    805 	case CRK_MOD_ADDINV:
    806 		if (in == 2 && out == 1)
    807 			break;
    808 		return EINVAL;
    809 	case CRK_MOD_SUB:
    810 		if (in == 3 && out == 1)
    811 			break;
    812 		return EINVAL;
    813 	case CRK_MOD_MULT:
    814 		if (in == 3 && out == 1)
    815 			break;
    816 		return EINVAL;
    817 	case CRK_MOD_MULTINV:
    818 		if (in == 2 && out == 1)
    819 			break;
    820 		return EINVAL;
    821 	case CRK_MOD:
    822 		if (in == 2 && out == 1)
    823 			break;
    824 		return EINVAL;
    825 	default:
    826 		return EINVAL;
    827 	}
    828 
    829 	krp = pool_get(&cryptkop_pool, PR_WAITOK);
    830 	(void)memset(krp, 0, sizeof *krp);
    831 	cv_init(&krp->krp_cv, "crykdev");
    832 	krp->krp_op = kop->crk_op;
    833 	krp->krp_status = kop->crk_status;
    834 	krp->krp_iparams = kop->crk_iparams;
    835 	krp->krp_oparams = kop->crk_oparams;
    836 	krp->krp_status = 0;
    837 	krp->krp_callback = (int (*) (struct cryptkop *)) cryptodevkey_cb;
    838 
    839 	for (i = 0; i < CRK_MAXPARAM; i++)
    840 		krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits;
    841 	for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
    842 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
    843 		if (size == 0)
    844 			continue;
    845 		krp->krp_param[i].crp_p = kmem_alloc(size, KM_SLEEP);
    846 		if (i >= krp->krp_iparams)
    847 			continue;
    848 		error = copyin(kop->crk_param[i].crp_p,
    849 		    krp->krp_param[i].crp_p, size);
    850 		if (error)
    851 			goto fail;
    852 	}
    853 
    854 	error = crypto_kdispatch(krp);
    855 	if (error != 0) {
    856 		goto fail;
    857 	}
    858 
    859 	mutex_spin_enter(&crypto_mtx);
    860 	while (!(krp->krp_flags & CRYPTO_F_DONE)) {
    861 		cv_wait(&krp->krp_cv, &crypto_mtx);	/* XXX cv_wait_sig? */
    862 	}
    863 	if (krp->krp_flags & CRYPTO_F_ONRETQ) {
    864 		DPRINTF(("cryptodev_key: DONE early, not via cryptoret.\n"));
    865 		(void)crypto_ret_kq_remove(krp);
    866 	}
    867 	mutex_spin_exit(&crypto_mtx);
    868 
    869 	if (krp->krp_status != 0) {
    870 		DPRINTF(("cryptodev_key: krp->krp_status 0x%08x\n",
    871 		    krp->krp_status));
    872 		error = krp->krp_status;
    873 		goto fail;
    874 	}
    875 
    876 	for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams;
    877 	    i++) {
    878 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
    879 		if (size == 0)
    880 			continue;
    881 		error = copyout(krp->krp_param[i].crp_p,
    882 		    kop->crk_param[i].crp_p, size);
    883 		if (error) {
    884 			DPRINTF(("cryptodev_key: copyout oparam %d failed, "
    885 			    "error=%d\n", i-krp->krp_iparams, error));
    886 			goto fail;
    887 		}
    888 	}
    889 
    890 fail:
    891 	kop->crk_status = krp->krp_status;
    892 	for (i = 0; i < CRK_MAXPARAM; i++) {
    893 		struct crparam *kp = &(krp->krp_param[i]);
    894 		if (krp->krp_param[i].crp_p) {
    895 			size = (kp->crp_nbits + 7)  / 8;
    896 			KASSERT(size > 0);
    897 			(void)memset(kp->crp_p, 0, size);
    898 			kmem_free(kp->crp_p, size);
    899 		}
    900 	}
    901 	cv_destroy(&krp->krp_cv);
    902 	pool_put(&cryptkop_pool, krp);
    903 	DPRINTF(("cryptodev_key: error=0x%08x\n", error));
    904 	return error;
    905 }
    906 
    907 /* ARGSUSED */
    908 static int
    909 cryptof_close(struct file *fp)
    910 {
    911 	struct fcrypt *fcr = fp->f_data;
    912 	struct csession *cse;
    913 
    914 	mutex_spin_enter(&crypto_mtx);
    915 	while ((cse = TAILQ_FIRST(&fcr->csessions))) {
    916 		TAILQ_REMOVE(&fcr->csessions, cse, next);
    917 		(void)csefree(cse);
    918 	}
    919 	seldestroy(&fcr->sinfo);
    920 	fp->f_data = NULL;
    921 	mutex_spin_exit(&crypto_mtx);
    922 
    923 	pool_put(&fcrpl, fcr);
    924 	return 0;
    925 }
    926 
    927 /* needed for compatibility module */
    928 struct	csession *cryptodev_csefind(struct fcrypt *fcr, u_int ses)
    929 {
    930 	return csefind(fcr, ses);
    931 }
    932 
    933 /* csefind: call with crypto_mtx held. */
    934 static struct csession *
    935 csefind(struct fcrypt *fcr, u_int ses)
    936 {
    937 	struct csession *cse, *cnext, *ret = NULL;
    938 
    939 	KASSERT(mutex_owned(&crypto_mtx));
    940 	TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext)
    941 		if (cse->ses == ses)
    942 			ret = cse;
    943 
    944 	return ret;
    945 }
    946 
    947 /* csedelete: call with crypto_mtx held. */
    948 static int
    949 csedelete(struct fcrypt *fcr, struct csession *cse_del)
    950 {
    951 	struct csession *cse, *cnext;
    952 	int ret = 0;
    953 
    954 	KASSERT(mutex_owned(&crypto_mtx));
    955 	TAILQ_FOREACH_SAFE(cse, &fcr->csessions, next, cnext) {
    956 		if (cse == cse_del) {
    957 			TAILQ_REMOVE(&fcr->csessions, cse, next);
    958 			ret = 1;
    959 		}
    960 	}
    961 	return ret;
    962 }
    963 
    964 /* cseadd: call with crypto_mtx held. */
    965 static struct csession *
    966 cseadd(struct fcrypt *fcr, struct csession *cse)
    967 {
    968 	KASSERT(mutex_owned(&crypto_mtx));
    969 	/* don't let session ID wrap! */
    970 	if (fcr->sesn + 1 == 0) return NULL;
    971 	TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
    972 	cse->ses = fcr->sesn++;
    973 	return cse;
    974 }
    975 
    976 /* csecreate: call with crypto_mtx held. */
    977 static struct csession *
    978 csecreate(struct fcrypt *fcr, u_int64_t sid, void *key, u_int64_t keylen,
    979     void *mackey, u_int64_t mackeylen, u_int32_t cipher, u_int32_t mac,
    980     u_int32_t comp_alg, struct enc_xform *txform, struct auth_hash *thash,
    981     struct comp_algo *tcomp)
    982 {
    983 	struct csession *cse;
    984 
    985 	KASSERT(mutex_owned(&crypto_mtx));
    986 	cse = pool_get(&csepl, PR_NOWAIT);
    987 	if (cse == NULL)
    988 		return NULL;
    989 	cse->key = key;
    990 	cse->keylen = keylen/8;
    991 	cse->mackey = mackey;
    992 	cse->mackeylen = mackeylen/8;
    993 	cse->sid = sid;
    994 	cse->cipher = cipher;
    995 	cse->mac = mac;
    996 	cse->comp_alg = comp_alg;
    997 	cse->txform = txform;
    998 	cse->thash = thash;
    999 	cse->tcomp = tcomp;
   1000 	cse->error = 0;
   1001 	if (cseadd(fcr, cse))
   1002 		return cse;
   1003 	else {
   1004 		pool_put(&csepl, cse);
   1005 		return NULL;
   1006 	}
   1007 }
   1008 
   1009 /* csefree: call with crypto_mtx held. */
   1010 static int
   1011 csefree(struct csession *cse)
   1012 {
   1013 	int error;
   1014 
   1015 	KASSERT(mutex_owned(&crypto_mtx));
   1016 	error = crypto_freesession(cse->sid);
   1017 	if (cse->key)
   1018 		free(cse->key, M_XDATA);
   1019 	if (cse->mackey)
   1020 		free(cse->mackey, M_XDATA);
   1021 	pool_put(&csepl, cse);
   1022 	return error;
   1023 }
   1024 
   1025 static int
   1026 cryptoopen(dev_t dev, int flag, int mode,
   1027     struct lwp *l)
   1028 {
   1029 	file_t *fp;
   1030         struct fcrypt *fcr;
   1031         int fd, error;
   1032 
   1033 	if (crypto_usercrypto == 0)
   1034 		return ENXIO;
   1035 
   1036 	if ((error = fd_allocfile(&fp, &fd)) != 0)
   1037 		return error;
   1038 
   1039 	fcr = pool_get(&fcrpl, PR_WAITOK);
   1040 	getnanotime(&fcr->btime);
   1041 	fcr->atime = fcr->mtime = fcr->btime;
   1042 	mutex_spin_enter(&crypto_mtx);
   1043 	TAILQ_INIT(&fcr->csessions);
   1044 	TAILQ_INIT(&fcr->crp_ret_mq);
   1045 	TAILQ_INIT(&fcr->crp_ret_mkq);
   1046 	selinit(&fcr->sinfo);
   1047 	/*
   1048 	 * Don't ever return session 0, to allow detection of
   1049 	 * failed creation attempts with multi-create ioctl.
   1050 	 */
   1051 	fcr->sesn = 1;
   1052 	fcr->requestid = 1;
   1053 	mutex_spin_exit(&crypto_mtx);
   1054 	return fd_clone(fp, fd, flag, &cryptofops, fcr);
   1055 }
   1056 
   1057 static int
   1058 cryptoread(dev_t dev, struct uio *uio, int ioflag)
   1059 {
   1060 	return EIO;
   1061 }
   1062 
   1063 static int
   1064 cryptowrite(dev_t dev, struct uio *uio, int ioflag)
   1065 {
   1066 	return EIO;
   1067 }
   1068 
   1069 int
   1070 cryptoselect(dev_t dev, int rw, struct lwp *l)
   1071 {
   1072 	return 0;
   1073 }
   1074 
   1075 /*static*/
   1076 struct cdevsw crypto_cdevsw = {
   1077 	/* open */	cryptoopen,
   1078 	/* close */	noclose,
   1079 	/* read */	cryptoread,
   1080 	/* write */	cryptowrite,
   1081 	/* ioctl */	noioctl,
   1082 	/* ttstop?*/	nostop,
   1083 	/* ??*/		notty,
   1084 	/* poll */	cryptoselect /*nopoll*/,
   1085 	/* mmap */	nommap,
   1086 	/* kqfilter */	nokqfilter,
   1087 	/* type */	D_OTHER,
   1088 };
   1089 
   1090 int
   1091 cryptodev_mop(struct fcrypt *fcr,
   1092               struct crypt_n_op * cnop,
   1093               int count, struct lwp *l)
   1094 {
   1095 	struct cryptop *crp = NULL;
   1096 	struct cryptodesc *crde = NULL, *crda = NULL, *crdc = NULL;
   1097 	int req, error=0;
   1098 	struct csession *cse;
   1099 	int flags=0;
   1100 	int iov_len;
   1101 
   1102 	for (req = 0; req < count; req++) {
   1103 		mutex_spin_enter(&crypto_mtx);
   1104 		cse = csefind(fcr, cnop[req].ses);
   1105 		if (cse == NULL) {
   1106 			DPRINTF(("csefind failed\n"));
   1107 			cnop[req].status = EINVAL;
   1108 			mutex_spin_exit(&crypto_mtx);
   1109 			continue;
   1110 		}
   1111 		mutex_spin_exit(&crypto_mtx);
   1112 
   1113 		if (cnop[req].len > 256*1024-4) {
   1114 			DPRINTF(("length failed\n"));
   1115 			cnop[req].status = EINVAL;
   1116 			continue;
   1117 		}
   1118 		if (cse->txform) {
   1119 			if (cnop[req].len == 0 ||
   1120 			    (cnop[req].len % cse->txform->blocksize) != 0) {
   1121 				cnop[req].status = EINVAL;
   1122 				continue;
   1123 			}
   1124 		}
   1125 
   1126 		crp = crypto_getreq((cse->txform != NULL) +
   1127 				    (cse->thash != NULL) +
   1128 				    (cse->tcomp != NULL));
   1129 		if (crp == NULL) {
   1130 			cnop[req].status = ENOMEM;
   1131 			goto bail;
   1132 		}
   1133 
   1134 		iov_len = cnop[req].len;
   1135 		/* got a compression/decompression max size? */
   1136 		if ((cse->tcomp) && cnop[req].dst_len) {
   1137 			if (iov_len < cnop[req].dst_len) {
   1138 				/* Need larger iov to deal with decompress */
   1139 				iov_len = cnop[req].dst_len;
   1140 			}
   1141 			DPRINTF(("cryptodev_mop: iov_len -> %d for decompress\n", iov_len));
   1142 		}
   1143 
   1144 		(void)memset(&crp->uio, 0, sizeof(crp->uio));
   1145 		crp->uio.uio_iovcnt = 1;
   1146 		crp->uio.uio_resid = 0;
   1147 		crp->uio.uio_rw = UIO_WRITE;
   1148 		crp->uio.uio_iov = crp->iovec;
   1149 		UIO_SETUP_SYSSPACE(&crp->uio);
   1150 		memset(&crp->iovec, 0, sizeof(crp->iovec));
   1151 		crp->uio.uio_iov[0].iov_len = iov_len;
   1152 		DPRINTF(("cryptodev_mop: kmem_alloc(%d) for iov \n", iov_len));
   1153 		crp->uio.uio_iov[0].iov_base = kmem_alloc(iov_len, KM_SLEEP);
   1154 		crp->uio.uio_resid = crp->uio.uio_iov[0].iov_len;
   1155 
   1156 		if (cse->tcomp) {
   1157 			crdc = crp->crp_desc;
   1158 		}
   1159 
   1160 		if (cse->thash) {
   1161 			crda = crdc ? crdc->crd_next : crp->crp_desc;
   1162 			if (cse->txform && crda)
   1163 				crde = crda->crd_next;
   1164 		} else {
   1165 			if (cse->txform) {
   1166 				crde = crdc ? crdc->crd_next : crp->crp_desc;
   1167 			} else if (!cse->tcomp) {
   1168 				error = EINVAL;
   1169 				goto bail;
   1170 			}
   1171 		}
   1172 
   1173 		if ((copyin(cnop[req].src,
   1174 		    crp->uio.uio_iov[0].iov_base, cnop[req].len))) {
   1175 			cnop[req].status = EINVAL;
   1176 			goto bail;
   1177 		}
   1178 
   1179 		if (crdc) {
   1180 			switch (cnop[req].op) {
   1181 			case COP_COMP:
   1182 				crdc->crd_flags |= CRD_F_COMP;
   1183 				break;
   1184 			case COP_DECOMP:
   1185 				crdc->crd_flags &= ~CRD_F_COMP;
   1186 				break;
   1187 			default:
   1188 				break;
   1189 			}
   1190 			/* more data to follow? */
   1191 			if (cnop[req].flags & COP_F_MORE) {
   1192 				flags |= CRYPTO_F_MORE;
   1193 			}
   1194 			crdc->crd_len = cnop[req].len;
   1195 			crdc->crd_inject = 0;
   1196 
   1197 			crdc->crd_alg = cse->comp_alg;
   1198 			crdc->crd_key = NULL;
   1199 			crdc->crd_klen = 0;
   1200 			DPRINTF(("cryptodev_mop[%d]: crdc setup for comp_alg %d"
   1201 				 " len %d.\n",
   1202 				(uint32_t)cse->sid, crdc->crd_alg,
   1203 				crdc->crd_len));
   1204 		}
   1205 
   1206 		if (crda) {
   1207 			crda->crd_skip = 0;
   1208 			crda->crd_len = cnop[req].len;
   1209 			crda->crd_inject = 0;	/* ??? */
   1210 
   1211 			crda->crd_alg = cse->mac;
   1212 			crda->crd_key = cse->mackey;
   1213 			crda->crd_klen = cse->mackeylen * 8;
   1214 		}
   1215 
   1216 		if (crde) {
   1217 			if (cnop[req].op == COP_ENCRYPT)
   1218 				crde->crd_flags |= CRD_F_ENCRYPT;
   1219 			else
   1220 				crde->crd_flags &= ~CRD_F_ENCRYPT;
   1221 			crde->crd_len = cnop[req].len;
   1222 			crde->crd_inject = 0;
   1223 
   1224 			crde->crd_alg = cse->cipher;
   1225 #ifdef notyet		/* XXX must notify h/w driver new key, drain */
   1226 			if(cnop[req].key && cnop[req].keylen) {
   1227 				crde->crd_key = malloc(cnop[req].keylen,
   1228 						    M_XDATA, M_WAITOK);
   1229 				if((error = copyin(cnop[req].key,
   1230 				    crde->crd_key, cnop[req].keylen))) {
   1231 					cnop[req].status = EINVAL;
   1232 					goto bail;
   1233 				}
   1234 				crde->crd_klen =  cnop[req].keylen * 8;
   1235 			} else { ... }
   1236 #endif
   1237 			crde->crd_key = cse->key;
   1238 			crde->crd_klen = cse->keylen * 8;
   1239 		}
   1240 
   1241 		crp->crp_ilen = cnop[req].len;
   1242 		crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM |
   1243 		    (cnop[req].flags & COP_F_BATCH) | flags;
   1244 		crp->crp_buf = (void *)&crp->uio;
   1245 		crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_mcb;
   1246 		crp->crp_sid = cse->sid;
   1247 		crp->crp_opaque = (void *)cse;
   1248 		crp->fcrp = fcr;
   1249 		crp->dst = cnop[req].dst;
   1250 		crp->len = cnop[req].len; /* input len, iov may be larger */
   1251 		crp->mac = cnop[req].mac;
   1252 		DPRINTF(("cryptodev_mop: iov_base %p dst %p len %d mac %p\n",
   1253 			    crp->uio.uio_iov[0].iov_base, crp->dst, crp->len,
   1254 			    crp->mac));
   1255 
   1256 		if (cnop[req].iv) {
   1257 			if (crde == NULL) {
   1258 				cnop[req].status = EINVAL;
   1259 				goto bail;
   1260 			}
   1261 			if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
   1262 				cnop[req].status = EINVAL;
   1263 				goto bail;
   1264 			}
   1265 			if ((error = copyin(cnop[req].iv, crp->tmp_iv,
   1266 			    cse->txform->blocksize))) {
   1267 				cnop[req].status = EINVAL;
   1268 				goto bail;
   1269 			}
   1270 			(void)memcpy(crde->crd_iv, crp->tmp_iv,
   1271 			    cse->txform->blocksize);
   1272 			crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
   1273 			crde->crd_skip = 0;
   1274 		} else if (crde) {
   1275 			if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
   1276 				crde->crd_skip = 0;
   1277 			} else {
   1278 				crde->crd_flags |= CRD_F_IV_PRESENT;
   1279 				crde->crd_skip = cse->txform->blocksize;
   1280 				crde->crd_len -= cse->txform->blocksize;
   1281 			}
   1282 		}
   1283 
   1284 		if (cnop[req].mac) {
   1285 			if (crda == NULL) {
   1286 				cnop[req].status = EINVAL;
   1287 				goto bail;
   1288 			}
   1289 			crp->crp_mac=cse->tmp_mac;
   1290 		}
   1291 		cnop[req].reqid = atomic_inc_32_nv(&(fcr->requestid));
   1292 		crp->crp_reqid = cnop[req].reqid;
   1293 		crp->crp_usropaque = cnop[req].opaque;
   1294 #ifdef notyet
   1295 eagain:
   1296 #endif
   1297 		cnop[req].status = crypto_dispatch(crp);
   1298 		mutex_spin_enter(&crypto_mtx);	/* XXX why mutex? */
   1299 
   1300 		switch (cnop[req].status) {
   1301 #ifdef notyet	/* don't loop forever -- but EAGAIN not possible here yet */
   1302 		case EAGAIN:
   1303 			mutex_spin_exit(&crypto_mtx);
   1304 			goto eagain;
   1305 			break;
   1306 #endif
   1307 		case 0:
   1308 			break;
   1309 		default:
   1310 			DPRINTF(("cryptodev_op: not waiting, error.\n"));
   1311 			mutex_spin_exit(&crypto_mtx);
   1312 			goto bail;
   1313 		}
   1314 
   1315 		mutex_spin_exit(&crypto_mtx);
   1316 bail:
   1317 		if (cnop[req].status) {
   1318 			if (crp) {
   1319 				if (crp->uio.uio_iov[0].iov_base) {
   1320 					kmem_free(crp->uio.uio_iov[0].iov_base,
   1321 					    crp->uio.uio_iov[0].iov_len);
   1322 				}
   1323 				crypto_freereq(crp);
   1324 			}
   1325 			error = 0;
   1326 		}
   1327 	}
   1328 	return error;
   1329 }
   1330 
   1331 static int
   1332 cryptodev_mkey(struct fcrypt *fcr, struct crypt_n_kop *kop, int count)
   1333 {
   1334 	struct cryptkop *krp = NULL;
   1335 	int error = EINVAL;
   1336 	int in, out, size, i, req;
   1337 
   1338 	for (req = 0; req < count; req++) {
   1339 		if (kop[req].crk_iparams + kop[req].crk_oparams > CRK_MAXPARAM)
   1340 			return EFBIG;
   1341 
   1342 		in = kop[req].crk_iparams;
   1343 		out = kop[req].crk_oparams;
   1344 		switch (kop[req].crk_op) {
   1345 		case CRK_MOD_EXP:
   1346 			if (in == 3 && out == 1)
   1347 				break;
   1348 			kop[req].crk_status = EINVAL;
   1349 			continue;
   1350 		case CRK_MOD_EXP_CRT:
   1351 			if (in == 6 && out == 1)
   1352 				break;
   1353 			kop[req].crk_status = EINVAL;
   1354 			continue;
   1355 		case CRK_DSA_SIGN:
   1356 			if (in == 5 && out == 2)
   1357 				break;
   1358 			kop[req].crk_status = EINVAL;
   1359 			continue;
   1360 		case CRK_DSA_VERIFY:
   1361 			if (in == 7 && out == 0)
   1362 				break;
   1363 			kop[req].crk_status = EINVAL;
   1364 			continue;
   1365 		case CRK_DH_COMPUTE_KEY:
   1366 			if (in == 3 && out == 1)
   1367 				break;
   1368 			kop[req].crk_status = EINVAL;
   1369 			continue;
   1370 		case CRK_MOD_ADD:
   1371 			if (in == 3 && out == 1)
   1372 				break;
   1373 			kop[req].crk_status = EINVAL;
   1374 			continue;
   1375 		case CRK_MOD_ADDINV:
   1376 			if (in == 2 && out == 1)
   1377 				break;
   1378 			kop[req].crk_status = EINVAL;
   1379 			continue;
   1380 		case CRK_MOD_SUB:
   1381 			if (in == 3 && out == 1)
   1382 				break;
   1383 			kop[req].crk_status = EINVAL;
   1384 			continue;
   1385 		case CRK_MOD_MULT:
   1386 			if (in == 3 && out == 1)
   1387 				break;
   1388 			kop[req].crk_status = EINVAL;
   1389 			continue;
   1390 		case CRK_MOD_MULTINV:
   1391 			if (in == 2 && out == 1)
   1392 				break;
   1393 			kop[req].crk_status = EINVAL;
   1394 			continue;
   1395 		case CRK_MOD:
   1396 			if (in == 2 && out == 1)
   1397 				break;
   1398 			kop[req].crk_status = EINVAL;
   1399 			continue;
   1400 		default:
   1401 			kop[req].crk_status = EINVAL;
   1402 			continue;
   1403 		}
   1404 
   1405 		krp = pool_get(&cryptkop_pool, PR_WAITOK);
   1406 		(void)memset(krp, 0, sizeof *krp);
   1407 		cv_init(&krp->krp_cv, "crykdev");
   1408 		krp->krp_op = kop[req].crk_op;
   1409 		krp->krp_status = kop[req].crk_status;
   1410 		krp->krp_iparams = kop[req].crk_iparams;
   1411 		krp->krp_oparams = kop[req].crk_oparams;
   1412 		krp->krp_status = 0;
   1413 		krp->krp_callback =
   1414 		    (int (*) (struct cryptkop *)) cryptodevkey_mcb;
   1415 		(void)memcpy(krp->crk_param, kop[req].crk_param,
   1416 		    sizeof(kop[req].crk_param));
   1417 
   1418 		krp->krp_flags = CRYPTO_F_CBIMM;
   1419 
   1420 		for (i = 0; i < CRK_MAXPARAM; i++)
   1421 			krp->krp_param[i].crp_nbits =
   1422 			    kop[req].crk_param[i].crp_nbits;
   1423 		for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
   1424 			size = (krp->krp_param[i].crp_nbits + 7) / 8;
   1425 			if (size == 0)
   1426 				continue;
   1427 			krp->krp_param[i].crp_p =
   1428 			    kmem_alloc(size, KM_SLEEP);
   1429 			if (i >= krp->krp_iparams)
   1430 				continue;
   1431 			kop[req].crk_status =
   1432 			    copyin(kop[req].crk_param[i].crp_p,
   1433 			    krp->krp_param[i].crp_p, size);
   1434 			if (kop[req].crk_status)
   1435 				goto fail;
   1436 		}
   1437 		krp->fcrp = fcr;
   1438 
   1439 		kop[req].crk_reqid = atomic_inc_32_nv(&(fcr->requestid));
   1440 		krp->krp_reqid = kop[req].crk_reqid;
   1441 		krp->krp_usropaque = kop[req].crk_opaque;
   1442 
   1443 		kop[req].crk_status = crypto_kdispatch(krp);
   1444 		if (kop[req].crk_status != 0) {
   1445 			goto fail;
   1446 		}
   1447 
   1448 fail:
   1449 		if(kop[req].crk_status) {
   1450 			if (krp) {
   1451 				kop[req].crk_status = krp->krp_status;
   1452 				for (i = 0; i < CRK_MAXPARAM; i++) {
   1453 					struct crparam *kp =
   1454 						&(krp->krp_param[i]);
   1455 					if (kp->crp_p) {
   1456 						size = (kp->crp_nbits + 7) / 8;
   1457 						KASSERT(size > 0);
   1458 						memset(kp->crp_p, 0, size);
   1459 						kmem_free(kp->crp_p, size);
   1460 					}
   1461 				}
   1462 				cv_destroy(&krp->krp_cv);
   1463 				pool_put(&cryptkop_pool, krp);
   1464 			}
   1465 		}
   1466 		error = 0;
   1467 	}
   1468 	DPRINTF(("cryptodev_key: error=0x%08x\n", error));
   1469 	return error;
   1470 }
   1471 
   1472 int
   1473 cryptodev_session(struct fcrypt *fcr, struct session_op *sop)
   1474 {
   1475 	struct cryptoini cria, crie;
   1476 	struct cryptoini cric;		/* compressor */
   1477 	struct cryptoini *crihead = NULL;
   1478 	struct enc_xform *txform = NULL;
   1479 	struct auth_hash *thash = NULL;
   1480 	struct comp_algo *tcomp = NULL;
   1481 	struct csession *cse;
   1482 	u_int64_t sid;
   1483 	int error = 0;
   1484 
   1485 	DPRINTF(("cryptodev_session() cipher=%d, mac=%d\n", sop->cipher, sop->mac));
   1486 
   1487 	/* XXX there must be a way to not embed the list of xforms here */
   1488 	switch (sop->cipher) {
   1489 	case 0:
   1490 		break;
   1491 	case CRYPTO_DES_CBC:
   1492 		txform = &enc_xform_des;
   1493 		break;
   1494 	case CRYPTO_3DES_CBC:
   1495 		txform = &enc_xform_3des;
   1496 		break;
   1497 	case CRYPTO_BLF_CBC:
   1498 		txform = &enc_xform_blf;
   1499 		break;
   1500 	case CRYPTO_CAST_CBC:
   1501 		txform = &enc_xform_cast5;
   1502 		break;
   1503 	case CRYPTO_SKIPJACK_CBC:
   1504 		txform = &enc_xform_skipjack;
   1505 		break;
   1506 	case CRYPTO_AES_CBC:
   1507 		txform = &enc_xform_rijndael128;
   1508 		break;
   1509 	case CRYPTO_NULL_CBC:
   1510 		txform = &enc_xform_null;
   1511 		break;
   1512 	case CRYPTO_ARC4:
   1513 		txform = &enc_xform_arc4;
   1514 		break;
   1515 	default:
   1516 		DPRINTF(("Invalid cipher %d\n", sop->cipher));
   1517 		return EINVAL;
   1518 	}
   1519 
   1520 	switch (sop->comp_alg) {
   1521 	case 0:
   1522 		break;
   1523 	case CRYPTO_DEFLATE_COMP:
   1524 		tcomp = &comp_algo_deflate;
   1525 		break;
   1526 	case CRYPTO_GZIP_COMP:
   1527 		tcomp = &comp_algo_gzip;
   1528 		DPRINTF(("cryptodev_session() tcomp for GZIP\n"));
   1529 		break;
   1530 	default:
   1531 		DPRINTF(("Invalid compression alg %d\n", sop->comp_alg));
   1532 		return EINVAL;
   1533 	}
   1534 
   1535 	switch (sop->mac) {
   1536 	case 0:
   1537 		break;
   1538 	case CRYPTO_MD5_HMAC:
   1539 		thash = &auth_hash_hmac_md5;
   1540 		break;
   1541 	case CRYPTO_SHA1_HMAC:
   1542 		thash = &auth_hash_hmac_sha1;
   1543 		break;
   1544 	case CRYPTO_MD5_HMAC_96:
   1545 		thash = &auth_hash_hmac_md5_96;
   1546 		break;
   1547 	case CRYPTO_SHA1_HMAC_96:
   1548 		thash = &auth_hash_hmac_sha1_96;
   1549 		break;
   1550 	case CRYPTO_SHA2_HMAC:
   1551 		/* XXX switching on key length seems questionable */
   1552 		if (sop->mackeylen == auth_hash_hmac_sha2_256.keysize) {
   1553 			thash = &auth_hash_hmac_sha2_256;
   1554 		} else if (sop->mackeylen == auth_hash_hmac_sha2_384.keysize) {
   1555 			thash = &auth_hash_hmac_sha2_384;
   1556 		} else if (sop->mackeylen == auth_hash_hmac_sha2_512.keysize) {
   1557 			thash = &auth_hash_hmac_sha2_512;
   1558 		} else {
   1559 			DPRINTF(("Invalid mackeylen %d\n", sop->mackeylen));
   1560 			return EINVAL;
   1561 		}
   1562 		break;
   1563 	case CRYPTO_RIPEMD160_HMAC:
   1564 		thash = &auth_hash_hmac_ripemd_160;
   1565 		break;
   1566 	case CRYPTO_RIPEMD160_HMAC_96:
   1567 		thash = &auth_hash_hmac_ripemd_160_96;
   1568 		break;
   1569 	case CRYPTO_MD5:
   1570 		thash = &auth_hash_md5;
   1571 		break;
   1572 	case CRYPTO_SHA1:
   1573 		thash = &auth_hash_sha1;
   1574 		break;
   1575 	case CRYPTO_NULL_HMAC:
   1576 		thash = &auth_hash_null;
   1577 		break;
   1578 	default:
   1579 		DPRINTF(("Invalid mac %d\n", sop->mac));
   1580 		return EINVAL;
   1581 	}
   1582 
   1583 	memset(&crie, 0, sizeof(crie));
   1584 	memset(&cria, 0, sizeof(cria));
   1585 	memset(&cric, 0, sizeof(cric));
   1586 
   1587 	if (tcomp) {
   1588 		cric.cri_alg = tcomp->type;
   1589 		cric.cri_klen = 0;
   1590 		DPRINTF(("tcomp->type = %d\n", tcomp->type));
   1591 
   1592 		crihead = &cric;
   1593 		if (thash) {
   1594 			cric.cri_next = &cria;
   1595 		} else if (txform) {
   1596 			cric.cri_next = &crie;
   1597 		}
   1598 	}
   1599 
   1600 	if (txform) {
   1601 		crie.cri_alg = txform->type;
   1602 		crie.cri_klen = sop->keylen * 8;
   1603 		if (sop->keylen > txform->maxkey ||
   1604 		    sop->keylen < txform->minkey) {
   1605 			DPRINTF(("keylen %d not in [%d,%d]\n",
   1606 			    sop->keylen, txform->minkey, txform->maxkey));
   1607 			error = EINVAL;
   1608 			goto bail;
   1609 		}
   1610 
   1611 		crie.cri_key = malloc(crie.cri_klen / 8, M_XDATA, M_WAITOK);
   1612 		if ((error = copyin(sop->key, crie.cri_key, crie.cri_klen / 8)))
   1613 			goto bail;
   1614 		if (!crihead) {
   1615 			crihead = &crie;
   1616 		}
   1617 	}
   1618 
   1619 	if (thash) {
   1620 		cria.cri_alg = thash->type;
   1621 		cria.cri_klen = sop->mackeylen * 8;
   1622 		if (sop->mackeylen != thash->keysize) {
   1623 			DPRINTF(("mackeylen %d != keysize %d\n",
   1624 			    sop->mackeylen, thash->keysize));
   1625 			error = EINVAL;
   1626 			goto bail;
   1627 		}
   1628 		if (cria.cri_klen) {
   1629 			cria.cri_key = malloc(cria.cri_klen / 8, M_XDATA,
   1630 			    M_WAITOK);
   1631 			if ((error = copyin(sop->mackey, cria.cri_key,
   1632 			    cria.cri_klen / 8))) {
   1633 				goto bail;
   1634 			}
   1635 		}
   1636 		if (txform)
   1637 			cria.cri_next = &crie;	/* XXX forces enc then hash? */
   1638 		if (!crihead) {
   1639 			crihead = &cria;
   1640 		}
   1641 	}
   1642 
   1643 	/* crypto_newsession requires that we hold the mutex. */
   1644 	mutex_spin_enter(&crypto_mtx);
   1645 	error = crypto_newsession(&sid, crihead, crypto_devallowsoft);
   1646 	if (!error) {
   1647 		DPRINTF(("cyrptodev_session: got session %d\n", (uint32_t)sid));
   1648 		cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen,
   1649 		    cria.cri_key, cria.cri_klen, (txform ? sop->cipher : 0), sop->mac,
   1650 		    (tcomp ? sop->comp_alg : 0), txform, thash, tcomp);
   1651 		if (cse != NULL) {
   1652 			sop->ses = cse->ses;
   1653 		} else {
   1654 			DPRINTF(("csecreate failed\n"));
   1655 			crypto_freesession(sid);
   1656 			error = EINVAL;
   1657 		}
   1658 	} else {
   1659 		DPRINTF(("SIOCSESSION violates kernel parameters %d\n",
   1660 		    error));
   1661 	}
   1662 	mutex_spin_exit(&crypto_mtx);
   1663 bail:
   1664 	if (error) {
   1665 		if (crie.cri_key) {
   1666 			memset(crie.cri_key, 0, crie.cri_klen / 8);
   1667 			free(crie.cri_key, M_XDATA);
   1668 		}
   1669 		if (cria.cri_key) {
   1670 			memset(cria.cri_key, 0, cria.cri_klen / 8);
   1671 			free(cria.cri_key, M_XDATA);
   1672 		}
   1673 	}
   1674 	return error;
   1675 }
   1676 
   1677 int
   1678 cryptodev_msession(struct fcrypt *fcr, struct session_n_op *sn_ops,
   1679 		   int count)
   1680 {
   1681 	int i;
   1682 
   1683 	for (i = 0; i < count; i++, sn_ops++) {
   1684 		struct session_op s_op;
   1685 		s_op.cipher =		sn_ops->cipher;
   1686 		s_op.mac =		sn_ops->mac;
   1687 		s_op.keylen =		sn_ops->keylen;
   1688 		s_op.key =		sn_ops->key;
   1689 		s_op.mackeylen =	sn_ops->mackeylen;
   1690 		s_op.mackey =		sn_ops->mackey;
   1691 
   1692 		sn_ops->status = cryptodev_session(fcr, &s_op);
   1693 		sn_ops->ses =		s_op.ses;
   1694 	}
   1695 
   1696 	return 0;
   1697 }
   1698 
   1699 static int
   1700 cryptodev_msessionfin(struct fcrypt *fcr, int count, u_int32_t *sesid)
   1701 {
   1702 	struct csession *cse;
   1703 	int req, error = 0;
   1704 
   1705 	mutex_spin_enter(&crypto_mtx);
   1706 	for(req = 0; req < count; req++) {
   1707 		cse = csefind(fcr, sesid[req]);
   1708 		if (cse == NULL)
   1709 			continue;
   1710 		csedelete(fcr, cse);
   1711 		error = csefree(cse);
   1712 	}
   1713 	mutex_spin_exit(&crypto_mtx);
   1714 	return 0;
   1715 }
   1716 
   1717 /*
   1718  * collect as many completed requests as are availble, or count completed
   1719  * requests whichever is less.
   1720  * return the number of requests.
   1721  */
   1722 static int
   1723 cryptodev_getmstatus(struct fcrypt *fcr, struct crypt_result *crypt_res,
   1724     int count)
   1725 {
   1726 	struct cryptop *crp = NULL;
   1727 	struct cryptkop *krp = NULL;
   1728 	struct csession *cse;
   1729 	int i, size, req = 0;
   1730 	int completed=0;
   1731 
   1732 	/* On queue so nobody else can grab them
   1733 	 * and copyout can be delayed-- no locking */
   1734 	TAILQ_HEAD(, cryptop) crp_delfree_q =
   1735 		TAILQ_HEAD_INITIALIZER(crp_delfree_q);
   1736 	TAILQ_HEAD(, cryptkop) krp_delfree_q =
   1737 		TAILQ_HEAD_INITIALIZER(krp_delfree_q);
   1738 
   1739 	/* at this point we do not know which response user is requesting for
   1740 	 * (symmetric or asymmetric) so we copyout one from each i.e if the
   1741 	 * count is 2 then 1 from symmetric and 1 from asymmetric queue and
   1742 	 * if 3 then 2 symmetric and 1 asymmetric and so on */
   1743 
   1744 	/* pull off a list of requests while protected from changes */
   1745 	mutex_spin_enter(&crypto_mtx);
   1746 	while (req < count) {
   1747 		crp = TAILQ_FIRST(&fcr->crp_ret_mq);
   1748 		if (crp) {
   1749 			TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next);
   1750 			TAILQ_INSERT_TAIL(&crp_delfree_q, crp, crp_next);
   1751 			cse = (struct csession *)crp->crp_opaque;
   1752 
   1753 			/* see if the session is still valid */
   1754 			cse = csefind(fcr, cse->ses);
   1755 			if (cse != NULL) {
   1756 				crypt_res[req].status = 0;
   1757 			} else {
   1758 				DPRINTF(("csefind failed\n"));
   1759 				crypt_res[req].status = EINVAL;
   1760 			}
   1761 			req++;
   1762 		}
   1763 		if(req < count) {
   1764 			crypt_res[req].status = 0;
   1765 			krp = TAILQ_FIRST(&fcr->crp_ret_mkq);
   1766 			if (krp) {
   1767 				TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next);
   1768 				TAILQ_INSERT_TAIL(&krp_delfree_q, krp, krp_next);
   1769 			req++;
   1770 			}
   1771 		}
   1772 	}
   1773 	mutex_spin_exit(&crypto_mtx);
   1774 
   1775 	/* now do all the work outside the mutex */
   1776 	for(req=0; req < count ;) {
   1777 		crp = TAILQ_FIRST(&crp_delfree_q);
   1778 		if (crp) {
   1779 			if (crypt_res[req].status != 0) {
   1780 				/* csefind failed during collection */
   1781 				goto bail;
   1782 			}
   1783 			cse = (struct csession *)crp->crp_opaque;
   1784 			crypt_res[req].reqid = crp->crp_reqid;
   1785 			crypt_res[req].opaque = crp->crp_usropaque;
   1786 			completed++;
   1787 
   1788 			if (crp->crp_etype != 0) {
   1789 				crypt_res[req].status = crp->crp_etype;
   1790 				goto bail;
   1791 			}
   1792 
   1793 			if (cse->error) {
   1794 				crypt_res[req].status = cse->error;
   1795 				goto bail;
   1796 			}
   1797 
   1798 			if (crp->dst && (crypt_res[req].status =
   1799 			    copyout(crp->uio.uio_iov[0].iov_base, crp->dst,
   1800 			    crp->len)))
   1801 				goto bail;
   1802 
   1803 			if (crp->mac && (crypt_res[req].status =
   1804 			    copyout(crp->crp_mac, crp->mac,
   1805 			    cse->thash->authsize)))
   1806 				goto bail;
   1807 
   1808 bail:
   1809 			TAILQ_REMOVE(&crp_delfree_q, crp, crp_next);
   1810 			kmem_free(crp->uio.uio_iov[0].iov_base,
   1811 			    crp->uio.uio_iov[0].iov_len);
   1812 			crypto_freereq(crp);
   1813 			req++;
   1814 		}
   1815 
   1816 		if (req < count) {
   1817 			krp = TAILQ_FIRST(&krp_delfree_q);
   1818 			if (krp) {
   1819 				crypt_res[req].reqid = krp->krp_reqid;
   1820 				crypt_res[req].opaque = krp->krp_usropaque;
   1821 				completed++;
   1822 				if (krp->krp_status != 0) {
   1823 					DPRINTF(("cryptodev_key: "
   1824 					    "krp->krp_status 0x%08x\n",
   1825 					    krp->krp_status));
   1826 					crypt_res[req].status = krp->krp_status;
   1827 					goto fail;
   1828 				}
   1829 
   1830 				for (i = krp->krp_iparams; i < krp->krp_iparams
   1831 				    + krp->krp_oparams; i++) {
   1832 					size = (krp->krp_param[i].crp_nbits
   1833 					    + 7) / 8;
   1834 					if (size == 0)
   1835 						continue;
   1836 					crypt_res[req].status = copyout
   1837 					    (krp->krp_param[i].crp_p,
   1838 					    krp->crk_param[i].crp_p, size);
   1839 					if (crypt_res[req].status) {
   1840 						DPRINTF(("cryptodev_key: "
   1841 						    "copyout oparam %d failed, "
   1842 						    "error=%d\n",
   1843 						    i - krp->krp_iparams,
   1844 						    crypt_res[req].status));
   1845 						goto fail;
   1846 					}
   1847 				}
   1848 fail:
   1849 				TAILQ_REMOVE(&krp_delfree_q, krp, krp_next);
   1850 				/* not sure what to do for this */
   1851 				/* kop[req].crk_status = krp->krp_status; */
   1852 				for (i = 0; i < CRK_MAXPARAM; i++) {
   1853 					struct crparam *kp = &(krp->krp_param[i]);
   1854 					if (kp->crp_p) {
   1855 						size = (kp->crp_nbits + 7) / 8;
   1856 						KASSERT(size > 0);
   1857 						(void)memset(kp->crp_p, 0, size);
   1858 						kmem_free(kp->crp_p, size);
   1859 					}
   1860 				}
   1861 				cv_destroy(&krp->krp_cv);
   1862 				pool_put(&cryptkop_pool, krp);
   1863 				req++;
   1864 			}
   1865 		}
   1866 	}
   1867 
   1868 	return completed;
   1869 }
   1870 
   1871 static int
   1872 cryptodev_getstatus (struct fcrypt *fcr, struct crypt_result *crypt_res)
   1873 {
   1874         struct cryptop *crp = NULL, *cnext;
   1875         struct cryptkop *krp = NULL, *knext;
   1876         struct csession *cse;
   1877         int i, size, req = 0;
   1878 
   1879 	mutex_spin_enter(&crypto_mtx);
   1880 	/* Here we dont know for which request the user is requesting the
   1881 	 * response so checking in both the queues */
   1882 	TAILQ_FOREACH_SAFE(crp, &fcr->crp_ret_mq, crp_next, cnext) {
   1883 		if(crp && (crp->crp_reqid == crypt_res->reqid)) {
   1884 			cse = (struct csession *)crp->crp_opaque;
   1885 		        crypt_res->opaque = crp->crp_usropaque;
   1886 			cse = csefind(fcr, cse->ses);
   1887 			if (cse == NULL) {
   1888 				DPRINTF(("csefind failed\n"));
   1889 				crypt_res->status = EINVAL;
   1890 				goto bail;
   1891 			}
   1892 
   1893 			if (crp->crp_etype != 0) {
   1894 				crypt_res->status = crp->crp_etype;
   1895 				goto bail;
   1896 			}
   1897 
   1898 			if (cse->error) {
   1899 				crypt_res->status = cse->error;
   1900 				goto bail;
   1901 			}
   1902 
   1903 			if (crp->dst && (crypt_res->status =
   1904 			    copyout(crp->uio.uio_iov[0].iov_base,
   1905 			    crp->dst, crp->len)))
   1906 				goto bail;
   1907 
   1908 			if (crp->mac && (crypt_res->status =
   1909 			    copyout(crp->crp_mac, crp->mac,
   1910 			    cse->thash->authsize)))
   1911 				goto bail;
   1912 bail:
   1913 			TAILQ_REMOVE(&fcr->crp_ret_mq, crp, crp_next);
   1914 
   1915 			mutex_spin_exit(&crypto_mtx);
   1916 			crypto_freereq(crp);
   1917 			return 0;
   1918 		}
   1919 	}
   1920 
   1921 	TAILQ_FOREACH_SAFE(krp, &fcr->crp_ret_mkq, krp_next, knext) {
   1922 		if(krp && (krp->krp_reqid == crypt_res->reqid)) {
   1923 			crypt_res[req].opaque = krp->krp_usropaque;
   1924 			if (krp->krp_status != 0) {
   1925 				DPRINTF(("cryptodev_key: "
   1926 				    "krp->krp_status 0x%08x\n",
   1927 				    krp->krp_status));
   1928 				crypt_res[req].status = krp->krp_status;
   1929 				goto fail;
   1930 			}
   1931 
   1932 			for (i = krp->krp_iparams; i < krp->krp_iparams +
   1933 			    krp->krp_oparams; i++) {
   1934 				size = (krp->krp_param[i].crp_nbits + 7) / 8;
   1935 				if (size == 0)
   1936 					continue;
   1937 				crypt_res[req].status = copyout(
   1938 				    krp->krp_param[i].crp_p,
   1939 				    krp->crk_param[i].crp_p, size);
   1940 				if (crypt_res[req].status) {
   1941 					DPRINTF(("cryptodev_key: copyout oparam"
   1942 					    "%d failed, error=%d\n",
   1943 					    i - krp->krp_iparams,
   1944 					    crypt_res[req].status));
   1945 					goto fail;
   1946 				}
   1947 			}
   1948 fail:
   1949 			TAILQ_REMOVE(&fcr->crp_ret_mkq, krp, krp_next);
   1950 			mutex_spin_exit(&crypto_mtx);
   1951 			/* not sure what to do for this */
   1952 			/* kop[req].crk_status = krp->krp_status; */
   1953 			for (i = 0; i < CRK_MAXPARAM; i++) {
   1954 				struct crparam *kp = &(krp->krp_param[i]);
   1955 				if (kp->crp_p) {
   1956 					size = (kp->crp_nbits + 7) / 8;
   1957 					KASSERT(size > 0);
   1958 					memset(kp->crp_p, 0, size);
   1959 					kmem_free(kp->crp_p, size);
   1960 				}
   1961 			}
   1962 			cv_destroy(&krp->krp_cv);
   1963 			pool_put(&cryptkop_pool, krp);
   1964 			return 0;
   1965 		}
   1966 	}
   1967 	mutex_spin_exit(&crypto_mtx);
   1968 	return EINPROGRESS;
   1969 }
   1970 
   1971 static int
   1972 cryptof_stat(struct file *fp, struct stat *st)
   1973 {
   1974 	struct fcrypt *fcr = fp->f_data;
   1975 
   1976 	(void)memset(st, 0, sizeof(st));
   1977 
   1978 	mutex_spin_enter(&crypto_mtx);
   1979 	st->st_dev = makedev(cdevsw_lookup_major(&crypto_cdevsw), fcr->sesn);
   1980 	st->st_atimespec = fcr->atime;
   1981 	st->st_mtimespec = fcr->mtime;
   1982 	st->st_ctimespec = st->st_birthtimespec = fcr->btime;
   1983 	st->st_uid = kauth_cred_geteuid(fp->f_cred);
   1984 	st->st_gid = kauth_cred_getegid(fp->f_cred);
   1985 	mutex_spin_exit(&crypto_mtx);
   1986 
   1987 	return 0;
   1988 }
   1989 
   1990 static int
   1991 cryptof_poll(struct file *fp, int events)
   1992 {
   1993 	struct fcrypt *fcr = (struct fcrypt *)fp->f_data;
   1994 	int revents = 0;
   1995 
   1996 	if (!(events & (POLLIN | POLLRDNORM))) {
   1997 		/* only support read and POLLIN */
   1998 		return 0;
   1999 	}
   2000 
   2001 	mutex_spin_enter(&crypto_mtx);
   2002 	if (TAILQ_EMPTY(&fcr->crp_ret_mq) && TAILQ_EMPTY(&fcr->crp_ret_mkq)) {
   2003 		/* no completed requests pending, save the poll for later */
   2004 		selrecord(curlwp, &fcr->sinfo);
   2005 	} else {
   2006 		/* let the app(s) know that there are completed requests */
   2007 		revents = events & (POLLIN | POLLRDNORM);
   2008 	}
   2009 	mutex_spin_exit(&crypto_mtx);
   2010 
   2011 	return revents;
   2012 }
   2013 
   2014 /*
   2015  * Pseudo-device initialization routine for /dev/crypto
   2016  */
   2017 void	cryptoattach(int);
   2018 
   2019 void
   2020 cryptoattach(int num)
   2021 {
   2022 	pool_init(&fcrpl, sizeof(struct fcrypt), 0, 0, 0, "fcrpl",
   2023 	    NULL, IPL_NET);	/* XXX IPL_NET ("splcrypto") */
   2024 	pool_init(&csepl, sizeof(struct csession), 0, 0, 0, "csepl",
   2025 	    NULL, IPL_NET);	/* XXX IPL_NET ("splcrypto") */
   2026 
   2027 	/*
   2028 	 * Preallocate space for 64 users, with 5 sessions each.
   2029 	 * (consider that a TLS protocol session requires at least
   2030 	 * 3DES, MD5, and SHA1 (both hashes are used in the PRF) for
   2031 	 * the negotiation, plus HMAC_SHA1 for the actual SSL records,
   2032 	 * consuming one session here for each algorithm.
   2033 	 */
   2034 	pool_prime(&fcrpl, 64);
   2035 	pool_prime(&csepl, 64 * 5);
   2036 }
   2037