Home | History | Annotate | Line # | Download | only in netipsec
xform_esp.c revision 1.73
      1 /*	$NetBSD: xform_esp.c,v 1.73 2018/01/24 13:54:16 maxv Exp $	*/
      2 /*	$FreeBSD: src/sys/netipsec/xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $	*/
      3 /*	$OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */
      4 
      5 /*
      6  * The authors of this code are John Ioannidis (ji (at) tla.org),
      7  * Angelos D. Keromytis (kermit (at) csd.uch.gr) and
      8  * Niels Provos (provos (at) physnet.uni-hamburg.de).
      9  *
     10  * The original version of this code was written by John Ioannidis
     11  * for BSD/OS in Athens, Greece, in November 1995.
     12  *
     13  * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
     14  * by Angelos D. Keromytis.
     15  *
     16  * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
     17  * and Niels Provos.
     18  *
     19  * Additional features in 1999 by Angelos D. Keromytis.
     20  *
     21  * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
     22  * Angelos D. Keromytis and Niels Provos.
     23  * Copyright (c) 2001 Angelos D. Keromytis.
     24  *
     25  * Permission to use, copy, and modify this software with or without fee
     26  * is hereby granted, provided that this entire notice is included in
     27  * all copies of any software which is or includes a copy or
     28  * modification of this software.
     29  * You may use this code under the GNU public license if you so wish. Please
     30  * contribute changes back to the authors under this freer than GPL license
     31  * so that we may further the use of strong encryption without limitations to
     32  * all.
     33  *
     34  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
     35  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
     36  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
     37  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
     38  * PURPOSE.
     39  */
     40 
     41 #include <sys/cdefs.h>
     42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.73 2018/01/24 13:54:16 maxv Exp $");
     43 
     44 #if defined(_KERNEL_OPT)
     45 #include "opt_inet.h"
     46 #include "opt_ipsec.h"
     47 #endif
     48 
     49 #include <sys/param.h>
     50 #include <sys/systm.h>
     51 #include <sys/mbuf.h>
     52 #include <sys/socket.h>
     53 #include <sys/syslog.h>
     54 #include <sys/kernel.h>
     55 #include <sys/sysctl.h>
     56 #include <sys/cprng.h>
     57 #include <sys/pool.h>
     58 #include <sys/pserialize.h>
     59 
     60 #include <net/if.h>
     61 
     62 #include <netinet/in.h>
     63 #include <netinet/in_systm.h>
     64 #include <netinet/ip.h>
     65 #include <netinet/ip_ecn.h>
     66 #include <netinet/ip6.h>
     67 
     68 #include <net/route.h>
     69 #include <netipsec/ipsec.h>
     70 #include <netipsec/ipsec_private.h>
     71 #include <netipsec/ah.h>
     72 #include <netipsec/ah_var.h>
     73 #include <netipsec/esp.h>
     74 #include <netipsec/esp_var.h>
     75 #include <netipsec/xform.h>
     76 
     77 #ifdef INET6
     78 #include <netinet6/ip6_var.h>
     79 #include <netipsec/ipsec6.h>
     80 #endif
     81 
     82 #include <netipsec/key.h>
     83 #include <netipsec/key_debug.h>
     84 
     85 #include <opencrypto/cryptodev.h>
     86 #include <opencrypto/xform.h>
     87 
     88 percpu_t *espstat_percpu;
     89 
     90 int	esp_enable = 1;
     91 
     92 #ifdef __FreeBSD__
     93 SYSCTL_DECL(_net_inet_esp);
     94 SYSCTL_INT(_net_inet_esp, OID_AUTO,
     95 	esp_enable,	CTLFLAG_RW,	&esp_enable,	0, "");
     96 SYSCTL_STRUCT(_net_inet_esp, IPSECCTL_STATS,
     97 	stats,		CTLFLAG_RD,	&espstat,	espstat, "");
     98 #endif /* __FreeBSD__ */
     99 
    100 static	int esp_max_ivlen;		/* max iv length over all algorithms */
    101 
    102 static int esp_input_cb(struct cryptop *op);
    103 static int esp_output_cb(struct cryptop *crp);
    104 
    105 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT };
    106 
    107 static pool_cache_t esp_tdb_crypto_pool_cache;
    108 static size_t esp_pool_item_size;
    109 
    110 /*
    111  * NB: this is public for use by the PF_KEY support.
    112  * NB: if you add support here; be sure to add code to esp_attach below!
    113  */
    114 const struct enc_xform *
    115 esp_algorithm_lookup(int alg)
    116 {
    117 
    118 	switch (alg) {
    119 	case SADB_EALG_DESCBC:
    120 		return &enc_xform_des;
    121 	case SADB_EALG_3DESCBC:
    122 		return &enc_xform_3des;
    123 	case SADB_X_EALG_AES:
    124 		return &enc_xform_rijndael128;
    125 	case SADB_X_EALG_BLOWFISHCBC:
    126 		return &enc_xform_blf;
    127 	case SADB_X_EALG_CAST128CBC:
    128 		return &enc_xform_cast5;
    129 	case SADB_X_EALG_SKIPJACK:
    130 		return &enc_xform_skipjack;
    131 	case SADB_X_EALG_CAMELLIACBC:
    132 		return &enc_xform_camellia;
    133 	case SADB_X_EALG_AESCTR:
    134 		return &enc_xform_aes_ctr;
    135 	case SADB_X_EALG_AESGCM16:
    136 		return &enc_xform_aes_gcm;
    137 	case SADB_X_EALG_AESGMAC:
    138 		return &enc_xform_aes_gmac;
    139 	case SADB_EALG_NULL:
    140 		return &enc_xform_null;
    141 	}
    142 	return NULL;
    143 }
    144 
    145 size_t
    146 esp_hdrsiz(const struct secasvar *sav)
    147 {
    148 	size_t size;
    149 
    150 	if (sav != NULL) {
    151 		/*XXX not right for null algorithm--does it matter??*/
    152 		KASSERT(sav->tdb_encalgxform != NULL);
    153 		if (sav->flags & SADB_X_EXT_OLD)
    154 			size = sizeof(struct esp);
    155 		else
    156 			size = sizeof(struct newesp);
    157 		size += sav->tdb_encalgxform->ivsize + 9;
    158 		/*XXX need alg check???*/
    159 		if (sav->tdb_authalgxform != NULL && sav->replay)
    160 			size += ah_hdrsiz(sav);
    161 	} else {
    162 		/*
    163 		 *   base header size
    164 		 * + max iv length for CBC mode
    165 		 * + max pad length
    166 		 * + sizeof(pad length field)
    167 		 * + sizeof(next header field)
    168 		 * + max icv supported.
    169 		 */
    170 		size = sizeof(struct newesp) + esp_max_ivlen + 9 +
    171 		    ah_hdrsiz(NULL);
    172 	}
    173 	return size;
    174 }
    175 
    176 /*
    177  * esp_init() is called when an SPI is being set up.
    178  */
    179 static int
    180 esp_init(struct secasvar *sav, const struct xformsw *xsp)
    181 {
    182 	const struct enc_xform *txform;
    183 	struct cryptoini cria, crie, *cr;
    184 	int keylen;
    185 	int error;
    186 
    187 	txform = esp_algorithm_lookup(sav->alg_enc);
    188 	if (txform == NULL) {
    189 		DPRINTF(("%s: unsupported encryption algorithm %d\n", __func__,
    190 		    sav->alg_enc));
    191 		return EINVAL;
    192 	}
    193 	if (sav->key_enc == NULL) {
    194 		DPRINTF(("%s: no encoding key for %s algorithm\n", __func__,
    195 		    txform->name));
    196 		return EINVAL;
    197 	}
    198 	if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) {
    199 		DPRINTF(("%s: 4-byte IV not supported with protocol\n",
    200 		    __func__));
    201 		return EINVAL;
    202 	}
    203 	keylen = _KEYLEN(sav->key_enc);
    204 	if (txform->minkey > keylen || keylen > txform->maxkey) {
    205 		DPRINTF(("%s: invalid key length %u, must be in "
    206 		    "the range [%u..%u] for algorithm %s\n", __func__,
    207 		    keylen, txform->minkey, txform->maxkey, txform->name));
    208 		return EINVAL;
    209 	}
    210 
    211 	sav->ivlen = txform->ivsize;
    212 
    213 	/*
    214 	 * Setup AH-related state.
    215 	 */
    216 	if (sav->alg_auth != 0) {
    217 		error = ah_init0(sav, xsp, &cria);
    218 		if (error)
    219 			return error;
    220 	}
    221 
    222 	/* NB: override anything set in ah_init0 */
    223 	sav->tdb_xform = xsp;
    224 	sav->tdb_encalgxform = txform;
    225 
    226 	switch (sav->alg_enc) {
    227 	case SADB_X_EALG_AESGCM16:
    228 	case SADB_X_EALG_AESGMAC:
    229 		switch (keylen) {
    230 		case 20:
    231 			sav->alg_auth = SADB_X_AALG_AES128GMAC;
    232 			sav->tdb_authalgxform = &auth_hash_gmac_aes_128;
    233 			break;
    234 		case 28:
    235 			sav->alg_auth = SADB_X_AALG_AES192GMAC;
    236 			sav->tdb_authalgxform = &auth_hash_gmac_aes_192;
    237 			break;
    238 		case 36:
    239 			sav->alg_auth = SADB_X_AALG_AES256GMAC;
    240 			sav->tdb_authalgxform = &auth_hash_gmac_aes_256;
    241 			break;
    242 		default:
    243 			DPRINTF(("%s: invalid key length %u, must be either of "
    244 				"20, 28 or 36\n", __func__, keylen));
    245 			return EINVAL;
    246                 }
    247 
    248 		memset(&cria, 0, sizeof(cria));
    249 		cria.cri_alg = sav->tdb_authalgxform->type;
    250 		cria.cri_klen = _KEYBITS(sav->key_enc);
    251 		cria.cri_key = _KEYBUF(sav->key_enc);
    252 		break;
    253 	default:
    254 		break;
    255 	}
    256 
    257 	/* Initialize crypto session. */
    258 	memset(&crie, 0, sizeof(crie));
    259 	crie.cri_alg = sav->tdb_encalgxform->type;
    260 	crie.cri_klen = _KEYBITS(sav->key_enc);
    261 	crie.cri_key = _KEYBUF(sav->key_enc);
    262 	/* XXX Rounds ? */
    263 
    264 	if (sav->tdb_authalgxform && sav->tdb_encalgxform) {
    265 		/* init both auth & enc */
    266 		crie.cri_next = &cria;
    267 		cr = &crie;
    268 	} else if (sav->tdb_encalgxform) {
    269 		cr = &crie;
    270 	} else if (sav->tdb_authalgxform) {
    271 		cr = &cria;
    272 	} else {
    273 		/* XXX cannot happen? */
    274 		DPRINTF(("%s: no encoding OR authentication xform!\n",
    275 		    __func__));
    276 		return EINVAL;
    277 	}
    278 
    279 	return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support);
    280 }
    281 
    282 /*
    283  * Paranoia.
    284  */
    285 static int
    286 esp_zeroize(struct secasvar *sav)
    287 {
    288 	/* NB: ah_zerorize free's the crypto session state */
    289 	int error = ah_zeroize(sav);
    290 
    291 	if (sav->key_enc) {
    292 		explicit_memset(_KEYBUF(sav->key_enc), 0,
    293 		    _KEYLEN(sav->key_enc));
    294 	}
    295 	sav->tdb_encalgxform = NULL;
    296 	sav->tdb_xform = NULL;
    297 	return error;
    298 }
    299 
    300 /*
    301  * ESP input processing, called (eventually) through the protocol switch.
    302  */
    303 static int
    304 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
    305 {
    306 	const struct auth_hash *esph;
    307 	const struct enc_xform *espx;
    308 	struct tdb_crypto *tc;
    309 	int plen, alen, hlen, error;
    310 	struct newesp *esp;
    311 
    312 	struct cryptodesc *crde;
    313 	struct cryptop *crp;
    314 
    315 	IPSEC_SPLASSERT_SOFTNET(__func__);
    316 
    317 	KASSERT(sav != NULL);
    318 	KASSERT(sav->tdb_encalgxform != NULL);
    319 	KASSERTMSG((skip&3) == 0 && (m->m_pkthdr.len&3) == 0,
    320 	    "misaligned packet, skip %u pkt len %u",
    321 	    skip, m->m_pkthdr.len);
    322 
    323 	/* XXX don't pullup, just copy header */
    324 	IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof(struct newesp));
    325 	if (esp == NULL) {
    326 		/* m already freed */
    327 		return EINVAL;
    328 	}
    329 
    330 	esph = sav->tdb_authalgxform;
    331 	espx = sav->tdb_encalgxform;
    332 
    333 	/* Determine the ESP header length */
    334 	if (sav->flags & SADB_X_EXT_OLD)
    335 		hlen = sizeof(struct esp) + sav->ivlen;
    336 	else
    337 		hlen = sizeof(struct newesp) + sav->ivlen;
    338 	/* Authenticator hash size */
    339 	alen = esph ? esph->authsize : 0;
    340 
    341 	/*
    342 	 * Verify payload length is multiple of encryption algorithm
    343 	 * block size.
    344 	 *
    345 	 * NB: This works for the null algorithm because the blocksize
    346 	 *     is 4 and all packets must be 4-byte aligned regardless
    347 	 *     of the algorithm.
    348 	 */
    349 	plen = m->m_pkthdr.len - (skip + hlen + alen);
    350 	if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
    351 		char buf[IPSEC_ADDRSTRLEN];
    352 		DPRINTF(("%s: payload of %d octets not a multiple of %d octets,"
    353 		    "  SA %s/%08lx\n", __func__, plen, espx->blocksize,
    354 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
    355 		    (u_long) ntohl(sav->spi)));
    356 		ESP_STATINC(ESP_STAT_BADILEN);
    357 		m_freem(m);
    358 		return EINVAL;
    359 	}
    360 
    361 	/*
    362 	 * Check sequence number.
    363 	 */
    364 	if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
    365 		char logbuf[IPSEC_LOGSASTRLEN];
    366 		DPRINTF(("%s: packet replay check for %s\n", __func__,
    367 		    ipsec_logsastr(sav, logbuf, sizeof(logbuf))));	/*XXX*/
    368 		ESP_STATINC(ESP_STAT_REPLAY);
    369 		m_freem(m);
    370 		return ENOBUFS;		/*XXX*/
    371 	}
    372 
    373 	/* Update the counters */
    374 	ESP_STATADD(ESP_STAT_IBYTES, m->m_pkthdr.len - skip - hlen - alen);
    375 
    376 	/* Get crypto descriptors */
    377 	crp = crypto_getreq(esph && espx ? 2 : 1);
    378 	if (crp == NULL) {
    379 		DPRINTF(("%s: failed to acquire crypto descriptors\n",
    380 		    __func__));
    381 		error = ENOBUFS;
    382 		goto out;
    383 	}
    384 
    385 	/* Get IPsec-specific opaque pointer */
    386 	size_t extra __diagused = esph == NULL ? 0 : alen;
    387 	KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size,
    388 	    "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n",
    389 	    sizeof(*tc) + extra, esp_pool_item_size);
    390 	tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
    391 	if (tc == NULL) {
    392 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
    393 		error = ENOBUFS;
    394 		goto out1;
    395 	}
    396 
    397 	error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT);
    398 	if (error) {
    399 		DPRINTF(("%s: m_makewritable failed\n", __func__));
    400 		goto out2;
    401 	}
    402 
    403 	if (esph) {
    404 		struct cryptodesc *crda;
    405 
    406 		KASSERT(crp->crp_desc != NULL);
    407 		crda = crp->crp_desc;
    408 
    409 		/* Authentication descriptor */
    410 		crda->crd_skip = skip;
    411 		if (espx && espx->type == CRYPTO_AES_GCM_16)
    412 			crda->crd_len = hlen - sav->ivlen;
    413 		else
    414 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
    415 		crda->crd_inject = m->m_pkthdr.len - alen;
    416 
    417 		crda->crd_alg = esph->type;
    418 		if (espx && (espx->type == CRYPTO_AES_GCM_16 ||
    419 			     espx->type == CRYPTO_AES_GMAC)) {
    420 			crda->crd_key = _KEYBUF(sav->key_enc);
    421 			crda->crd_klen = _KEYBITS(sav->key_enc);
    422 		} else {
    423 			crda->crd_key = _KEYBUF(sav->key_auth);
    424 			crda->crd_klen = _KEYBITS(sav->key_auth);
    425 		}
    426 
    427 		/* Copy the authenticator */
    428 		m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1));
    429 
    430 		/* Chain authentication request */
    431 		crde = crda->crd_next;
    432 	} else {
    433 		crde = crp->crp_desc;
    434 	}
    435 
    436     {
    437 	int s = pserialize_read_enter();
    438 
    439 	/*
    440 	 * Take another reference to the SA for opencrypto callback.
    441 	 */
    442 	if (__predict_false(sav->state == SADB_SASTATE_DEAD)) {
    443 		pserialize_read_exit(s);
    444 		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    445 		crypto_freereq(crp);
    446 		ESP_STATINC(ESP_STAT_NOTDB);
    447 		return ENOENT;
    448 	}
    449 	KEY_SA_REF(sav);
    450 	pserialize_read_exit(s);
    451     }
    452 
    453 	/* Crypto operation descriptor */
    454 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
    455 	crp->crp_flags = CRYPTO_F_IMBUF;
    456 	crp->crp_buf = m;
    457 	crp->crp_callback = esp_input_cb;
    458 	crp->crp_sid = sav->tdb_cryptoid;
    459 	crp->crp_opaque = tc;
    460 
    461 	/* These are passed as-is to the callback */
    462 	tc->tc_spi = sav->spi;
    463 	tc->tc_dst = sav->sah->saidx.dst;
    464 	tc->tc_proto = sav->sah->saidx.proto;
    465 	tc->tc_protoff = protoff;
    466 	tc->tc_skip = skip;
    467 	tc->tc_sav = sav;
    468 
    469 	/* Decryption descriptor */
    470 	if (espx) {
    471 		KASSERTMSG(crde != NULL, "null esp crypto descriptor");
    472 		crde->crd_skip = skip + hlen;
    473 		if (espx->type == CRYPTO_AES_GMAC)
    474 			crde->crd_len = 0;
    475 		else
    476 			crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
    477 		crde->crd_inject = skip + hlen - sav->ivlen;
    478 
    479 		crde->crd_alg = espx->type;
    480 		crde->crd_key = _KEYBUF(sav->key_enc);
    481 		crde->crd_klen = _KEYBITS(sav->key_enc);
    482 		/* XXX Rounds ? */
    483 	}
    484 
    485 	return crypto_dispatch(crp);
    486 
    487 out2:
    488 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    489 out1:
    490 	crypto_freereq(crp);
    491 out:
    492 	ESP_STATINC(ESP_STAT_CRYPTO);
    493 	m_freem(m);
    494 	return error;
    495 }
    496 
    497 #ifdef INET6
    498 #define	IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do {		     \
    499 	if (saidx->dst.sa.sa_family == AF_INET6) {			     \
    500 		error = ipsec6_common_input_cb(m, sav, skip, protoff);	     \
    501 	} else {							     \
    502 		error = ipsec4_common_input_cb(m, sav, skip, protoff);	     \
    503 	}								     \
    504 } while (0)
    505 #else
    506 #define	IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff)			     \
    507 	(error = ipsec4_common_input_cb(m, sav, skip, protoff))
    508 #endif
    509 
    510 /*
    511  * ESP input callback from the crypto driver.
    512  */
    513 static int
    514 esp_input_cb(struct cryptop *crp)
    515 {
    516 	char buf[IPSEC_ADDRSTRLEN];
    517 	uint8_t lastthree[3], aalg[AH_ALEN_MAX];
    518 	int hlen, skip, protoff, error;
    519 	struct mbuf *m;
    520 	const struct auth_hash *esph;
    521 	struct tdb_crypto *tc;
    522 	struct secasvar *sav;
    523 	struct secasindex *saidx;
    524 	void *ptr;
    525 	uint16_t dport;
    526 	uint16_t sport;
    527 	IPSEC_DECLARE_LOCK_VARIABLE;
    528 
    529 	KASSERT(crp->crp_desc != NULL);
    530 	KASSERT(crp->crp_opaque != NULL);
    531 
    532 	tc = crp->crp_opaque;
    533 	skip = tc->tc_skip;
    534 	protoff = tc->tc_protoff;
    535 	m = crp->crp_buf;
    536 
    537 	/* find the source port for NAT-T */
    538 	nat_t_ports_get(m, &dport, &sport);
    539 
    540 	IPSEC_ACQUIRE_GLOBAL_LOCKS();
    541 
    542 	sav = tc->tc_sav;
    543 	if (__predict_false(!SADB_SASTATE_USABLE_P(sav))) {
    544 		KEY_SA_UNREF(&sav);
    545 		sav = KEY_LOOKUP_SA(&tc->tc_dst, tc->tc_proto, tc->tc_spi,
    546 		    sport, dport);
    547 		if (sav == NULL) {
    548 			ESP_STATINC(ESP_STAT_NOTDB);
    549 			DPRINTF(("%s: SA expired while in crypto "
    550 			    "(SA %s/%08lx proto %u)\n", __func__,
    551 			    ipsec_address(&tc->tc_dst, buf, sizeof(buf)),
    552 			    (u_long) ntohl(tc->tc_spi), tc->tc_proto));
    553 			error = ENOBUFS;		/*XXX*/
    554 			goto bad;
    555 		}
    556 	}
    557 
    558 	saidx = &sav->sah->saidx;
    559 	KASSERTMSG(saidx->dst.sa.sa_family == AF_INET ||
    560 	    saidx->dst.sa.sa_family == AF_INET6,
    561 	    "unexpected protocol family %u", saidx->dst.sa.sa_family);
    562 
    563 	esph = sav->tdb_authalgxform;
    564 
    565 	/* Check for crypto errors */
    566 	if (crp->crp_etype) {
    567 		/* Reset the session ID */
    568 		if (sav->tdb_cryptoid != 0)
    569 			sav->tdb_cryptoid = crp->crp_sid;
    570 
    571 		if (crp->crp_etype == EAGAIN) {
    572 			KEY_SA_UNREF(&sav);
    573 			IPSEC_RELEASE_GLOBAL_LOCKS();
    574 			return crypto_dispatch(crp);
    575 		}
    576 
    577 		ESP_STATINC(ESP_STAT_NOXFORM);
    578 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
    579 		error = crp->crp_etype;
    580 		goto bad;
    581 	}
    582 
    583 	ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]);
    584 
    585 	/* If authentication was performed, check now. */
    586 	if (esph != NULL) {
    587 		/*
    588 		 * If we have a tag, it means an IPsec-aware NIC did
    589 		 * the verification for us.  Otherwise we need to
    590 		 * check the authentication calculation.
    591 		 */
    592 		AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
    593 		/* Copy the authenticator from the packet */
    594 		m_copydata(m, m->m_pkthdr.len - esph->authsize,
    595 			esph->authsize, aalg);
    596 
    597 		ptr = (tc + 1);
    598 
    599 		/* Verify authenticator */
    600 		if (!consttime_memequal(ptr, aalg, esph->authsize)) {
    601 			DPRINTF(("%s: authentication hash mismatch "
    602 			    "for packet in SA %s/%08lx\n", __func__,
    603 			    ipsec_address(&saidx->dst, buf,
    604 			    sizeof(buf)), (u_long) ntohl(sav->spi)));
    605 			ESP_STATINC(ESP_STAT_BADAUTH);
    606 			error = EACCES;
    607 			goto bad;
    608 		}
    609 
    610 		/* Remove trailing authenticator */
    611 		m_adj(m, -(esph->authsize));
    612 	}
    613 
    614 	/* Release the crypto descriptors */
    615 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    616 	tc = NULL;
    617 	crypto_freereq(crp), crp = NULL;
    618 
    619 	/*
    620 	 * Packet is now decrypted.
    621 	 */
    622 	m->m_flags |= M_DECRYPTED;
    623 
    624 	/*
    625 	 * Update replay sequence number, if appropriate.
    626 	 */
    627 	if (sav->replay) {
    628 		uint32_t seq;
    629 
    630 		m_copydata(m, skip + offsetof(struct newesp, esp_seq),
    631 		    sizeof(seq), &seq);
    632 		if (ipsec_updatereplay(ntohl(seq), sav)) {
    633 			char logbuf[IPSEC_LOGSASTRLEN];
    634 			DPRINTF(("%s: packet replay check for %s\n", __func__,
    635 			    ipsec_logsastr(sav, logbuf, sizeof(logbuf))));
    636 			ESP_STATINC(ESP_STAT_REPLAY);
    637 			error = ENOBUFS;
    638 			goto bad;
    639 		}
    640 	}
    641 
    642 	/* Determine the ESP header length */
    643 	if (sav->flags & SADB_X_EXT_OLD)
    644 		hlen = sizeof(struct esp) + sav->ivlen;
    645 	else
    646 		hlen = sizeof(struct newesp) + sav->ivlen;
    647 
    648 	/* Remove the ESP header and IV from the mbuf. */
    649 	error = m_striphdr(m, skip, hlen);
    650 	if (error) {
    651 		ESP_STATINC(ESP_STAT_HDROPS);
    652 		DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__,
    653 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
    654 		    (u_long) ntohl(sav->spi)));
    655 		goto bad;
    656 	}
    657 
    658 	/* Save the last three bytes of decrypted data */
    659 	m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree);
    660 
    661 	/* Verify pad length */
    662 	if (lastthree[1] + 2 > m->m_pkthdr.len - skip) {
    663 		ESP_STATINC(ESP_STAT_BADILEN);
    664 		DPRINTF(("%s: invalid padding length %d "
    665 		    "for %u byte packet in SA %s/%08lx\n", __func__,
    666 		    lastthree[1], m->m_pkthdr.len - skip,
    667 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
    668 		    (u_long) ntohl(sav->spi)));
    669 		error = EINVAL;
    670 		goto bad;
    671 	}
    672 
    673 	/* Verify correct decryption by checking the last padding bytes */
    674 	if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) {
    675 		if (lastthree[1] != lastthree[0] && lastthree[1] != 0) {
    676 			ESP_STATINC(ESP_STAT_BADENC);
    677 			DPRINTF(("%s: decryption failed for packet in SA "
    678 			    "%s/%08lx\n", __func__,
    679 			    ipsec_address(&sav->sah->saidx.dst, buf,
    680 			    sizeof(buf)), (u_long) ntohl(sav->spi)));
    681 			DPRINTF(("%s: %x %x\n", __func__, lastthree[0],
    682 			    lastthree[1]));
    683 			error = EINVAL;
    684 			goto bad;
    685 		}
    686 	}
    687 
    688 	/* Trim the mbuf chain to remove trailing authenticator and padding */
    689 	m_adj(m, -(lastthree[1] + 2));
    690 
    691 	/* Restore the Next Protocol field */
    692 	m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2);
    693 
    694 	IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff);
    695 
    696 	KEY_SA_UNREF(&sav);
    697 	IPSEC_RELEASE_GLOBAL_LOCKS();
    698 	return error;
    699 bad:
    700 	if (sav)
    701 		KEY_SA_UNREF(&sav);
    702 	IPSEC_RELEASE_GLOBAL_LOCKS();
    703 	if (m != NULL)
    704 		m_freem(m);
    705 	if (tc != NULL)
    706 		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    707 	if (crp != NULL)
    708 		crypto_freereq(crp);
    709 	return error;
    710 }
    711 
    712 /*
    713  * ESP output routine, called by ipsec[46]_process_packet().
    714  */
    715 static int
    716 esp_output(
    717     struct mbuf *m,
    718     const struct ipsecrequest *isr,
    719     struct secasvar *sav,
    720     struct mbuf **mp,
    721     int skip,
    722     int protoff
    723 )
    724 {
    725 	char buf[IPSEC_ADDRSTRLEN];
    726 	const struct enc_xform *espx;
    727 	const struct auth_hash *esph;
    728 	int hlen, rlen, padding, blks, alen, i, roff;
    729 	struct mbuf *mo = NULL;
    730 	struct tdb_crypto *tc;
    731 	struct secasindex *saidx;
    732 	unsigned char *pad;
    733 	uint8_t prot;
    734 	int error, maxpacketsize;
    735 
    736 	struct cryptodesc *crde = NULL, *crda = NULL;
    737 	struct cryptop *crp;
    738 
    739 	IPSEC_SPLASSERT_SOFTNET(__func__);
    740 
    741 	esph = sav->tdb_authalgxform;
    742 	KASSERT(sav->tdb_encalgxform != NULL);
    743 	espx = sav->tdb_encalgxform;
    744 
    745 	if (sav->flags & SADB_X_EXT_OLD)
    746 		hlen = sizeof(struct esp) + sav->ivlen;
    747 	else
    748 		hlen = sizeof(struct newesp) + sav->ivlen;
    749 
    750 	rlen = m->m_pkthdr.len - skip;	/* Raw payload length. */
    751 	/*
    752 	 * NB: The null encoding transform has a blocksize of 4
    753 	 *     so that headers are properly aligned.
    754 	 */
    755 	blks = espx->blocksize;		/* IV blocksize */
    756 
    757 	/* XXX clamp padding length a la KAME??? */
    758 	padding = ((blks - ((rlen + 2) % blks)) % blks) + 2;
    759 
    760 	if (esph)
    761 		alen = esph->authsize;
    762 	else
    763 		alen = 0;
    764 
    765 	ESP_STATINC(ESP_STAT_OUTPUT);
    766 
    767 	saidx = &sav->sah->saidx;
    768 	/* Check for maximum packet size violations. */
    769 	switch (saidx->dst.sa.sa_family) {
    770 #ifdef INET
    771 	case AF_INET:
    772 		maxpacketsize = IP_MAXPACKET;
    773 		break;
    774 #endif /* INET */
    775 #ifdef INET6
    776 	case AF_INET6:
    777 		maxpacketsize = IPV6_MAXPACKET;
    778 		break;
    779 #endif /* INET6 */
    780 	default:
    781 		DPRINTF(("%s: unknown/unsupported protocol family %d, "
    782 		    "SA %s/%08lx\n", __func__, saidx->dst.sa.sa_family,
    783 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    784 		    (u_long)ntohl(sav->spi)));
    785 		ESP_STATINC(ESP_STAT_NOPF);
    786 		error = EPFNOSUPPORT;
    787 		goto bad;
    788 	}
    789 	if (skip + hlen + rlen + padding + alen > maxpacketsize) {
    790 		DPRINTF(("%s: packet in SA %s/%08lx got too big (len %u, "
    791 		    "max len %u)\n", __func__,
    792 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    793 		    (u_long) ntohl(sav->spi),
    794 		    skip + hlen + rlen + padding + alen, maxpacketsize));
    795 		ESP_STATINC(ESP_STAT_TOOBIG);
    796 		error = EMSGSIZE;
    797 		goto bad;
    798 	}
    799 
    800 	/* Update the counters. */
    801 	ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip);
    802 
    803 	m = m_clone(m);
    804 	if (m == NULL) {
    805 		DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__,
    806 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    807 		    (u_long) ntohl(sav->spi)));
    808 		ESP_STATINC(ESP_STAT_HDROPS);
    809 		error = ENOBUFS;
    810 		goto bad;
    811 	}
    812 
    813 	/* Inject ESP header. */
    814 	mo = m_makespace(m, skip, hlen, &roff);
    815 	if (mo == NULL) {
    816 		DPRINTF(("%s: failed to inject %u byte ESP hdr for SA "
    817 		    "%s/%08lx\n", __func__, hlen,
    818 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    819 		    (u_long) ntohl(sav->spi)));
    820 		ESP_STATINC(ESP_STAT_HDROPS);	/* XXX diffs from openbsd */
    821 		error = ENOBUFS;
    822 		goto bad;
    823 	}
    824 
    825 	/* Initialize ESP header. */
    826 	memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t));
    827 	if (sav->replay) {
    828 		uint32_t replay;
    829 
    830 #ifdef IPSEC_DEBUG
    831 		/* Emulate replay attack when ipsec_replay is TRUE. */
    832 		if (!ipsec_replay)
    833 #endif
    834 			sav->replay->count++;
    835 
    836 		replay = htonl(sav->replay->count);
    837 		memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay,
    838 		    sizeof(uint32_t));
    839 	}
    840 
    841 	/*
    842 	 * Add padding -- better to do it ourselves than use the crypto engine,
    843 	 * although if/when we support compression, we'd have to do that.
    844 	 */
    845 	pad = m_pad(m, padding + alen);
    846 	if (pad == NULL) {
    847 		DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__,
    848 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    849 		    (u_long) ntohl(sav->spi)));
    850 		m = NULL;		/* NB: free'd by m_pad */
    851 		error = ENOBUFS;
    852 		goto bad;
    853 	}
    854 
    855 	/*
    856 	 * Add padding: random, zero, or self-describing.
    857 	 * XXX catch unexpected setting
    858 	 */
    859 	switch (sav->flags & SADB_X_EXT_PMASK) {
    860 	case SADB_X_EXT_PRAND:
    861 		(void) cprng_fast(pad, padding - 2);
    862 		break;
    863 	case SADB_X_EXT_PZERO:
    864 		memset(pad, 0, padding - 2);
    865 		break;
    866 	case SADB_X_EXT_PSEQ:
    867 		for (i = 0; i < padding - 2; i++)
    868 			pad[i] = i+1;
    869 		break;
    870 	}
    871 
    872 	/* Fix padding length and Next Protocol in padding itself. */
    873 	pad[padding - 2] = padding - 2;
    874 	m_copydata(m, protoff, sizeof(uint8_t), pad + padding - 1);
    875 
    876 	/* Fix Next Protocol in IPv4/IPv6 header. */
    877 	prot = IPPROTO_ESP;
    878 	m_copyback(m, protoff, sizeof(uint8_t), &prot);
    879 
    880 	/* Get crypto descriptors. */
    881 	crp = crypto_getreq(esph && espx ? 2 : 1);
    882 	if (crp == NULL) {
    883 		DPRINTF(("%s: failed to acquire crypto descriptors\n",
    884 		    __func__));
    885 		ESP_STATINC(ESP_STAT_CRYPTO);
    886 		error = ENOBUFS;
    887 		goto bad;
    888 	}
    889 
    890 	if (espx) {
    891 		crde = crp->crp_desc;
    892 		crda = crde->crd_next;
    893 
    894 		/* Encryption descriptor. */
    895 		crde->crd_skip = skip + hlen;
    896 		if (espx->type == CRYPTO_AES_GMAC)
    897 			crde->crd_len = 0;
    898 		else
    899 			crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
    900 		crde->crd_flags = CRD_F_ENCRYPT;
    901 		crde->crd_inject = skip + hlen - sav->ivlen;
    902 
    903 		/* Encryption operation. */
    904 		crde->crd_alg = espx->type;
    905 		crde->crd_key = _KEYBUF(sav->key_enc);
    906 		crde->crd_klen = _KEYBITS(sav->key_enc);
    907 		/* XXX Rounds ? */
    908 	} else
    909 		crda = crp->crp_desc;
    910 
    911 	/* IPsec-specific opaque crypto info. */
    912 	tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
    913 	if (tc == NULL) {
    914 		crypto_freereq(crp);
    915 		DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
    916 		ESP_STATINC(ESP_STAT_CRYPTO);
    917 		error = ENOBUFS;
    918 		goto bad;
    919 	}
    920 
    921     {
    922 	int s = pserialize_read_enter();
    923 
    924 	/*
    925 	 * Take another reference to the SP and the SA for opencrypto callback.
    926 	 */
    927 	if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD ||
    928 	    sav->state == SADB_SASTATE_DEAD)) {
    929 		pserialize_read_exit(s);
    930 		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    931 		crypto_freereq(crp);
    932 		ESP_STATINC(ESP_STAT_NOTDB);
    933 		error = ENOENT;
    934 		goto bad;
    935 	}
    936 	KEY_SP_REF(isr->sp);
    937 	KEY_SA_REF(sav);
    938 	pserialize_read_exit(s);
    939     }
    940 
    941 	/* Callback parameters */
    942 	tc->tc_isr = isr;
    943 	tc->tc_spi = sav->spi;
    944 	tc->tc_dst = saidx->dst;
    945 	tc->tc_proto = saidx->proto;
    946 	tc->tc_sav = sav;
    947 
    948 	/* Crypto operation descriptor. */
    949 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
    950 	crp->crp_flags = CRYPTO_F_IMBUF;
    951 	crp->crp_buf = m;
    952 	crp->crp_callback = esp_output_cb;
    953 	crp->crp_opaque = tc;
    954 	crp->crp_sid = sav->tdb_cryptoid;
    955 
    956 	if (esph) {
    957 		/* Authentication descriptor. */
    958 		crda->crd_skip = skip;
    959 		if (espx && espx->type == CRYPTO_AES_GCM_16)
    960 			crda->crd_len = hlen - sav->ivlen;
    961 		else
    962 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
    963 		crda->crd_inject = m->m_pkthdr.len - alen;
    964 
    965 		/* Authentication operation. */
    966 		crda->crd_alg = esph->type;
    967 		if (espx && (espx->type == CRYPTO_AES_GCM_16 ||
    968 			     espx->type == CRYPTO_AES_GMAC)) {
    969 			crda->crd_key = _KEYBUF(sav->key_enc);
    970 			crda->crd_klen = _KEYBITS(sav->key_enc);
    971 		} else {
    972 			crda->crd_key = _KEYBUF(sav->key_auth);
    973 			crda->crd_klen = _KEYBITS(sav->key_auth);
    974 		}
    975 	}
    976 
    977 	return crypto_dispatch(crp);
    978 bad:
    979 	if (m)
    980 		m_freem(m);
    981 	return (error);
    982 }
    983 
    984 /*
    985  * ESP output callback from the crypto driver.
    986  */
    987 static int
    988 esp_output_cb(struct cryptop *crp)
    989 {
    990 	struct tdb_crypto *tc;
    991 	const struct ipsecrequest *isr;
    992 	struct secasvar *sav;
    993 	struct mbuf *m;
    994 	int err, error;
    995 	IPSEC_DECLARE_LOCK_VARIABLE;
    996 
    997 	KASSERT(crp->crp_opaque != NULL);
    998 	tc = crp->crp_opaque;
    999 	m = crp->crp_buf;
   1000 
   1001 	IPSEC_ACQUIRE_GLOBAL_LOCKS();
   1002 
   1003 	isr = tc->tc_isr;
   1004 	sav = tc->tc_sav;
   1005 	if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD)) {
   1006 		ESP_STATINC(ESP_STAT_NOTDB);
   1007 		IPSECLOG(LOG_DEBUG,
   1008 		    "SP is being destroyed while in crypto (id=%u)\n",
   1009 		    isr->sp->id);
   1010 		error = ENOENT;
   1011 		goto bad;
   1012 	}
   1013 	if (__predict_false(!SADB_SASTATE_USABLE_P(sav))) {
   1014 		KEY_SA_UNREF(&sav);
   1015 		sav = KEY_LOOKUP_SA(&tc->tc_dst, tc->tc_proto, tc->tc_spi, 0, 0);
   1016 		if (sav == NULL) {
   1017 			char buf[IPSEC_ADDRSTRLEN];
   1018 			ESP_STATINC(ESP_STAT_NOTDB);
   1019 			DPRINTF(("%s: SA expired while in crypto (SA %s/%08lx "
   1020 			    "proto %u)\n", __func__,
   1021 			    ipsec_address(&tc->tc_dst, buf, sizeof(buf)),
   1022 			    (u_long) ntohl(tc->tc_spi), tc->tc_proto));
   1023 			error = ENOBUFS;		/*XXX*/
   1024 			goto bad;
   1025 		}
   1026 	}
   1027 
   1028 	/* Check for crypto errors. */
   1029 	if (crp->crp_etype) {
   1030 		/* Reset session ID. */
   1031 		if (sav->tdb_cryptoid != 0)
   1032 			sav->tdb_cryptoid = crp->crp_sid;
   1033 
   1034 		if (crp->crp_etype == EAGAIN) {
   1035 			IPSEC_RELEASE_GLOBAL_LOCKS();
   1036 			return crypto_dispatch(crp);
   1037 		}
   1038 
   1039 		ESP_STATINC(ESP_STAT_NOXFORM);
   1040 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
   1041 		error = crp->crp_etype;
   1042 		goto bad;
   1043 	}
   1044 
   1045 	ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]);
   1046 	if (sav->tdb_authalgxform != NULL)
   1047 		AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
   1048 
   1049 	/* Release crypto descriptors. */
   1050 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
   1051 	crypto_freereq(crp);
   1052 
   1053 #ifdef IPSEC_DEBUG
   1054 	/* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */
   1055 	if (ipsec_integrity) {
   1056 		static unsigned char ipseczeroes[AH_ALEN_MAX];
   1057 		const struct auth_hash *esph;
   1058 
   1059 		/*
   1060 		 * Corrupt HMAC if we want to test integrity verification of
   1061 		 * the other side.
   1062 		 */
   1063 		esph = sav->tdb_authalgxform;
   1064 		if (esph !=  NULL) {
   1065 			m_copyback(m, m->m_pkthdr.len - esph->authsize,
   1066 			    esph->authsize, ipseczeroes);
   1067 		}
   1068 	}
   1069 #endif
   1070 
   1071 	/* NB: m is reclaimed by ipsec_process_done. */
   1072 	err = ipsec_process_done(m, isr, sav);
   1073 	KEY_SA_UNREF(&sav);
   1074 	KEY_SP_UNREF(&isr->sp);
   1075 	IPSEC_RELEASE_GLOBAL_LOCKS();
   1076 	return err;
   1077 bad:
   1078 	if (sav)
   1079 		KEY_SA_UNREF(&sav);
   1080 	KEY_SP_UNREF(&isr->sp);
   1081 	IPSEC_RELEASE_GLOBAL_LOCKS();
   1082 	if (m)
   1083 		m_freem(m);
   1084 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
   1085 	crypto_freereq(crp);
   1086 	return error;
   1087 }
   1088 
   1089 static struct xformsw esp_xformsw = {
   1090 	.xf_type	= XF_ESP,
   1091 	.xf_flags	= XFT_CONF|XFT_AUTH,
   1092 	.xf_name	= "IPsec ESP",
   1093 	.xf_init	= esp_init,
   1094 	.xf_zeroize	= esp_zeroize,
   1095 	.xf_input	= esp_input,
   1096 	.xf_output	= esp_output,
   1097 	.xf_next	= NULL,
   1098 };
   1099 
   1100 void
   1101 esp_attach(void)
   1102 {
   1103 
   1104 	espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS);
   1105 
   1106 	extern int ah_max_authsize;
   1107 	KASSERT(ah_max_authsize != 0);
   1108 	esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize;
   1109 	esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size,
   1110 	    coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET,
   1111 	    NULL, NULL, NULL);
   1112 
   1113 #define	MAXIV(xform)					\
   1114 	if (xform.ivsize > esp_max_ivlen)		\
   1115 		esp_max_ivlen = xform.ivsize		\
   1116 
   1117 	esp_max_ivlen = 0;
   1118 	MAXIV(enc_xform_des);		/* SADB_EALG_DESCBC */
   1119 	MAXIV(enc_xform_3des);		/* SADB_EALG_3DESCBC */
   1120 	MAXIV(enc_xform_rijndael128);	/* SADB_X_EALG_AES */
   1121 	MAXIV(enc_xform_blf);		/* SADB_X_EALG_BLOWFISHCBC */
   1122 	MAXIV(enc_xform_cast5);		/* SADB_X_EALG_CAST128CBC */
   1123 	MAXIV(enc_xform_skipjack);	/* SADB_X_EALG_SKIPJACK */
   1124 	MAXIV(enc_xform_camellia);	/* SADB_X_EALG_CAMELLIACBC */
   1125 	MAXIV(enc_xform_aes_ctr);	/* SADB_X_EALG_AESCTR */
   1126 	MAXIV(enc_xform_null);		/* SADB_EALG_NULL */
   1127 
   1128 	xform_register(&esp_xformsw);
   1129 #undef MAXIV
   1130 }
   1131