Home | History | Annotate | Line # | Download | only in netipsec
xform_esp.c revision 1.106
      1 /*	$NetBSD: xform_esp.c,v 1.106 2022/05/25 04:15:44 ozaki-r Exp $	*/
      2 /*	$FreeBSD: xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $	*/
      3 /*	$OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */
      4 
      5 /*
      6  * The authors of this code are John Ioannidis (ji (at) tla.org),
      7  * Angelos D. Keromytis (kermit (at) csd.uch.gr) and
      8  * Niels Provos (provos (at) physnet.uni-hamburg.de).
      9  *
     10  * The original version of this code was written by John Ioannidis
     11  * for BSD/OS in Athens, Greece, in November 1995.
     12  *
     13  * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
     14  * by Angelos D. Keromytis.
     15  *
     16  * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
     17  * and Niels Provos.
     18  *
     19  * Additional features in 1999 by Angelos D. Keromytis.
     20  *
     21  * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
     22  * Angelos D. Keromytis and Niels Provos.
     23  * Copyright (c) 2001 Angelos D. Keromytis.
     24  *
     25  * Permission to use, copy, and modify this software with or without fee
     26  * is hereby granted, provided that this entire notice is included in
     27  * all copies of any software which is or includes a copy or
     28  * modification of this software.
     29  * You may use this code under the GNU public license if you so wish. Please
     30  * contribute changes back to the authors under this freer than GPL license
     31  * so that we may further the use of strong encryption without limitations to
     32  * all.
     33  *
     34  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
     35  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
     36  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
     37  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
     38  * PURPOSE.
     39  */
     40 
     41 #include <sys/cdefs.h>
     42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.106 2022/05/25 04:15:44 ozaki-r Exp $");
     43 
     44 #if defined(_KERNEL_OPT)
     45 #include "opt_inet.h"
     46 #include "opt_ipsec.h"
     47 #endif
     48 
     49 #include <sys/param.h>
     50 #include <sys/systm.h>
     51 #include <sys/mbuf.h>
     52 #include <sys/socket.h>
     53 #include <sys/syslog.h>
     54 #include <sys/kernel.h>
     55 #include <sys/sysctl.h>
     56 #include <sys/cprng.h>
     57 #include <sys/pool.h>
     58 #include <sys/pserialize.h>
     59 
     60 #include <net/if.h>
     61 
     62 #include <netinet/in.h>
     63 #include <netinet/in_systm.h>
     64 #include <netinet/ip.h>
     65 #include <netinet/ip_ecn.h>
     66 #include <netinet/ip6.h>
     67 
     68 #include <net/route.h>
     69 #include <netipsec/ipsec.h>
     70 #include <netipsec/ipsec_private.h>
     71 #include <netipsec/ah.h>
     72 #include <netipsec/ah_var.h>
     73 #include <netipsec/esp.h>
     74 #include <netipsec/esp_var.h>
     75 #include <netipsec/xform.h>
     76 
     77 #ifdef INET6
     78 #include <netinet6/ip6_var.h>
     79 #include <netipsec/ipsec6.h>
     80 #endif
     81 
     82 #include <netipsec/key.h>
     83 #include <netipsec/key_debug.h>
     84 
     85 #include <opencrypto/cryptodev.h>
     86 
     87 percpu_t *espstat_percpu;
     88 
     89 int esp_enable = 1;
     90 
     91 static int esp_max_ivlen;		/* max iv length over all algorithms */
     92 
     93 static void esp_input_cb(struct cryptop *op);
     94 static void esp_output_cb(struct cryptop *crp);
     95 
     96 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT };
     97 
     98 static pool_cache_t esp_tdb_crypto_pool_cache;
     99 static size_t esp_pool_item_size;
    100 
    101 /*
    102  * NB: this is public for use by the PF_KEY support.
    103  * NB: if you add support here; be sure to add code to esp_attach below!
    104  */
    105 const struct enc_xform *
    106 esp_algorithm_lookup(int alg)
    107 {
    108 
    109 	switch (alg) {
    110 	case SADB_EALG_DESCBC:
    111 		return &enc_xform_des;
    112 	case SADB_EALG_3DESCBC:
    113 		return &enc_xform_3des;
    114 	case SADB_X_EALG_AES:
    115 		return &enc_xform_aes;
    116 	case SADB_X_EALG_BLOWFISHCBC:
    117 		return &enc_xform_blf;
    118 	case SADB_X_EALG_CAST128CBC:
    119 		return &enc_xform_cast5;
    120 	case SADB_X_EALG_SKIPJACK:
    121 		return &enc_xform_skipjack;
    122 	case SADB_X_EALG_CAMELLIACBC:
    123 		return &enc_xform_camellia;
    124 	case SADB_X_EALG_AESCTR:
    125 		return &enc_xform_aes_ctr;
    126 	case SADB_X_EALG_AESGCM16:
    127 		return &enc_xform_aes_gcm;
    128 	case SADB_X_EALG_AESGMAC:
    129 		return &enc_xform_aes_gmac;
    130 	case SADB_EALG_NULL:
    131 		return &enc_xform_null;
    132 	}
    133 	return NULL;
    134 }
    135 
    136 size_t
    137 esp_hdrsiz(const struct secasvar *sav)
    138 {
    139 	size_t size;
    140 
    141 	if (sav != NULL) {
    142 		/*XXX not right for null algorithm--does it matter??*/
    143 		KASSERT(sav->tdb_encalgxform != NULL);
    144 
    145 		/*
    146 		 *   base header size
    147 		 * + iv length for CBC mode
    148 		 * + max pad length
    149 		 * + sizeof(esp trailer)
    150 		 * + icv length (if any).
    151 		 */
    152 		if (sav->flags & SADB_X_EXT_OLD)
    153 			size = sizeof(struct esp);
    154 		else
    155 			size = sizeof(struct newesp);
    156 		size += sav->tdb_encalgxform->ivsize + 9 +
    157 		    sizeof(struct esptail);
    158 
    159 		/*XXX need alg check???*/
    160 		if (sav->tdb_authalgxform != NULL && sav->replay)
    161 			size += ah_authsiz(sav);
    162 	} else {
    163 		/*
    164 		 *   base header size
    165 		 * + max iv length for CBC mode
    166 		 * + max pad length
    167 		 * + sizeof(esp trailer)
    168 		 * + max icv supported.
    169 		 */
    170 		size = sizeof(struct newesp) + esp_max_ivlen + 9 +
    171 		    sizeof(struct esptail) + ah_authsiz(NULL);
    172 	}
    173 	return size;
    174 }
    175 
    176 /*
    177  * esp_init() is called when an SPI is being set up.
    178  */
    179 static int
    180 esp_init(struct secasvar *sav, const struct xformsw *xsp)
    181 {
    182 	const struct enc_xform *txform;
    183 	struct cryptoini cria, crie, *cr;
    184 	int keylen;
    185 	int error;
    186 
    187 	txform = esp_algorithm_lookup(sav->alg_enc);
    188 	if (txform == NULL) {
    189 		DPRINTF("unsupported encryption algorithm %d\n",
    190 		    sav->alg_enc);
    191 		return EINVAL;
    192 	}
    193 	if (sav->key_enc == NULL) {
    194 		DPRINTF("no encoding key for %s algorithm\n",
    195 		    txform->name);
    196 		return EINVAL;
    197 	}
    198 	if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) {
    199 		DPRINTF("4-byte IV not supported with protocol\n");
    200 		return EINVAL;
    201 	}
    202 	keylen = _KEYLEN(sav->key_enc);
    203 	if (txform->minkey > keylen || keylen > txform->maxkey) {
    204 		DPRINTF("invalid key length %u, must be in "
    205 		    "the range [%u..%u] for algorithm %s\n",
    206 		    keylen, txform->minkey, txform->maxkey, txform->name);
    207 		return EINVAL;
    208 	}
    209 
    210 	sav->ivlen = txform->ivsize;
    211 
    212 	/*
    213 	 * Setup AH-related state.
    214 	 */
    215 	if (sav->alg_auth != 0) {
    216 		error = ah_init0(sav, xsp, &cria);
    217 		if (error)
    218 			return error;
    219 	}
    220 
    221 	/* NB: override anything set in ah_init0 */
    222 	sav->tdb_xform = xsp;
    223 	sav->tdb_encalgxform = txform;
    224 
    225 	switch (sav->alg_enc) {
    226 	case SADB_X_EALG_AESGCM16:
    227 	case SADB_X_EALG_AESGMAC:
    228 		switch (keylen) {
    229 		case 20:
    230 			sav->alg_auth = SADB_X_AALG_AES128GMAC;
    231 			sav->tdb_authalgxform = &auth_hash_gmac_aes_128;
    232 			break;
    233 		case 28:
    234 			sav->alg_auth = SADB_X_AALG_AES192GMAC;
    235 			sav->tdb_authalgxform = &auth_hash_gmac_aes_192;
    236 			break;
    237 		case 36:
    238 			sav->alg_auth = SADB_X_AALG_AES256GMAC;
    239 			sav->tdb_authalgxform = &auth_hash_gmac_aes_256;
    240 			break;
    241 		default:
    242 			DPRINTF("invalid key length %u, must be either of "
    243 			    "20, 28 or 36\n", keylen);
    244 			return EINVAL;
    245 		}
    246 
    247 		memset(&cria, 0, sizeof(cria));
    248 		cria.cri_alg = sav->tdb_authalgxform->type;
    249 		cria.cri_klen = _KEYBITS(sav->key_enc);
    250 		cria.cri_key = _KEYBUF(sav->key_enc);
    251 		break;
    252 	default:
    253 		break;
    254 	}
    255 
    256 	/* Initialize crypto session. */
    257 	memset(&crie, 0, sizeof(crie));
    258 	crie.cri_alg = sav->tdb_encalgxform->type;
    259 	crie.cri_klen = _KEYBITS(sav->key_enc);
    260 	crie.cri_key = _KEYBUF(sav->key_enc);
    261 	/* XXX Rounds ? */
    262 
    263 	if (sav->tdb_authalgxform && sav->tdb_encalgxform) {
    264 		/* init both auth & enc */
    265 		crie.cri_next = &cria;
    266 		cr = &crie;
    267 	} else if (sav->tdb_encalgxform) {
    268 		cr = &crie;
    269 	} else if (sav->tdb_authalgxform) {
    270 		cr = &cria;
    271 	} else {
    272 		/* XXX cannot happen? */
    273 		DPRINTF("no encoding OR authentication xform!\n");
    274 		return EINVAL;
    275 	}
    276 
    277 	return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support);
    278 }
    279 
    280 /*
    281  * Paranoia.
    282  */
    283 static void
    284 esp_zeroize(struct secasvar *sav)
    285 {
    286 	/* NB: ah_zerorize free's the crypto session state */
    287 	ah_zeroize(sav);
    288 
    289 	if (sav->key_enc) {
    290 		explicit_memset(_KEYBUF(sav->key_enc), 0,
    291 		    _KEYLEN(sav->key_enc));
    292 	}
    293 	sav->tdb_encalgxform = NULL;
    294 	sav->tdb_xform = NULL;
    295 }
    296 
    297 /*
    298  * ESP input processing, called (eventually) through the protocol switch.
    299  */
    300 static int
    301 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
    302 {
    303 	const struct auth_hash *esph;
    304 	const struct enc_xform *espx;
    305 	struct tdb_crypto *tc;
    306 	int plen, alen, hlen, error, stat = ESP_STAT_CRYPTO;
    307 	struct newesp *esp;
    308 	struct cryptodesc *crde;
    309 	struct cryptop *crp;
    310 
    311 	KASSERT(sav != NULL);
    312 	KASSERT(sav->tdb_encalgxform != NULL);
    313 	if (__predict_false((skip & 3) != 0 || (m->m_pkthdr.len & 3) != 0)) {
    314 		DPRINTF("%s: misaligned packet, skip %u pkt len %u", __func__,
    315 		    skip, m->m_pkthdr.len);
    316 		stat = ESP_STAT_BADILEN; /* Same as FreeBSD */
    317 		error = EINVAL;
    318 		goto out;
    319 	}
    320 
    321 	/* XXX don't pullup, just copy header */
    322 	M_REGION_GET(esp, struct newesp *, m, skip, sizeof(struct newesp));
    323 	if (esp == NULL) {
    324 		/* m already freed */
    325 		return ENOBUFS;
    326 	}
    327 
    328 	esph = sav->tdb_authalgxform;
    329 	espx = sav->tdb_encalgxform;
    330 	KASSERT(espx != NULL);
    331 
    332 	/* Determine the ESP header length */
    333 	if (sav->flags & SADB_X_EXT_OLD)
    334 		hlen = sizeof(struct esp) + sav->ivlen;
    335 	else
    336 		hlen = sizeof(struct newesp) + sav->ivlen;
    337 	/* Authenticator hash size */
    338 	alen = esph ? esph->authsize : 0;
    339 
    340 	/*
    341 	 * Verify payload length is multiple of encryption algorithm block
    342 	 * size.
    343 	 *
    344 	 * The payload must also be 4-byte-aligned. This is implicitly
    345 	 * verified here too, since the blocksize is always 4-byte-aligned.
    346 	 */
    347 	plen = m->m_pkthdr.len - (skip + hlen + alen);
    348 	KASSERT((espx->blocksize & 3) == 0);
    349 	if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
    350 		char buf[IPSEC_ADDRSTRLEN];
    351 		DPRINTF("payload of %d octets not a multiple of %d octets,"
    352 		    "  SA %s/%08lx\n", plen, espx->blocksize,
    353 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
    354 		    (u_long) ntohl(sav->spi));
    355 		stat = ESP_STAT_BADILEN;
    356 		error = EINVAL;
    357 		goto out;
    358 	}
    359 
    360 	/*
    361 	 * Check sequence number.
    362 	 */
    363 	if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
    364 		char logbuf[IPSEC_LOGSASTRLEN];
    365 		DPRINTF("packet replay check for %s\n",
    366 		    ipsec_logsastr(sav, logbuf, sizeof(logbuf)));
    367 		stat = ESP_STAT_REPLAY;
    368 		error = EACCES;
    369 		goto out;
    370 	}
    371 
    372 	/* Update the counters */
    373 	ESP_STATADD(ESP_STAT_IBYTES, plen);
    374 
    375 	/* Get crypto descriptors */
    376 	crp = crypto_getreq(esph ? 2 : 1);
    377 	if (crp == NULL) {
    378 		DPRINTF("failed to acquire crypto descriptors\n");
    379 		error = ENOBUFS;
    380 		goto out;
    381 	}
    382 
    383 	/* Get IPsec-specific opaque pointer */
    384 	size_t extra __diagused = esph == NULL ? 0 : alen;
    385 	KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size,
    386 	    "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n",
    387 	    sizeof(*tc) + extra, esp_pool_item_size);
    388 	tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
    389 	if (tc == NULL) {
    390 		DPRINTF("failed to allocate tdb_crypto\n");
    391 		error = ENOBUFS;
    392 		goto out1;
    393 	}
    394 
    395 	error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT);
    396 	if (error) {
    397 		DPRINTF("m_makewritable failed\n");
    398 		goto out2;
    399 	}
    400 
    401 	if (esph) {
    402 		struct cryptodesc *crda;
    403 
    404 		KASSERT(crp->crp_desc != NULL);
    405 		crda = crp->crp_desc;
    406 
    407 		/* Authentication descriptor */
    408 		crda->crd_skip = skip;
    409 		if (espx->type == CRYPTO_AES_GCM_16)
    410 			crda->crd_len = hlen - sav->ivlen;
    411 		else
    412 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
    413 		crda->crd_inject = m->m_pkthdr.len - alen;
    414 
    415 		crda->crd_alg = esph->type;
    416 		if (espx->type == CRYPTO_AES_GCM_16 ||
    417 		    espx->type == CRYPTO_AES_GMAC) {
    418 			crda->crd_key = _KEYBUF(sav->key_enc);
    419 			crda->crd_klen = _KEYBITS(sav->key_enc);
    420 		} else {
    421 			crda->crd_key = _KEYBUF(sav->key_auth);
    422 			crda->crd_klen = _KEYBITS(sav->key_auth);
    423 		}
    424 
    425 		/* Copy the authenticator */
    426 		m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1));
    427 
    428 		/* Chain authentication request */
    429 		crde = crda->crd_next;
    430 	} else {
    431 		crde = crp->crp_desc;
    432 	}
    433 
    434     {
    435 	int s = pserialize_read_enter();
    436 
    437 	/*
    438 	 * Take another reference to the SA for opencrypto callback.
    439 	 */
    440 	if (__predict_false(sav->state == SADB_SASTATE_DEAD)) {
    441 		pserialize_read_exit(s);
    442 		stat = ESP_STAT_NOTDB;
    443 		error = ENOENT;
    444 		goto out2;
    445 	}
    446 	KEY_SA_REF(sav);
    447 	pserialize_read_exit(s);
    448     }
    449 
    450 	/* Crypto operation descriptor */
    451 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
    452 	crp->crp_flags = CRYPTO_F_IMBUF;
    453 	crp->crp_buf = m;
    454 	crp->crp_callback = esp_input_cb;
    455 	crp->crp_sid = sav->tdb_cryptoid;
    456 	crp->crp_opaque = tc;
    457 
    458 	/* These are passed as-is to the callback */
    459 	tc->tc_spi = sav->spi;
    460 	tc->tc_dst = sav->sah->saidx.dst;
    461 	tc->tc_proto = sav->sah->saidx.proto;
    462 	tc->tc_protoff = protoff;
    463 	tc->tc_skip = skip;
    464 	tc->tc_sav = sav;
    465 
    466 	/* Decryption descriptor */
    467 	KASSERTMSG(crde != NULL, "null esp crypto descriptor");
    468 	crde->crd_skip = skip + hlen;
    469 	if (espx->type == CRYPTO_AES_GMAC)
    470 		crde->crd_len = 0;
    471 	else
    472 		crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
    473 	crde->crd_inject = skip + hlen - sav->ivlen;
    474 	crde->crd_alg = espx->type;
    475 	crde->crd_key = _KEYBUF(sav->key_enc);
    476 	crde->crd_klen = _KEYBITS(sav->key_enc);
    477 	/* XXX Rounds ? */
    478 
    479 	crypto_dispatch(crp);
    480 	return 0;
    481 
    482 out2:
    483 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    484 out1:
    485 	crypto_freereq(crp);
    486 out:
    487 	ESP_STATINC(stat);
    488 	m_freem(m);
    489 	return error;
    490 }
    491 
    492 #ifdef INET6
    493 #define	IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do {		\
    494 	if (saidx->dst.sa.sa_family == AF_INET6) {			\
    495 		(void)ipsec6_common_input_cb(m, sav, skip, protoff);	\
    496 	} else {							\
    497 		(void)ipsec4_common_input_cb(m, sav, skip, protoff);	\
    498 	}								\
    499 } while (0)
    500 #else
    501 #define	IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff)			\
    502 	((void)ipsec4_common_input_cb(m, sav, skip, protoff))
    503 #endif
    504 
    505 /*
    506  * ESP input callback from the crypto driver.
    507  */
    508 static void
    509 esp_input_cb(struct cryptop *crp)
    510 {
    511 	char buf[IPSEC_ADDRSTRLEN];
    512 	uint8_t lastthree[3], aalg[AH_ALEN_MAX];
    513 	int hlen, skip, protoff;
    514 	struct mbuf *m;
    515 	const struct auth_hash *esph;
    516 	struct tdb_crypto *tc;
    517 	struct secasvar *sav;
    518 	struct secasindex *saidx;
    519 	void *ptr;
    520 	IPSEC_DECLARE_LOCK_VARIABLE;
    521 
    522 	KASSERT(crp->crp_desc != NULL);
    523 	KASSERT(crp->crp_opaque != NULL);
    524 
    525 	tc = crp->crp_opaque;
    526 	skip = tc->tc_skip;
    527 	protoff = tc->tc_protoff;
    528 	m = crp->crp_buf;
    529 
    530 	IPSEC_ACQUIRE_GLOBAL_LOCKS();
    531 
    532 	sav = tc->tc_sav;
    533 	saidx = &sav->sah->saidx;
    534 	KASSERTMSG(saidx->dst.sa.sa_family == AF_INET ||
    535 	    saidx->dst.sa.sa_family == AF_INET6,
    536 	    "unexpected protocol family %u", saidx->dst.sa.sa_family);
    537 
    538 	esph = sav->tdb_authalgxform;
    539 
    540 	/* Check for crypto errors */
    541 	if (crp->crp_etype) {
    542 		/* Reset the session ID */
    543 		if (sav->tdb_cryptoid != 0)
    544 			sav->tdb_cryptoid = crp->crp_sid;
    545 
    546 		ESP_STATINC(ESP_STAT_NOXFORM);
    547 		DPRINTF("crypto error %d\n", crp->crp_etype);
    548 		goto bad;
    549 	}
    550 
    551 	ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]);
    552 
    553 	/* If authentication was performed, check now. */
    554 	if (esph != NULL) {
    555 		/*
    556 		 * If we have a tag, it means an IPsec-aware NIC did
    557 		 * the verification for us.  Otherwise we need to
    558 		 * check the authentication calculation.
    559 		 */
    560 		AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
    561 		/* Copy the authenticator from the packet */
    562 		m_copydata(m, m->m_pkthdr.len - esph->authsize,
    563 			esph->authsize, aalg);
    564 
    565 		ptr = (tc + 1);
    566 
    567 		/* Verify authenticator */
    568 		if (!consttime_memequal(ptr, aalg, esph->authsize)) {
    569 			DPRINTF("authentication hash mismatch "
    570 			    "for packet in SA %s/%08lx\n",
    571 			    ipsec_address(&saidx->dst, buf,
    572 			    sizeof(buf)), (u_long) ntohl(sav->spi));
    573 			ESP_STATINC(ESP_STAT_BADAUTH);
    574 			goto bad;
    575 		}
    576 
    577 		/* Remove trailing authenticator */
    578 		m_adj(m, -(esph->authsize));
    579 	}
    580 
    581 	/* Release the crypto descriptors */
    582 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    583 	tc = NULL;
    584 	crypto_freereq(crp);
    585 	crp = NULL;
    586 
    587 	/*
    588 	 * Packet is now decrypted.
    589 	 */
    590 	m->m_flags |= M_DECRYPTED;
    591 
    592 	/*
    593 	 * Update replay sequence number, if appropriate.
    594 	 */
    595 	if (sav->replay) {
    596 		uint32_t seq;
    597 
    598 		m_copydata(m, skip + offsetof(struct newesp, esp_seq),
    599 		    sizeof(seq), &seq);
    600 		if (ipsec_updatereplay(ntohl(seq), sav)) {
    601 			char logbuf[IPSEC_LOGSASTRLEN];
    602 			DPRINTF("packet replay check for %s\n",
    603 			    ipsec_logsastr(sav, logbuf, sizeof(logbuf)));
    604 			ESP_STATINC(ESP_STAT_REPLAY);
    605 			goto bad;
    606 		}
    607 	}
    608 
    609 	/* Determine the ESP header length */
    610 	if (sav->flags & SADB_X_EXT_OLD)
    611 		hlen = sizeof(struct esp) + sav->ivlen;
    612 	else
    613 		hlen = sizeof(struct newesp) + sav->ivlen;
    614 
    615 	/* Remove the ESP header and IV from the mbuf. */
    616 	if (m_striphdr(m, skip, hlen) != 0) {
    617 		ESP_STATINC(ESP_STAT_HDROPS);
    618 		DPRINTF("bad mbuf chain, SA %s/%08lx\n",
    619 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
    620 		    (u_long) ntohl(sav->spi));
    621 		goto bad;
    622 	}
    623 
    624 	/* Save the last three bytes of decrypted data */
    625 	m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree);
    626 
    627 	/* Verify pad length */
    628 	if (lastthree[1] + 2 > m->m_pkthdr.len - skip) {
    629 		ESP_STATINC(ESP_STAT_BADILEN);
    630 		DPRINTF("invalid padding length %d "
    631 		    "for %u byte packet in SA %s/%08lx\n",
    632 		    lastthree[1], m->m_pkthdr.len - skip,
    633 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
    634 		    (u_long) ntohl(sav->spi));
    635 		goto bad;
    636 	}
    637 
    638 	/* Verify correct decryption by checking the last padding bytes */
    639 	if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) {
    640 		if (lastthree[1] != lastthree[0] && lastthree[1] != 0) {
    641 			ESP_STATINC(ESP_STAT_BADENC);
    642 			DPRINTF("decryption failed for packet in SA "
    643 			    "%s/%08lx\n",
    644 			    ipsec_address(&sav->sah->saidx.dst, buf,
    645 			    sizeof(buf)), (u_long) ntohl(sav->spi));
    646 			DPRINTF("%x %x\n", lastthree[0],
    647 			    lastthree[1]);
    648 			goto bad;
    649 		}
    650 	}
    651 
    652 	/* Trim the mbuf chain to remove trailing authenticator and padding */
    653 	m_adj(m, -(lastthree[1] + 2));
    654 
    655 	/* Restore the Next Protocol field */
    656 	m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2);
    657 
    658 	IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff);
    659 
    660 	KEY_SA_UNREF(&sav);
    661 	IPSEC_RELEASE_GLOBAL_LOCKS();
    662 	return;
    663 bad:
    664 	if (sav)
    665 		KEY_SA_UNREF(&sav);
    666 	IPSEC_RELEASE_GLOBAL_LOCKS();
    667 	if (m != NULL)
    668 		m_freem(m);
    669 	if (tc != NULL)
    670 		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    671 	if (crp != NULL)
    672 		crypto_freereq(crp);
    673 }
    674 
    675 /*
    676  * ESP output routine, called by ipsec[46]_process_packet().
    677  */
    678 static int
    679 esp_output(struct mbuf *m, const struct ipsecrequest *isr, struct secasvar *sav,
    680     int skip, int protoff, int flags)
    681 {
    682 	char buf[IPSEC_ADDRSTRLEN];
    683 	const struct enc_xform *espx;
    684 	const struct auth_hash *esph;
    685 	int hlen, rlen, tlen, padlen, blks, alen, i, roff;
    686 	struct mbuf *mo = NULL;
    687 	struct tdb_crypto *tc;
    688 	struct secasindex *saidx;
    689 	unsigned char *tail;
    690 	uint8_t prot;
    691 	int error, maxpacketsize;
    692 	struct esptail *esptail;
    693 	struct cryptodesc *crde, *crda;
    694 	struct cryptop *crp;
    695 
    696 	esph = sav->tdb_authalgxform;
    697 	espx = sav->tdb_encalgxform;
    698 	KASSERT(espx != NULL);
    699 
    700 	/* Determine the ESP header length */
    701 	if (sav->flags & SADB_X_EXT_OLD)
    702 		hlen = sizeof(struct esp) + sav->ivlen;
    703 	else
    704 		hlen = sizeof(struct newesp) + sav->ivlen;
    705 	/* Authenticator hash size */
    706 	alen = esph ? esph->authsize : 0;
    707 
    708 	/*
    709 	 * NB: The null encoding transform has a blocksize of 4
    710 	 *     so that headers are properly aligned.
    711 	 */
    712 	blks = espx->blocksize;		/* IV blocksize */
    713 
    714 	/* Raw payload length. */
    715 	rlen = m->m_pkthdr.len - skip;
    716 
    717 	/* Encryption padding. */
    718 	padlen = ((blks - ((rlen + sizeof(struct esptail)) % blks)) % blks);
    719 
    720 	/* Length of what we append (tail). */
    721 	tlen = padlen + sizeof(struct esptail) + alen;
    722 
    723 	ESP_STATINC(ESP_STAT_OUTPUT);
    724 
    725 	saidx = &sav->sah->saidx;
    726 	/* Check for maximum packet size violations. */
    727 	switch (saidx->dst.sa.sa_family) {
    728 #ifdef INET
    729 	case AF_INET:
    730 		maxpacketsize = IP_MAXPACKET;
    731 		break;
    732 #endif
    733 #ifdef INET6
    734 	case AF_INET6:
    735 		maxpacketsize = IPV6_MAXPACKET;
    736 		break;
    737 #endif
    738 	default:
    739 		DPRINTF("unknown/unsupported protocol family %d, "
    740 		    "SA %s/%08lx\n", saidx->dst.sa.sa_family,
    741 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    742 		    (u_long)ntohl(sav->spi));
    743 		ESP_STATINC(ESP_STAT_NOPF);
    744 		error = EPFNOSUPPORT;
    745 		goto bad;
    746 	}
    747 	if (skip + hlen + rlen + tlen > maxpacketsize) {
    748 		DPRINTF("packet in SA %s/%08lx got too big (len %u, "
    749 		    "max len %u)\n",
    750 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    751 		    (u_long) ntohl(sav->spi),
    752 		    skip + hlen + rlen + tlen, maxpacketsize);
    753 		ESP_STATINC(ESP_STAT_TOOBIG);
    754 		error = EMSGSIZE;
    755 		goto bad;
    756 	}
    757 
    758 	/* Update the counters. */
    759 	ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip);
    760 
    761 	m = m_clone(m);
    762 	if (m == NULL) {
    763 		DPRINTF("cannot clone mbuf chain, SA %s/%08lx\n",
    764 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    765 		    (u_long) ntohl(sav->spi));
    766 		ESP_STATINC(ESP_STAT_HDROPS);
    767 		error = ENOBUFS;
    768 		goto bad;
    769 	}
    770 
    771 	/* Inject ESP header. */
    772 	mo = m_makespace(m, skip, hlen, &roff);
    773 	if (mo == NULL) {
    774 		DPRINTF("failed to inject %u byte ESP hdr for SA "
    775 		    "%s/%08lx\n", hlen,
    776 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    777 		    (u_long) ntohl(sav->spi));
    778 		ESP_STATINC(ESP_STAT_HDROPS);
    779 		error = ENOBUFS;
    780 		goto bad;
    781 	}
    782 
    783 	/* Initialize ESP header. */
    784 	memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t));
    785 	if (sav->replay) {
    786 		uint32_t replay;
    787 
    788 #ifdef IPSEC_DEBUG
    789 		/* Emulate replay attack when ipsec_replay is TRUE. */
    790 		if (ipsec_replay)
    791 			replay = htonl(sav->replay->count);
    792 		else
    793 #endif
    794 			replay = htonl(atomic_inc_32_nv(&sav->replay->count));
    795 
    796 		memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay,
    797 		    sizeof(uint32_t));
    798 	}
    799 
    800 	/*
    801 	 * Grow the mbuf, we will append data at the tail.
    802 	 */
    803 	tail = m_pad(m, tlen);
    804 	if (tail == NULL) {
    805 		DPRINTF("m_pad failed for SA %s/%08lx\n",
    806 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
    807 		    (u_long) ntohl(sav->spi));
    808 		m = NULL;
    809 		error = ENOBUFS;
    810 		goto bad;
    811 	}
    812 
    813 	/*
    814 	 * Add padding: random, zero, or self-describing.
    815 	 */
    816 	switch (sav->flags & SADB_X_EXT_PMASK) {
    817 	case SADB_X_EXT_PSEQ:
    818 		for (i = 0; i < padlen; i++)
    819 			tail[i] = i + 1;
    820 		break;
    821 	case SADB_X_EXT_PRAND:
    822 		(void)cprng_fast(tail, padlen);
    823 		break;
    824 	case SADB_X_EXT_PZERO:
    825 	default:
    826 		memset(tail, 0, padlen);
    827 		break;
    828 	}
    829 
    830 	/* Build the ESP Trailer. */
    831 	esptail = (struct esptail *)&tail[padlen];
    832 	esptail->esp_padlen = padlen;
    833 	m_copydata(m, protoff, sizeof(uint8_t), &esptail->esp_nxt);
    834 
    835 	/* Fix Next Protocol in IPv4/IPv6 header. */
    836 	prot = IPPROTO_ESP;
    837 	m_copyback(m, protoff, sizeof(uint8_t), &prot);
    838 
    839 	/* Get crypto descriptors. */
    840 	crp = crypto_getreq(esph ? 2 : 1);
    841 	if (crp == NULL) {
    842 		DPRINTF("failed to acquire crypto descriptors\n");
    843 		ESP_STATINC(ESP_STAT_CRYPTO);
    844 		error = ENOBUFS;
    845 		goto bad;
    846 	}
    847 
    848 	/* Get the descriptors. */
    849 	crde = crp->crp_desc;
    850 	crda = crde->crd_next;
    851 
    852 	/* Encryption descriptor. */
    853 	crde->crd_skip = skip + hlen;
    854 	if (espx->type == CRYPTO_AES_GMAC)
    855 		crde->crd_len = 0;
    856 	else
    857 		crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
    858 	crde->crd_flags = CRD_F_ENCRYPT;
    859 	crde->crd_inject = skip + hlen - sav->ivlen;
    860 	crde->crd_alg = espx->type;
    861 	crde->crd_key = _KEYBUF(sav->key_enc);
    862 	crde->crd_klen = _KEYBITS(sav->key_enc);
    863 	/* XXX Rounds ? */
    864 
    865 	/* IPsec-specific opaque crypto info. */
    866 	tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
    867 	if (tc == NULL) {
    868 		crypto_freereq(crp);
    869 		DPRINTF("failed to allocate tdb_crypto\n");
    870 		ESP_STATINC(ESP_STAT_CRYPTO);
    871 		error = ENOBUFS;
    872 		goto bad;
    873 	}
    874 
    875     {
    876 	int s = pserialize_read_enter();
    877 
    878 	/*
    879 	 * Take another reference to the SP and the SA for opencrypto callback.
    880 	 */
    881 	if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD ||
    882 	    sav->state == SADB_SASTATE_DEAD)) {
    883 		pserialize_read_exit(s);
    884 		pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    885 		crypto_freereq(crp);
    886 		ESP_STATINC(ESP_STAT_NOTDB);
    887 		error = ENOENT;
    888 		goto bad;
    889 	}
    890 	KEY_SP_REF(isr->sp);
    891 	KEY_SA_REF(sav);
    892 	pserialize_read_exit(s);
    893     }
    894 
    895 	/* Callback parameters */
    896 	tc->tc_isr = isr;
    897 	tc->tc_spi = sav->spi;
    898 	tc->tc_dst = saidx->dst;
    899 	tc->tc_proto = saidx->proto;
    900 	tc->tc_flags = flags;
    901 	tc->tc_sav = sav;
    902 
    903 	/* Crypto operation descriptor. */
    904 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
    905 	crp->crp_flags = CRYPTO_F_IMBUF;
    906 	crp->crp_buf = m;
    907 	crp->crp_callback = esp_output_cb;
    908 	crp->crp_opaque = tc;
    909 	crp->crp_sid = sav->tdb_cryptoid;
    910 
    911 	if (esph) {
    912 		/* Authentication descriptor. */
    913 		crda->crd_skip = skip;
    914 		if (espx->type == CRYPTO_AES_GCM_16)
    915 			crda->crd_len = hlen - sav->ivlen;
    916 		else
    917 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
    918 		crda->crd_inject = m->m_pkthdr.len - alen;
    919 
    920 		/* Authentication operation. */
    921 		crda->crd_alg = esph->type;
    922 		if (espx->type == CRYPTO_AES_GCM_16 ||
    923 		    espx->type == CRYPTO_AES_GMAC) {
    924 			crda->crd_key = _KEYBUF(sav->key_enc);
    925 			crda->crd_klen = _KEYBITS(sav->key_enc);
    926 		} else {
    927 			crda->crd_key = _KEYBUF(sav->key_auth);
    928 			crda->crd_klen = _KEYBITS(sav->key_auth);
    929 		}
    930 	}
    931 
    932 	crypto_dispatch(crp);
    933 	return 0;
    934 
    935 bad:
    936 	if (m)
    937 		m_freem(m);
    938 	return error;
    939 }
    940 
    941 /*
    942  * ESP output callback from the crypto driver.
    943  */
    944 static void
    945 esp_output_cb(struct cryptop *crp)
    946 {
    947 	struct tdb_crypto *tc;
    948 	const struct ipsecrequest *isr;
    949 	struct secasvar *sav;
    950 	struct mbuf *m;
    951 	int flags;
    952 	IPSEC_DECLARE_LOCK_VARIABLE;
    953 
    954 	KASSERT(crp->crp_opaque != NULL);
    955 	tc = crp->crp_opaque;
    956 	m = crp->crp_buf;
    957 
    958 	IPSEC_ACQUIRE_GLOBAL_LOCKS();
    959 
    960 	isr = tc->tc_isr;
    961 	sav = tc->tc_sav;
    962 
    963 	/* Check for crypto errors. */
    964 	if (crp->crp_etype) {
    965 		/* Reset session ID. */
    966 		if (sav->tdb_cryptoid != 0)
    967 			sav->tdb_cryptoid = crp->crp_sid;
    968 
    969 		ESP_STATINC(ESP_STAT_NOXFORM);
    970 		DPRINTF("crypto error %d\n", crp->crp_etype);
    971 		goto bad;
    972 	}
    973 
    974 	ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]);
    975 	if (sav->tdb_authalgxform != NULL)
    976 		AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
    977 
    978 	flags = tc->tc_flags;
    979 	/* Release crypto descriptors. */
    980 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
    981 	crypto_freereq(crp);
    982 
    983 #ifdef IPSEC_DEBUG
    984 	/* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */
    985 	if (ipsec_integrity) {
    986 		static unsigned char ipseczeroes[AH_ALEN_MAX];
    987 		const struct auth_hash *esph;
    988 
    989 		/*
    990 		 * Corrupt HMAC if we want to test integrity verification of
    991 		 * the other side.
    992 		 */
    993 		esph = sav->tdb_authalgxform;
    994 		if (esph !=  NULL) {
    995 			m_copyback(m, m->m_pkthdr.len - esph->authsize,
    996 			    esph->authsize, ipseczeroes);
    997 		}
    998 	}
    999 #endif
   1000 
   1001 	/* NB: m is reclaimed by ipsec_process_done. */
   1002 	(void)ipsec_process_done(m, isr, sav, flags);
   1003 	KEY_SA_UNREF(&sav);
   1004 	KEY_SP_UNREF(&isr->sp);
   1005 	IPSEC_RELEASE_GLOBAL_LOCKS();
   1006 	return;
   1007 
   1008 bad:
   1009 	if (sav)
   1010 		KEY_SA_UNREF(&sav);
   1011 	KEY_SP_UNREF(&isr->sp);
   1012 	IPSEC_RELEASE_GLOBAL_LOCKS();
   1013 	if (m)
   1014 		m_freem(m);
   1015 	pool_cache_put(esp_tdb_crypto_pool_cache, tc);
   1016 	crypto_freereq(crp);
   1017 }
   1018 
   1019 static struct xformsw esp_xformsw = {
   1020 	.xf_type	= XF_ESP,
   1021 	.xf_flags	= XFT_CONF|XFT_AUTH,
   1022 	.xf_name	= "IPsec ESP",
   1023 	.xf_init	= esp_init,
   1024 	.xf_zeroize	= esp_zeroize,
   1025 	.xf_input	= esp_input,
   1026 	.xf_output	= esp_output,
   1027 	.xf_next	= NULL,
   1028 };
   1029 
   1030 void
   1031 esp_attach(void)
   1032 {
   1033 
   1034 	espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS);
   1035 
   1036 	extern int ah_max_authsize;
   1037 	KASSERT(ah_max_authsize != 0);
   1038 	esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize;
   1039 	esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size,
   1040 	    coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET,
   1041 	    NULL, NULL, NULL);
   1042 
   1043 #define	MAXIV(xform)					\
   1044 	if (xform.ivsize > esp_max_ivlen)		\
   1045 		esp_max_ivlen = xform.ivsize		\
   1046 
   1047 	esp_max_ivlen = 0;
   1048 	MAXIV(enc_xform_des);		/* SADB_EALG_DESCBC */
   1049 	MAXIV(enc_xform_3des);		/* SADB_EALG_3DESCBC */
   1050 	MAXIV(enc_xform_aes);		/* SADB_X_EALG_AES */
   1051 	MAXIV(enc_xform_blf);		/* SADB_X_EALG_BLOWFISHCBC */
   1052 	MAXIV(enc_xform_cast5);		/* SADB_X_EALG_CAST128CBC */
   1053 	MAXIV(enc_xform_skipjack);	/* SADB_X_EALG_SKIPJACK */
   1054 	MAXIV(enc_xform_camellia);	/* SADB_X_EALG_CAMELLIACBC */
   1055 	MAXIV(enc_xform_aes_ctr);	/* SADB_X_EALG_AESCTR */
   1056 	MAXIV(enc_xform_null);		/* SADB_EALG_NULL */
   1057 
   1058 	xform_register(&esp_xformsw);
   1059 #undef MAXIV
   1060 }
   1061