xform_esp.c revision 1.73 1 /* $NetBSD: xform_esp.c,v 1.73 2018/01/24 13:54:16 maxv Exp $ */
2 /* $FreeBSD: src/sys/netipsec/xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */
3 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */
4
5 /*
6 * The authors of this code are John Ioannidis (ji (at) tla.org),
7 * Angelos D. Keromytis (kermit (at) csd.uch.gr) and
8 * Niels Provos (provos (at) physnet.uni-hamburg.de).
9 *
10 * The original version of this code was written by John Ioannidis
11 * for BSD/OS in Athens, Greece, in November 1995.
12 *
13 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
14 * by Angelos D. Keromytis.
15 *
16 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
17 * and Niels Provos.
18 *
19 * Additional features in 1999 by Angelos D. Keromytis.
20 *
21 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
22 * Angelos D. Keromytis and Niels Provos.
23 * Copyright (c) 2001 Angelos D. Keromytis.
24 *
25 * Permission to use, copy, and modify this software with or without fee
26 * is hereby granted, provided that this entire notice is included in
27 * all copies of any software which is or includes a copy or
28 * modification of this software.
29 * You may use this code under the GNU public license if you so wish. Please
30 * contribute changes back to the authors under this freer than GPL license
31 * so that we may further the use of strong encryption without limitations to
32 * all.
33 *
34 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
35 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
36 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
37 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
38 * PURPOSE.
39 */
40
41 #include <sys/cdefs.h>
42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.73 2018/01/24 13:54:16 maxv Exp $");
43
44 #if defined(_KERNEL_OPT)
45 #include "opt_inet.h"
46 #include "opt_ipsec.h"
47 #endif
48
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/mbuf.h>
52 #include <sys/socket.h>
53 #include <sys/syslog.h>
54 #include <sys/kernel.h>
55 #include <sys/sysctl.h>
56 #include <sys/cprng.h>
57 #include <sys/pool.h>
58 #include <sys/pserialize.h>
59
60 #include <net/if.h>
61
62 #include <netinet/in.h>
63 #include <netinet/in_systm.h>
64 #include <netinet/ip.h>
65 #include <netinet/ip_ecn.h>
66 #include <netinet/ip6.h>
67
68 #include <net/route.h>
69 #include <netipsec/ipsec.h>
70 #include <netipsec/ipsec_private.h>
71 #include <netipsec/ah.h>
72 #include <netipsec/ah_var.h>
73 #include <netipsec/esp.h>
74 #include <netipsec/esp_var.h>
75 #include <netipsec/xform.h>
76
77 #ifdef INET6
78 #include <netinet6/ip6_var.h>
79 #include <netipsec/ipsec6.h>
80 #endif
81
82 #include <netipsec/key.h>
83 #include <netipsec/key_debug.h>
84
85 #include <opencrypto/cryptodev.h>
86 #include <opencrypto/xform.h>
87
88 percpu_t *espstat_percpu;
89
90 int esp_enable = 1;
91
92 #ifdef __FreeBSD__
93 SYSCTL_DECL(_net_inet_esp);
94 SYSCTL_INT(_net_inet_esp, OID_AUTO,
95 esp_enable, CTLFLAG_RW, &esp_enable, 0, "");
96 SYSCTL_STRUCT(_net_inet_esp, IPSECCTL_STATS,
97 stats, CTLFLAG_RD, &espstat, espstat, "");
98 #endif /* __FreeBSD__ */
99
100 static int esp_max_ivlen; /* max iv length over all algorithms */
101
102 static int esp_input_cb(struct cryptop *op);
103 static int esp_output_cb(struct cryptop *crp);
104
105 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT };
106
107 static pool_cache_t esp_tdb_crypto_pool_cache;
108 static size_t esp_pool_item_size;
109
110 /*
111 * NB: this is public for use by the PF_KEY support.
112 * NB: if you add support here; be sure to add code to esp_attach below!
113 */
114 const struct enc_xform *
115 esp_algorithm_lookup(int alg)
116 {
117
118 switch (alg) {
119 case SADB_EALG_DESCBC:
120 return &enc_xform_des;
121 case SADB_EALG_3DESCBC:
122 return &enc_xform_3des;
123 case SADB_X_EALG_AES:
124 return &enc_xform_rijndael128;
125 case SADB_X_EALG_BLOWFISHCBC:
126 return &enc_xform_blf;
127 case SADB_X_EALG_CAST128CBC:
128 return &enc_xform_cast5;
129 case SADB_X_EALG_SKIPJACK:
130 return &enc_xform_skipjack;
131 case SADB_X_EALG_CAMELLIACBC:
132 return &enc_xform_camellia;
133 case SADB_X_EALG_AESCTR:
134 return &enc_xform_aes_ctr;
135 case SADB_X_EALG_AESGCM16:
136 return &enc_xform_aes_gcm;
137 case SADB_X_EALG_AESGMAC:
138 return &enc_xform_aes_gmac;
139 case SADB_EALG_NULL:
140 return &enc_xform_null;
141 }
142 return NULL;
143 }
144
145 size_t
146 esp_hdrsiz(const struct secasvar *sav)
147 {
148 size_t size;
149
150 if (sav != NULL) {
151 /*XXX not right for null algorithm--does it matter??*/
152 KASSERT(sav->tdb_encalgxform != NULL);
153 if (sav->flags & SADB_X_EXT_OLD)
154 size = sizeof(struct esp);
155 else
156 size = sizeof(struct newesp);
157 size += sav->tdb_encalgxform->ivsize + 9;
158 /*XXX need alg check???*/
159 if (sav->tdb_authalgxform != NULL && sav->replay)
160 size += ah_hdrsiz(sav);
161 } else {
162 /*
163 * base header size
164 * + max iv length for CBC mode
165 * + max pad length
166 * + sizeof(pad length field)
167 * + sizeof(next header field)
168 * + max icv supported.
169 */
170 size = sizeof(struct newesp) + esp_max_ivlen + 9 +
171 ah_hdrsiz(NULL);
172 }
173 return size;
174 }
175
176 /*
177 * esp_init() is called when an SPI is being set up.
178 */
179 static int
180 esp_init(struct secasvar *sav, const struct xformsw *xsp)
181 {
182 const struct enc_xform *txform;
183 struct cryptoini cria, crie, *cr;
184 int keylen;
185 int error;
186
187 txform = esp_algorithm_lookup(sav->alg_enc);
188 if (txform == NULL) {
189 DPRINTF(("%s: unsupported encryption algorithm %d\n", __func__,
190 sav->alg_enc));
191 return EINVAL;
192 }
193 if (sav->key_enc == NULL) {
194 DPRINTF(("%s: no encoding key for %s algorithm\n", __func__,
195 txform->name));
196 return EINVAL;
197 }
198 if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) {
199 DPRINTF(("%s: 4-byte IV not supported with protocol\n",
200 __func__));
201 return EINVAL;
202 }
203 keylen = _KEYLEN(sav->key_enc);
204 if (txform->minkey > keylen || keylen > txform->maxkey) {
205 DPRINTF(("%s: invalid key length %u, must be in "
206 "the range [%u..%u] for algorithm %s\n", __func__,
207 keylen, txform->minkey, txform->maxkey, txform->name));
208 return EINVAL;
209 }
210
211 sav->ivlen = txform->ivsize;
212
213 /*
214 * Setup AH-related state.
215 */
216 if (sav->alg_auth != 0) {
217 error = ah_init0(sav, xsp, &cria);
218 if (error)
219 return error;
220 }
221
222 /* NB: override anything set in ah_init0 */
223 sav->tdb_xform = xsp;
224 sav->tdb_encalgxform = txform;
225
226 switch (sav->alg_enc) {
227 case SADB_X_EALG_AESGCM16:
228 case SADB_X_EALG_AESGMAC:
229 switch (keylen) {
230 case 20:
231 sav->alg_auth = SADB_X_AALG_AES128GMAC;
232 sav->tdb_authalgxform = &auth_hash_gmac_aes_128;
233 break;
234 case 28:
235 sav->alg_auth = SADB_X_AALG_AES192GMAC;
236 sav->tdb_authalgxform = &auth_hash_gmac_aes_192;
237 break;
238 case 36:
239 sav->alg_auth = SADB_X_AALG_AES256GMAC;
240 sav->tdb_authalgxform = &auth_hash_gmac_aes_256;
241 break;
242 default:
243 DPRINTF(("%s: invalid key length %u, must be either of "
244 "20, 28 or 36\n", __func__, keylen));
245 return EINVAL;
246 }
247
248 memset(&cria, 0, sizeof(cria));
249 cria.cri_alg = sav->tdb_authalgxform->type;
250 cria.cri_klen = _KEYBITS(sav->key_enc);
251 cria.cri_key = _KEYBUF(sav->key_enc);
252 break;
253 default:
254 break;
255 }
256
257 /* Initialize crypto session. */
258 memset(&crie, 0, sizeof(crie));
259 crie.cri_alg = sav->tdb_encalgxform->type;
260 crie.cri_klen = _KEYBITS(sav->key_enc);
261 crie.cri_key = _KEYBUF(sav->key_enc);
262 /* XXX Rounds ? */
263
264 if (sav->tdb_authalgxform && sav->tdb_encalgxform) {
265 /* init both auth & enc */
266 crie.cri_next = &cria;
267 cr = &crie;
268 } else if (sav->tdb_encalgxform) {
269 cr = &crie;
270 } else if (sav->tdb_authalgxform) {
271 cr = &cria;
272 } else {
273 /* XXX cannot happen? */
274 DPRINTF(("%s: no encoding OR authentication xform!\n",
275 __func__));
276 return EINVAL;
277 }
278
279 return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support);
280 }
281
282 /*
283 * Paranoia.
284 */
285 static int
286 esp_zeroize(struct secasvar *sav)
287 {
288 /* NB: ah_zerorize free's the crypto session state */
289 int error = ah_zeroize(sav);
290
291 if (sav->key_enc) {
292 explicit_memset(_KEYBUF(sav->key_enc), 0,
293 _KEYLEN(sav->key_enc));
294 }
295 sav->tdb_encalgxform = NULL;
296 sav->tdb_xform = NULL;
297 return error;
298 }
299
300 /*
301 * ESP input processing, called (eventually) through the protocol switch.
302 */
303 static int
304 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
305 {
306 const struct auth_hash *esph;
307 const struct enc_xform *espx;
308 struct tdb_crypto *tc;
309 int plen, alen, hlen, error;
310 struct newesp *esp;
311
312 struct cryptodesc *crde;
313 struct cryptop *crp;
314
315 IPSEC_SPLASSERT_SOFTNET(__func__);
316
317 KASSERT(sav != NULL);
318 KASSERT(sav->tdb_encalgxform != NULL);
319 KASSERTMSG((skip&3) == 0 && (m->m_pkthdr.len&3) == 0,
320 "misaligned packet, skip %u pkt len %u",
321 skip, m->m_pkthdr.len);
322
323 /* XXX don't pullup, just copy header */
324 IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof(struct newesp));
325 if (esp == NULL) {
326 /* m already freed */
327 return EINVAL;
328 }
329
330 esph = sav->tdb_authalgxform;
331 espx = sav->tdb_encalgxform;
332
333 /* Determine the ESP header length */
334 if (sav->flags & SADB_X_EXT_OLD)
335 hlen = sizeof(struct esp) + sav->ivlen;
336 else
337 hlen = sizeof(struct newesp) + sav->ivlen;
338 /* Authenticator hash size */
339 alen = esph ? esph->authsize : 0;
340
341 /*
342 * Verify payload length is multiple of encryption algorithm
343 * block size.
344 *
345 * NB: This works for the null algorithm because the blocksize
346 * is 4 and all packets must be 4-byte aligned regardless
347 * of the algorithm.
348 */
349 plen = m->m_pkthdr.len - (skip + hlen + alen);
350 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
351 char buf[IPSEC_ADDRSTRLEN];
352 DPRINTF(("%s: payload of %d octets not a multiple of %d octets,"
353 " SA %s/%08lx\n", __func__, plen, espx->blocksize,
354 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
355 (u_long) ntohl(sav->spi)));
356 ESP_STATINC(ESP_STAT_BADILEN);
357 m_freem(m);
358 return EINVAL;
359 }
360
361 /*
362 * Check sequence number.
363 */
364 if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) {
365 char logbuf[IPSEC_LOGSASTRLEN];
366 DPRINTF(("%s: packet replay check for %s\n", __func__,
367 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); /*XXX*/
368 ESP_STATINC(ESP_STAT_REPLAY);
369 m_freem(m);
370 return ENOBUFS; /*XXX*/
371 }
372
373 /* Update the counters */
374 ESP_STATADD(ESP_STAT_IBYTES, m->m_pkthdr.len - skip - hlen - alen);
375
376 /* Get crypto descriptors */
377 crp = crypto_getreq(esph && espx ? 2 : 1);
378 if (crp == NULL) {
379 DPRINTF(("%s: failed to acquire crypto descriptors\n",
380 __func__));
381 error = ENOBUFS;
382 goto out;
383 }
384
385 /* Get IPsec-specific opaque pointer */
386 size_t extra __diagused = esph == NULL ? 0 : alen;
387 KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size,
388 "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n",
389 sizeof(*tc) + extra, esp_pool_item_size);
390 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
391 if (tc == NULL) {
392 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
393 error = ENOBUFS;
394 goto out1;
395 }
396
397 error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT);
398 if (error) {
399 DPRINTF(("%s: m_makewritable failed\n", __func__));
400 goto out2;
401 }
402
403 if (esph) {
404 struct cryptodesc *crda;
405
406 KASSERT(crp->crp_desc != NULL);
407 crda = crp->crp_desc;
408
409 /* Authentication descriptor */
410 crda->crd_skip = skip;
411 if (espx && espx->type == CRYPTO_AES_GCM_16)
412 crda->crd_len = hlen - sav->ivlen;
413 else
414 crda->crd_len = m->m_pkthdr.len - (skip + alen);
415 crda->crd_inject = m->m_pkthdr.len - alen;
416
417 crda->crd_alg = esph->type;
418 if (espx && (espx->type == CRYPTO_AES_GCM_16 ||
419 espx->type == CRYPTO_AES_GMAC)) {
420 crda->crd_key = _KEYBUF(sav->key_enc);
421 crda->crd_klen = _KEYBITS(sav->key_enc);
422 } else {
423 crda->crd_key = _KEYBUF(sav->key_auth);
424 crda->crd_klen = _KEYBITS(sav->key_auth);
425 }
426
427 /* Copy the authenticator */
428 m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1));
429
430 /* Chain authentication request */
431 crde = crda->crd_next;
432 } else {
433 crde = crp->crp_desc;
434 }
435
436 {
437 int s = pserialize_read_enter();
438
439 /*
440 * Take another reference to the SA for opencrypto callback.
441 */
442 if (__predict_false(sav->state == SADB_SASTATE_DEAD)) {
443 pserialize_read_exit(s);
444 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
445 crypto_freereq(crp);
446 ESP_STATINC(ESP_STAT_NOTDB);
447 return ENOENT;
448 }
449 KEY_SA_REF(sav);
450 pserialize_read_exit(s);
451 }
452
453 /* Crypto operation descriptor */
454 crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
455 crp->crp_flags = CRYPTO_F_IMBUF;
456 crp->crp_buf = m;
457 crp->crp_callback = esp_input_cb;
458 crp->crp_sid = sav->tdb_cryptoid;
459 crp->crp_opaque = tc;
460
461 /* These are passed as-is to the callback */
462 tc->tc_spi = sav->spi;
463 tc->tc_dst = sav->sah->saidx.dst;
464 tc->tc_proto = sav->sah->saidx.proto;
465 tc->tc_protoff = protoff;
466 tc->tc_skip = skip;
467 tc->tc_sav = sav;
468
469 /* Decryption descriptor */
470 if (espx) {
471 KASSERTMSG(crde != NULL, "null esp crypto descriptor");
472 crde->crd_skip = skip + hlen;
473 if (espx->type == CRYPTO_AES_GMAC)
474 crde->crd_len = 0;
475 else
476 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
477 crde->crd_inject = skip + hlen - sav->ivlen;
478
479 crde->crd_alg = espx->type;
480 crde->crd_key = _KEYBUF(sav->key_enc);
481 crde->crd_klen = _KEYBITS(sav->key_enc);
482 /* XXX Rounds ? */
483 }
484
485 return crypto_dispatch(crp);
486
487 out2:
488 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
489 out1:
490 crypto_freereq(crp);
491 out:
492 ESP_STATINC(ESP_STAT_CRYPTO);
493 m_freem(m);
494 return error;
495 }
496
497 #ifdef INET6
498 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do { \
499 if (saidx->dst.sa.sa_family == AF_INET6) { \
500 error = ipsec6_common_input_cb(m, sav, skip, protoff); \
501 } else { \
502 error = ipsec4_common_input_cb(m, sav, skip, protoff); \
503 } \
504 } while (0)
505 #else
506 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) \
507 (error = ipsec4_common_input_cb(m, sav, skip, protoff))
508 #endif
509
510 /*
511 * ESP input callback from the crypto driver.
512 */
513 static int
514 esp_input_cb(struct cryptop *crp)
515 {
516 char buf[IPSEC_ADDRSTRLEN];
517 uint8_t lastthree[3], aalg[AH_ALEN_MAX];
518 int hlen, skip, protoff, error;
519 struct mbuf *m;
520 const struct auth_hash *esph;
521 struct tdb_crypto *tc;
522 struct secasvar *sav;
523 struct secasindex *saidx;
524 void *ptr;
525 uint16_t dport;
526 uint16_t sport;
527 IPSEC_DECLARE_LOCK_VARIABLE;
528
529 KASSERT(crp->crp_desc != NULL);
530 KASSERT(crp->crp_opaque != NULL);
531
532 tc = crp->crp_opaque;
533 skip = tc->tc_skip;
534 protoff = tc->tc_protoff;
535 m = crp->crp_buf;
536
537 /* find the source port for NAT-T */
538 nat_t_ports_get(m, &dport, &sport);
539
540 IPSEC_ACQUIRE_GLOBAL_LOCKS();
541
542 sav = tc->tc_sav;
543 if (__predict_false(!SADB_SASTATE_USABLE_P(sav))) {
544 KEY_SA_UNREF(&sav);
545 sav = KEY_LOOKUP_SA(&tc->tc_dst, tc->tc_proto, tc->tc_spi,
546 sport, dport);
547 if (sav == NULL) {
548 ESP_STATINC(ESP_STAT_NOTDB);
549 DPRINTF(("%s: SA expired while in crypto "
550 "(SA %s/%08lx proto %u)\n", __func__,
551 ipsec_address(&tc->tc_dst, buf, sizeof(buf)),
552 (u_long) ntohl(tc->tc_spi), tc->tc_proto));
553 error = ENOBUFS; /*XXX*/
554 goto bad;
555 }
556 }
557
558 saidx = &sav->sah->saidx;
559 KASSERTMSG(saidx->dst.sa.sa_family == AF_INET ||
560 saidx->dst.sa.sa_family == AF_INET6,
561 "unexpected protocol family %u", saidx->dst.sa.sa_family);
562
563 esph = sav->tdb_authalgxform;
564
565 /* Check for crypto errors */
566 if (crp->crp_etype) {
567 /* Reset the session ID */
568 if (sav->tdb_cryptoid != 0)
569 sav->tdb_cryptoid = crp->crp_sid;
570
571 if (crp->crp_etype == EAGAIN) {
572 KEY_SA_UNREF(&sav);
573 IPSEC_RELEASE_GLOBAL_LOCKS();
574 return crypto_dispatch(crp);
575 }
576
577 ESP_STATINC(ESP_STAT_NOXFORM);
578 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
579 error = crp->crp_etype;
580 goto bad;
581 }
582
583 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]);
584
585 /* If authentication was performed, check now. */
586 if (esph != NULL) {
587 /*
588 * If we have a tag, it means an IPsec-aware NIC did
589 * the verification for us. Otherwise we need to
590 * check the authentication calculation.
591 */
592 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
593 /* Copy the authenticator from the packet */
594 m_copydata(m, m->m_pkthdr.len - esph->authsize,
595 esph->authsize, aalg);
596
597 ptr = (tc + 1);
598
599 /* Verify authenticator */
600 if (!consttime_memequal(ptr, aalg, esph->authsize)) {
601 DPRINTF(("%s: authentication hash mismatch "
602 "for packet in SA %s/%08lx\n", __func__,
603 ipsec_address(&saidx->dst, buf,
604 sizeof(buf)), (u_long) ntohl(sav->spi)));
605 ESP_STATINC(ESP_STAT_BADAUTH);
606 error = EACCES;
607 goto bad;
608 }
609
610 /* Remove trailing authenticator */
611 m_adj(m, -(esph->authsize));
612 }
613
614 /* Release the crypto descriptors */
615 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
616 tc = NULL;
617 crypto_freereq(crp), crp = NULL;
618
619 /*
620 * Packet is now decrypted.
621 */
622 m->m_flags |= M_DECRYPTED;
623
624 /*
625 * Update replay sequence number, if appropriate.
626 */
627 if (sav->replay) {
628 uint32_t seq;
629
630 m_copydata(m, skip + offsetof(struct newesp, esp_seq),
631 sizeof(seq), &seq);
632 if (ipsec_updatereplay(ntohl(seq), sav)) {
633 char logbuf[IPSEC_LOGSASTRLEN];
634 DPRINTF(("%s: packet replay check for %s\n", __func__,
635 ipsec_logsastr(sav, logbuf, sizeof(logbuf))));
636 ESP_STATINC(ESP_STAT_REPLAY);
637 error = ENOBUFS;
638 goto bad;
639 }
640 }
641
642 /* Determine the ESP header length */
643 if (sav->flags & SADB_X_EXT_OLD)
644 hlen = sizeof(struct esp) + sav->ivlen;
645 else
646 hlen = sizeof(struct newesp) + sav->ivlen;
647
648 /* Remove the ESP header and IV from the mbuf. */
649 error = m_striphdr(m, skip, hlen);
650 if (error) {
651 ESP_STATINC(ESP_STAT_HDROPS);
652 DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__,
653 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
654 (u_long) ntohl(sav->spi)));
655 goto bad;
656 }
657
658 /* Save the last three bytes of decrypted data */
659 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree);
660
661 /* Verify pad length */
662 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) {
663 ESP_STATINC(ESP_STAT_BADILEN);
664 DPRINTF(("%s: invalid padding length %d "
665 "for %u byte packet in SA %s/%08lx\n", __func__,
666 lastthree[1], m->m_pkthdr.len - skip,
667 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
668 (u_long) ntohl(sav->spi)));
669 error = EINVAL;
670 goto bad;
671 }
672
673 /* Verify correct decryption by checking the last padding bytes */
674 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) {
675 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) {
676 ESP_STATINC(ESP_STAT_BADENC);
677 DPRINTF(("%s: decryption failed for packet in SA "
678 "%s/%08lx\n", __func__,
679 ipsec_address(&sav->sah->saidx.dst, buf,
680 sizeof(buf)), (u_long) ntohl(sav->spi)));
681 DPRINTF(("%s: %x %x\n", __func__, lastthree[0],
682 lastthree[1]));
683 error = EINVAL;
684 goto bad;
685 }
686 }
687
688 /* Trim the mbuf chain to remove trailing authenticator and padding */
689 m_adj(m, -(lastthree[1] + 2));
690
691 /* Restore the Next Protocol field */
692 m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2);
693
694 IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff);
695
696 KEY_SA_UNREF(&sav);
697 IPSEC_RELEASE_GLOBAL_LOCKS();
698 return error;
699 bad:
700 if (sav)
701 KEY_SA_UNREF(&sav);
702 IPSEC_RELEASE_GLOBAL_LOCKS();
703 if (m != NULL)
704 m_freem(m);
705 if (tc != NULL)
706 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
707 if (crp != NULL)
708 crypto_freereq(crp);
709 return error;
710 }
711
712 /*
713 * ESP output routine, called by ipsec[46]_process_packet().
714 */
715 static int
716 esp_output(
717 struct mbuf *m,
718 const struct ipsecrequest *isr,
719 struct secasvar *sav,
720 struct mbuf **mp,
721 int skip,
722 int protoff
723 )
724 {
725 char buf[IPSEC_ADDRSTRLEN];
726 const struct enc_xform *espx;
727 const struct auth_hash *esph;
728 int hlen, rlen, padding, blks, alen, i, roff;
729 struct mbuf *mo = NULL;
730 struct tdb_crypto *tc;
731 struct secasindex *saidx;
732 unsigned char *pad;
733 uint8_t prot;
734 int error, maxpacketsize;
735
736 struct cryptodesc *crde = NULL, *crda = NULL;
737 struct cryptop *crp;
738
739 IPSEC_SPLASSERT_SOFTNET(__func__);
740
741 esph = sav->tdb_authalgxform;
742 KASSERT(sav->tdb_encalgxform != NULL);
743 espx = sav->tdb_encalgxform;
744
745 if (sav->flags & SADB_X_EXT_OLD)
746 hlen = sizeof(struct esp) + sav->ivlen;
747 else
748 hlen = sizeof(struct newesp) + sav->ivlen;
749
750 rlen = m->m_pkthdr.len - skip; /* Raw payload length. */
751 /*
752 * NB: The null encoding transform has a blocksize of 4
753 * so that headers are properly aligned.
754 */
755 blks = espx->blocksize; /* IV blocksize */
756
757 /* XXX clamp padding length a la KAME??? */
758 padding = ((blks - ((rlen + 2) % blks)) % blks) + 2;
759
760 if (esph)
761 alen = esph->authsize;
762 else
763 alen = 0;
764
765 ESP_STATINC(ESP_STAT_OUTPUT);
766
767 saidx = &sav->sah->saidx;
768 /* Check for maximum packet size violations. */
769 switch (saidx->dst.sa.sa_family) {
770 #ifdef INET
771 case AF_INET:
772 maxpacketsize = IP_MAXPACKET;
773 break;
774 #endif /* INET */
775 #ifdef INET6
776 case AF_INET6:
777 maxpacketsize = IPV6_MAXPACKET;
778 break;
779 #endif /* INET6 */
780 default:
781 DPRINTF(("%s: unknown/unsupported protocol family %d, "
782 "SA %s/%08lx\n", __func__, saidx->dst.sa.sa_family,
783 ipsec_address(&saidx->dst, buf, sizeof(buf)),
784 (u_long)ntohl(sav->spi)));
785 ESP_STATINC(ESP_STAT_NOPF);
786 error = EPFNOSUPPORT;
787 goto bad;
788 }
789 if (skip + hlen + rlen + padding + alen > maxpacketsize) {
790 DPRINTF(("%s: packet in SA %s/%08lx got too big (len %u, "
791 "max len %u)\n", __func__,
792 ipsec_address(&saidx->dst, buf, sizeof(buf)),
793 (u_long) ntohl(sav->spi),
794 skip + hlen + rlen + padding + alen, maxpacketsize));
795 ESP_STATINC(ESP_STAT_TOOBIG);
796 error = EMSGSIZE;
797 goto bad;
798 }
799
800 /* Update the counters. */
801 ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip);
802
803 m = m_clone(m);
804 if (m == NULL) {
805 DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__,
806 ipsec_address(&saidx->dst, buf, sizeof(buf)),
807 (u_long) ntohl(sav->spi)));
808 ESP_STATINC(ESP_STAT_HDROPS);
809 error = ENOBUFS;
810 goto bad;
811 }
812
813 /* Inject ESP header. */
814 mo = m_makespace(m, skip, hlen, &roff);
815 if (mo == NULL) {
816 DPRINTF(("%s: failed to inject %u byte ESP hdr for SA "
817 "%s/%08lx\n", __func__, hlen,
818 ipsec_address(&saidx->dst, buf, sizeof(buf)),
819 (u_long) ntohl(sav->spi)));
820 ESP_STATINC(ESP_STAT_HDROPS); /* XXX diffs from openbsd */
821 error = ENOBUFS;
822 goto bad;
823 }
824
825 /* Initialize ESP header. */
826 memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t));
827 if (sav->replay) {
828 uint32_t replay;
829
830 #ifdef IPSEC_DEBUG
831 /* Emulate replay attack when ipsec_replay is TRUE. */
832 if (!ipsec_replay)
833 #endif
834 sav->replay->count++;
835
836 replay = htonl(sav->replay->count);
837 memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay,
838 sizeof(uint32_t));
839 }
840
841 /*
842 * Add padding -- better to do it ourselves than use the crypto engine,
843 * although if/when we support compression, we'd have to do that.
844 */
845 pad = m_pad(m, padding + alen);
846 if (pad == NULL) {
847 DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__,
848 ipsec_address(&saidx->dst, buf, sizeof(buf)),
849 (u_long) ntohl(sav->spi)));
850 m = NULL; /* NB: free'd by m_pad */
851 error = ENOBUFS;
852 goto bad;
853 }
854
855 /*
856 * Add padding: random, zero, or self-describing.
857 * XXX catch unexpected setting
858 */
859 switch (sav->flags & SADB_X_EXT_PMASK) {
860 case SADB_X_EXT_PRAND:
861 (void) cprng_fast(pad, padding - 2);
862 break;
863 case SADB_X_EXT_PZERO:
864 memset(pad, 0, padding - 2);
865 break;
866 case SADB_X_EXT_PSEQ:
867 for (i = 0; i < padding - 2; i++)
868 pad[i] = i+1;
869 break;
870 }
871
872 /* Fix padding length and Next Protocol in padding itself. */
873 pad[padding - 2] = padding - 2;
874 m_copydata(m, protoff, sizeof(uint8_t), pad + padding - 1);
875
876 /* Fix Next Protocol in IPv4/IPv6 header. */
877 prot = IPPROTO_ESP;
878 m_copyback(m, protoff, sizeof(uint8_t), &prot);
879
880 /* Get crypto descriptors. */
881 crp = crypto_getreq(esph && espx ? 2 : 1);
882 if (crp == NULL) {
883 DPRINTF(("%s: failed to acquire crypto descriptors\n",
884 __func__));
885 ESP_STATINC(ESP_STAT_CRYPTO);
886 error = ENOBUFS;
887 goto bad;
888 }
889
890 if (espx) {
891 crde = crp->crp_desc;
892 crda = crde->crd_next;
893
894 /* Encryption descriptor. */
895 crde->crd_skip = skip + hlen;
896 if (espx->type == CRYPTO_AES_GMAC)
897 crde->crd_len = 0;
898 else
899 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
900 crde->crd_flags = CRD_F_ENCRYPT;
901 crde->crd_inject = skip + hlen - sav->ivlen;
902
903 /* Encryption operation. */
904 crde->crd_alg = espx->type;
905 crde->crd_key = _KEYBUF(sav->key_enc);
906 crde->crd_klen = _KEYBITS(sav->key_enc);
907 /* XXX Rounds ? */
908 } else
909 crda = crp->crp_desc;
910
911 /* IPsec-specific opaque crypto info. */
912 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT);
913 if (tc == NULL) {
914 crypto_freereq(crp);
915 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__));
916 ESP_STATINC(ESP_STAT_CRYPTO);
917 error = ENOBUFS;
918 goto bad;
919 }
920
921 {
922 int s = pserialize_read_enter();
923
924 /*
925 * Take another reference to the SP and the SA for opencrypto callback.
926 */
927 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD ||
928 sav->state == SADB_SASTATE_DEAD)) {
929 pserialize_read_exit(s);
930 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
931 crypto_freereq(crp);
932 ESP_STATINC(ESP_STAT_NOTDB);
933 error = ENOENT;
934 goto bad;
935 }
936 KEY_SP_REF(isr->sp);
937 KEY_SA_REF(sav);
938 pserialize_read_exit(s);
939 }
940
941 /* Callback parameters */
942 tc->tc_isr = isr;
943 tc->tc_spi = sav->spi;
944 tc->tc_dst = saidx->dst;
945 tc->tc_proto = saidx->proto;
946 tc->tc_sav = sav;
947
948 /* Crypto operation descriptor. */
949 crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
950 crp->crp_flags = CRYPTO_F_IMBUF;
951 crp->crp_buf = m;
952 crp->crp_callback = esp_output_cb;
953 crp->crp_opaque = tc;
954 crp->crp_sid = sav->tdb_cryptoid;
955
956 if (esph) {
957 /* Authentication descriptor. */
958 crda->crd_skip = skip;
959 if (espx && espx->type == CRYPTO_AES_GCM_16)
960 crda->crd_len = hlen - sav->ivlen;
961 else
962 crda->crd_len = m->m_pkthdr.len - (skip + alen);
963 crda->crd_inject = m->m_pkthdr.len - alen;
964
965 /* Authentication operation. */
966 crda->crd_alg = esph->type;
967 if (espx && (espx->type == CRYPTO_AES_GCM_16 ||
968 espx->type == CRYPTO_AES_GMAC)) {
969 crda->crd_key = _KEYBUF(sav->key_enc);
970 crda->crd_klen = _KEYBITS(sav->key_enc);
971 } else {
972 crda->crd_key = _KEYBUF(sav->key_auth);
973 crda->crd_klen = _KEYBITS(sav->key_auth);
974 }
975 }
976
977 return crypto_dispatch(crp);
978 bad:
979 if (m)
980 m_freem(m);
981 return (error);
982 }
983
984 /*
985 * ESP output callback from the crypto driver.
986 */
987 static int
988 esp_output_cb(struct cryptop *crp)
989 {
990 struct tdb_crypto *tc;
991 const struct ipsecrequest *isr;
992 struct secasvar *sav;
993 struct mbuf *m;
994 int err, error;
995 IPSEC_DECLARE_LOCK_VARIABLE;
996
997 KASSERT(crp->crp_opaque != NULL);
998 tc = crp->crp_opaque;
999 m = crp->crp_buf;
1000
1001 IPSEC_ACQUIRE_GLOBAL_LOCKS();
1002
1003 isr = tc->tc_isr;
1004 sav = tc->tc_sav;
1005 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD)) {
1006 ESP_STATINC(ESP_STAT_NOTDB);
1007 IPSECLOG(LOG_DEBUG,
1008 "SP is being destroyed while in crypto (id=%u)\n",
1009 isr->sp->id);
1010 error = ENOENT;
1011 goto bad;
1012 }
1013 if (__predict_false(!SADB_SASTATE_USABLE_P(sav))) {
1014 KEY_SA_UNREF(&sav);
1015 sav = KEY_LOOKUP_SA(&tc->tc_dst, tc->tc_proto, tc->tc_spi, 0, 0);
1016 if (sav == NULL) {
1017 char buf[IPSEC_ADDRSTRLEN];
1018 ESP_STATINC(ESP_STAT_NOTDB);
1019 DPRINTF(("%s: SA expired while in crypto (SA %s/%08lx "
1020 "proto %u)\n", __func__,
1021 ipsec_address(&tc->tc_dst, buf, sizeof(buf)),
1022 (u_long) ntohl(tc->tc_spi), tc->tc_proto));
1023 error = ENOBUFS; /*XXX*/
1024 goto bad;
1025 }
1026 }
1027
1028 /* Check for crypto errors. */
1029 if (crp->crp_etype) {
1030 /* Reset session ID. */
1031 if (sav->tdb_cryptoid != 0)
1032 sav->tdb_cryptoid = crp->crp_sid;
1033
1034 if (crp->crp_etype == EAGAIN) {
1035 IPSEC_RELEASE_GLOBAL_LOCKS();
1036 return crypto_dispatch(crp);
1037 }
1038
1039 ESP_STATINC(ESP_STAT_NOXFORM);
1040 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
1041 error = crp->crp_etype;
1042 goto bad;
1043 }
1044
1045 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]);
1046 if (sav->tdb_authalgxform != NULL)
1047 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]);
1048
1049 /* Release crypto descriptors. */
1050 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
1051 crypto_freereq(crp);
1052
1053 #ifdef IPSEC_DEBUG
1054 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */
1055 if (ipsec_integrity) {
1056 static unsigned char ipseczeroes[AH_ALEN_MAX];
1057 const struct auth_hash *esph;
1058
1059 /*
1060 * Corrupt HMAC if we want to test integrity verification of
1061 * the other side.
1062 */
1063 esph = sav->tdb_authalgxform;
1064 if (esph != NULL) {
1065 m_copyback(m, m->m_pkthdr.len - esph->authsize,
1066 esph->authsize, ipseczeroes);
1067 }
1068 }
1069 #endif
1070
1071 /* NB: m is reclaimed by ipsec_process_done. */
1072 err = ipsec_process_done(m, isr, sav);
1073 KEY_SA_UNREF(&sav);
1074 KEY_SP_UNREF(&isr->sp);
1075 IPSEC_RELEASE_GLOBAL_LOCKS();
1076 return err;
1077 bad:
1078 if (sav)
1079 KEY_SA_UNREF(&sav);
1080 KEY_SP_UNREF(&isr->sp);
1081 IPSEC_RELEASE_GLOBAL_LOCKS();
1082 if (m)
1083 m_freem(m);
1084 pool_cache_put(esp_tdb_crypto_pool_cache, tc);
1085 crypto_freereq(crp);
1086 return error;
1087 }
1088
1089 static struct xformsw esp_xformsw = {
1090 .xf_type = XF_ESP,
1091 .xf_flags = XFT_CONF|XFT_AUTH,
1092 .xf_name = "IPsec ESP",
1093 .xf_init = esp_init,
1094 .xf_zeroize = esp_zeroize,
1095 .xf_input = esp_input,
1096 .xf_output = esp_output,
1097 .xf_next = NULL,
1098 };
1099
1100 void
1101 esp_attach(void)
1102 {
1103
1104 espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS);
1105
1106 extern int ah_max_authsize;
1107 KASSERT(ah_max_authsize != 0);
1108 esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize;
1109 esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size,
1110 coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET,
1111 NULL, NULL, NULL);
1112
1113 #define MAXIV(xform) \
1114 if (xform.ivsize > esp_max_ivlen) \
1115 esp_max_ivlen = xform.ivsize \
1116
1117 esp_max_ivlen = 0;
1118 MAXIV(enc_xform_des); /* SADB_EALG_DESCBC */
1119 MAXIV(enc_xform_3des); /* SADB_EALG_3DESCBC */
1120 MAXIV(enc_xform_rijndael128); /* SADB_X_EALG_AES */
1121 MAXIV(enc_xform_blf); /* SADB_X_EALG_BLOWFISHCBC */
1122 MAXIV(enc_xform_cast5); /* SADB_X_EALG_CAST128CBC */
1123 MAXIV(enc_xform_skipjack); /* SADB_X_EALG_SKIPJACK */
1124 MAXIV(enc_xform_camellia); /* SADB_X_EALG_CAMELLIACBC */
1125 MAXIV(enc_xform_aes_ctr); /* SADB_X_EALG_AESCTR */
1126 MAXIV(enc_xform_null); /* SADB_EALG_NULL */
1127
1128 xform_register(&esp_xformsw);
1129 #undef MAXIV
1130 }
1131