1 /* $NetBSD: xform_esp.c,v 1.107 2024/07/05 04:31:54 rin Exp $ */ 2 /* $FreeBSD: xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */ 3 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ 4 5 /* 6 * The authors of this code are John Ioannidis (ji (at) tla.org), 7 * Angelos D. Keromytis (kermit (at) csd.uch.gr) and 8 * Niels Provos (provos (at) physnet.uni-hamburg.de). 9 * 10 * The original version of this code was written by John Ioannidis 11 * for BSD/OS in Athens, Greece, in November 1995. 12 * 13 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, 14 * by Angelos D. Keromytis. 15 * 16 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis 17 * and Niels Provos. 18 * 19 * Additional features in 1999 by Angelos D. Keromytis. 20 * 21 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, 22 * Angelos D. Keromytis and Niels Provos. 23 * Copyright (c) 2001 Angelos D. Keromytis. 24 * 25 * Permission to use, copy, and modify this software with or without fee 26 * is hereby granted, provided that this entire notice is included in 27 * all copies of any software which is or includes a copy or 28 * modification of this software. 29 * You may use this code under the GNU public license if you so wish. Please 30 * contribute changes back to the authors under this freer than GPL license 31 * so that we may further the use of strong encryption without limitations to 32 * all. 33 * 34 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 35 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 36 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 37 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 38 * PURPOSE. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.107 2024/07/05 04:31:54 rin Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_inet.h" 46 #include "opt_ipsec.h" 47 #endif 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/mbuf.h> 52 #include <sys/socket.h> 53 #include <sys/syslog.h> 54 #include <sys/kernel.h> 55 #include <sys/sysctl.h> 56 #include <sys/cprng.h> 57 #include <sys/pool.h> 58 #include <sys/pserialize.h> 59 60 #include <net/if.h> 61 62 #include <netinet/in.h> 63 #include <netinet/in_systm.h> 64 #include <netinet/ip.h> 65 #include <netinet/ip_ecn.h> 66 #include <netinet/ip6.h> 67 68 #include <net/route.h> 69 #include <netipsec/ipsec.h> 70 #include <netipsec/ipsec_private.h> 71 #include <netipsec/ah.h> 72 #include <netipsec/ah_var.h> 73 #include <netipsec/esp.h> 74 #include <netipsec/esp_var.h> 75 #include <netipsec/xform.h> 76 77 #ifdef INET6 78 #include <netinet6/ip6_var.h> 79 #include <netipsec/ipsec6.h> 80 #endif 81 82 #include <netipsec/key.h> 83 #include <netipsec/key_debug.h> 84 85 #include <opencrypto/cryptodev.h> 86 87 percpu_t *espstat_percpu; 88 89 int esp_enable = 1; 90 91 static int esp_max_ivlen; /* max iv length over all algorithms */ 92 93 static void esp_input_cb(struct cryptop *op); 94 static void esp_output_cb(struct cryptop *crp); 95 96 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT }; 97 98 static pool_cache_t esp_tdb_crypto_pool_cache; 99 static size_t esp_pool_item_size; 100 101 /* 102 * NB: this is public for use by the PF_KEY support. 103 * NB: if you add support here; be sure to add code to esp_attach below! 104 */ 105 const struct enc_xform * 106 esp_algorithm_lookup(int alg) 107 { 108 109 switch (alg) { 110 case SADB_EALG_DESCBC: 111 return &enc_xform_des; 112 case SADB_EALG_3DESCBC: 113 return &enc_xform_3des; 114 case SADB_X_EALG_AES: 115 return &enc_xform_aes; 116 case SADB_X_EALG_BLOWFISHCBC: 117 return &enc_xform_blf; 118 case SADB_X_EALG_CAST128CBC: 119 return &enc_xform_cast5; 120 case SADB_X_EALG_SKIPJACK: 121 return &enc_xform_skipjack; 122 case SADB_X_EALG_CAMELLIACBC: 123 return &enc_xform_camellia; 124 case SADB_X_EALG_AESCTR: 125 return &enc_xform_aes_ctr; 126 case SADB_X_EALG_AESGCM16: 127 return &enc_xform_aes_gcm; 128 case SADB_X_EALG_AESGMAC: 129 return &enc_xform_aes_gmac; 130 case SADB_EALG_NULL: 131 return &enc_xform_null; 132 } 133 return NULL; 134 } 135 136 size_t 137 esp_hdrsiz(const struct secasvar *sav) 138 { 139 size_t size; 140 141 if (sav != NULL) { 142 /*XXX not right for null algorithm--does it matter??*/ 143 KASSERT(sav->tdb_encalgxform != NULL); 144 145 /* 146 * base header size 147 * + iv length for CBC mode 148 * + max pad length 149 * + sizeof(esp trailer) 150 * + icv length (if any). 151 */ 152 if (sav->flags & SADB_X_EXT_OLD) 153 size = sizeof(struct esp); 154 else 155 size = sizeof(struct newesp); 156 size += sav->tdb_encalgxform->ivsize + 9 + 157 sizeof(struct esptail); 158 159 /*XXX need alg check???*/ 160 if (sav->tdb_authalgxform != NULL && sav->replay) 161 size += ah_authsiz(sav); 162 } else { 163 /* 164 * base header size 165 * + max iv length for CBC mode 166 * + max pad length 167 * + sizeof(esp trailer) 168 * + max icv supported. 169 */ 170 size = sizeof(struct newesp) + esp_max_ivlen + 9 + 171 sizeof(struct esptail) + ah_authsiz(NULL); 172 } 173 return size; 174 } 175 176 /* 177 * esp_init() is called when an SPI is being set up. 178 */ 179 static int 180 esp_init(struct secasvar *sav, const struct xformsw *xsp) 181 { 182 const struct enc_xform *txform; 183 struct cryptoini cria, crie, *cr; 184 int keylen; 185 int error; 186 187 txform = esp_algorithm_lookup(sav->alg_enc); 188 if (txform == NULL) { 189 DPRINTF("unsupported encryption algorithm %d\n", 190 sav->alg_enc); 191 return EINVAL; 192 } 193 if (sav->key_enc == NULL) { 194 DPRINTF("no encoding key for %s algorithm\n", 195 txform->name); 196 return EINVAL; 197 } 198 if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) { 199 DPRINTF("4-byte IV not supported with protocol\n"); 200 return EINVAL; 201 } 202 keylen = _KEYLEN(sav->key_enc); 203 if (txform->minkey > keylen || keylen > txform->maxkey) { 204 DPRINTF("invalid key length %u, must be in " 205 "the range [%u..%u] for algorithm %s\n", 206 keylen, txform->minkey, txform->maxkey, txform->name); 207 return EINVAL; 208 } 209 210 sav->ivlen = txform->ivsize; 211 212 /* 213 * Setup AH-related state. 214 */ 215 if (sav->alg_auth != 0) { 216 error = ah_init0(sav, xsp, &cria); 217 if (error) 218 return error; 219 } 220 221 /* NB: override anything set in ah_init0 */ 222 sav->tdb_xform = xsp; 223 sav->tdb_encalgxform = txform; 224 225 switch (sav->alg_enc) { 226 case SADB_X_EALG_AESGCM16: 227 case SADB_X_EALG_AESGMAC: 228 switch (keylen) { 229 case 20: 230 sav->alg_auth = SADB_X_AALG_AES128GMAC; 231 sav->tdb_authalgxform = &auth_hash_gmac_aes_128; 232 break; 233 case 28: 234 sav->alg_auth = SADB_X_AALG_AES192GMAC; 235 sav->tdb_authalgxform = &auth_hash_gmac_aes_192; 236 break; 237 case 36: 238 sav->alg_auth = SADB_X_AALG_AES256GMAC; 239 sav->tdb_authalgxform = &auth_hash_gmac_aes_256; 240 break; 241 default: 242 DPRINTF("invalid key length %u, must be either of " 243 "20, 28 or 36\n", keylen); 244 return EINVAL; 245 } 246 247 memset(&cria, 0, sizeof(cria)); 248 cria.cri_alg = sav->tdb_authalgxform->type; 249 cria.cri_klen = _KEYBITS(sav->key_enc); 250 cria.cri_key = _KEYBUF(sav->key_enc); 251 break; 252 default: 253 break; 254 } 255 256 /* Initialize crypto session. */ 257 memset(&crie, 0, sizeof(crie)); 258 crie.cri_alg = sav->tdb_encalgxform->type; 259 crie.cri_klen = _KEYBITS(sav->key_enc); 260 crie.cri_key = _KEYBUF(sav->key_enc); 261 /* XXX Rounds ? */ 262 263 if (sav->tdb_authalgxform && sav->tdb_encalgxform) { 264 /* init both auth & enc */ 265 crie.cri_next = &cria; 266 cr = &crie; 267 } else if (sav->tdb_encalgxform) { 268 cr = &crie; 269 } else if (sav->tdb_authalgxform) { 270 cr = &cria; 271 } else { 272 /* XXX cannot happen? */ 273 DPRINTF("no encoding OR authentication xform!\n"); 274 return EINVAL; 275 } 276 277 return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support); 278 } 279 280 /* 281 * Paranoia. 282 */ 283 static void 284 esp_zeroize(struct secasvar *sav) 285 { 286 /* NB: ah_zerorize free's the crypto session state */ 287 ah_zeroize(sav); 288 289 if (sav->key_enc) { 290 explicit_memset(_KEYBUF(sav->key_enc), 0, 291 _KEYLEN(sav->key_enc)); 292 } 293 sav->tdb_encalgxform = NULL; 294 sav->tdb_xform = NULL; 295 } 296 297 /* 298 * ESP input processing, called (eventually) through the protocol switch. 299 */ 300 static int 301 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) 302 { 303 const struct auth_hash *esph; 304 const struct enc_xform *espx; 305 struct tdb_crypto *tc; 306 int plen, alen, hlen, error, stat = ESP_STAT_CRYPTO; 307 struct newesp *esp; 308 struct cryptodesc *crde; 309 struct cryptop *crp; 310 311 KASSERT(sav != NULL); 312 KASSERT(sav->tdb_encalgxform != NULL); 313 if (__predict_false((skip & 3) != 0 || (m->m_pkthdr.len & 3) != 0)) { 314 DPRINTF("%s: misaligned packet, skip %u pkt len %u", __func__, 315 skip, m->m_pkthdr.len); 316 stat = ESP_STAT_BADILEN; /* Same as FreeBSD */ 317 error = EINVAL; 318 goto out; 319 } 320 321 /* XXX don't pullup, just copy header */ 322 M_REGION_GET(esp, struct newesp *, m, skip, sizeof(struct newesp)); 323 if (esp == NULL) { 324 /* m already freed */ 325 return ENOBUFS; 326 } 327 328 esph = sav->tdb_authalgxform; 329 espx = sav->tdb_encalgxform; 330 KASSERT(espx != NULL); 331 332 /* Determine the ESP header length */ 333 if (sav->flags & SADB_X_EXT_OLD) 334 hlen = sizeof(struct esp) + sav->ivlen; 335 else 336 hlen = sizeof(struct newesp) + sav->ivlen; 337 /* Authenticator hash size */ 338 alen = esph ? esph->authsize : 0; 339 340 /* 341 * Verify payload length is multiple of encryption algorithm block 342 * size. 343 * 344 * The payload must also be 4-byte-aligned. This is implicitly 345 * verified here too, since the blocksize is always 4-byte-aligned. 346 */ 347 plen = m->m_pkthdr.len - (skip + hlen + alen); 348 KASSERT((espx->blocksize & 3) == 0); 349 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) { 350 char buf[IPSEC_ADDRSTRLEN]; 351 DPRINTF("payload of %d octets not a multiple of %d octets," 352 " SA %s/%08lx\n", plen, espx->blocksize, 353 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 354 (u_long) ntohl(sav->spi)); 355 stat = ESP_STAT_BADILEN; 356 error = EINVAL; 357 goto out; 358 } 359 360 /* 361 * Check sequence number. 362 */ 363 if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) { 364 char logbuf[IPSEC_LOGSASTRLEN]; 365 DPRINTF("packet replay check for %s\n", 366 ipsec_logsastr(sav, logbuf, sizeof(logbuf))); 367 stat = ESP_STAT_REPLAY; 368 error = EACCES; 369 goto out; 370 } 371 372 /* Update the counters */ 373 ESP_STATADD(ESP_STAT_IBYTES, plen); 374 375 /* Get crypto descriptors */ 376 crp = crypto_getreq(esph ? 2 : 1); 377 if (crp == NULL) { 378 DPRINTF("failed to acquire crypto descriptors\n"); 379 error = ENOBUFS; 380 goto out; 381 } 382 383 /* Get IPsec-specific opaque pointer */ 384 size_t extra __diagused = esph == NULL ? 0 : alen; 385 KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size, 386 "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n", 387 sizeof(*tc) + extra, esp_pool_item_size); 388 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 389 if (tc == NULL) { 390 DPRINTF("failed to allocate tdb_crypto\n"); 391 error = ENOBUFS; 392 goto out1; 393 } 394 395 error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT); 396 if (error) { 397 DPRINTF("m_makewritable failed\n"); 398 goto out2; 399 } 400 401 if (esph) { 402 struct cryptodesc *crda; 403 404 KASSERT(crp->crp_desc != NULL); 405 crda = crp->crp_desc; 406 407 /* Authentication descriptor */ 408 crda->crd_skip = skip; 409 if (espx->type == CRYPTO_AES_GCM_16) 410 crda->crd_len = hlen - sav->ivlen; 411 else 412 crda->crd_len = m->m_pkthdr.len - (skip + alen); 413 crda->crd_inject = m->m_pkthdr.len - alen; 414 415 crda->crd_alg = esph->type; 416 if (espx->type == CRYPTO_AES_GCM_16 || 417 espx->type == CRYPTO_AES_GMAC) { 418 crda->crd_key = _KEYBUF(sav->key_enc); 419 crda->crd_klen = _KEYBITS(sav->key_enc); 420 } else { 421 crda->crd_key = _KEYBUF(sav->key_auth); 422 crda->crd_klen = _KEYBITS(sav->key_auth); 423 } 424 425 /* Copy the authenticator */ 426 m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1)); 427 428 /* Chain authentication request */ 429 crde = crda->crd_next; 430 } else { 431 crde = crp->crp_desc; 432 } 433 434 { 435 int s = pserialize_read_enter(); 436 437 /* 438 * Take another reference to the SA for opencrypto callback. 439 */ 440 if (__predict_false(sav->state == SADB_SASTATE_DEAD)) { 441 pserialize_read_exit(s); 442 stat = ESP_STAT_NOTDB; 443 error = ENOENT; 444 goto out2; 445 } 446 KEY_SA_REF(sav); 447 pserialize_read_exit(s); 448 } 449 450 /* Crypto operation descriptor */ 451 crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ 452 crp->crp_flags = CRYPTO_F_IMBUF; 453 crp->crp_buf = m; 454 crp->crp_callback = esp_input_cb; 455 crp->crp_sid = sav->tdb_cryptoid; 456 crp->crp_opaque = tc; 457 458 /* These are passed as-is to the callback */ 459 tc->tc_spi = sav->spi; 460 tc->tc_dst = sav->sah->saidx.dst; 461 tc->tc_proto = sav->sah->saidx.proto; 462 tc->tc_protoff = protoff; 463 tc->tc_skip = skip; 464 tc->tc_sav = sav; 465 466 /* Decryption descriptor */ 467 KASSERTMSG(crde != NULL, "null esp crypto descriptor"); 468 crde->crd_skip = skip + hlen; 469 if (espx->type == CRYPTO_AES_GMAC) 470 crde->crd_len = 0; 471 else 472 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 473 crde->crd_inject = skip + hlen - sav->ivlen; 474 crde->crd_alg = espx->type; 475 crde->crd_key = _KEYBUF(sav->key_enc); 476 crde->crd_klen = _KEYBITS(sav->key_enc); 477 /* XXX Rounds ? */ 478 479 crypto_dispatch(crp); 480 return 0; 481 482 out2: 483 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 484 out1: 485 crypto_freereq(crp); 486 out: 487 ESP_STATINC(stat); 488 m_freem(m); 489 return error; 490 } 491 492 #ifdef INET6 493 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do { \ 494 if (saidx->dst.sa.sa_family == AF_INET6) { \ 495 (void)ipsec6_common_input_cb(m, sav, skip, protoff); \ 496 } else { \ 497 (void)ipsec4_common_input_cb(m, sav, skip, protoff); \ 498 } \ 499 } while (0) 500 #else 501 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) \ 502 ((void)ipsec4_common_input_cb(m, sav, skip, protoff)) 503 #endif 504 505 /* 506 * ESP input callback from the crypto driver. 507 */ 508 static void 509 esp_input_cb(struct cryptop *crp) 510 { 511 char buf[IPSEC_ADDRSTRLEN]; 512 uint8_t lastthree[3], aalg[AH_ALEN_MAX]; 513 int hlen, skip, protoff; 514 struct mbuf *m; 515 const struct auth_hash *esph; 516 struct tdb_crypto *tc; 517 struct secasvar *sav; 518 struct secasindex *saidx; 519 void *ptr; 520 IPSEC_DECLARE_LOCK_VARIABLE; 521 522 KASSERT(crp->crp_desc != NULL); 523 KASSERT(crp->crp_opaque != NULL); 524 525 tc = crp->crp_opaque; 526 skip = tc->tc_skip; 527 protoff = tc->tc_protoff; 528 m = crp->crp_buf; 529 530 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 531 532 sav = tc->tc_sav; 533 saidx = &sav->sah->saidx; 534 KASSERTMSG(saidx->dst.sa.sa_family == AF_INET || 535 saidx->dst.sa.sa_family == AF_INET6, 536 "unexpected protocol family %u", saidx->dst.sa.sa_family); 537 538 esph = sav->tdb_authalgxform; 539 540 /* Check for crypto errors */ 541 if (crp->crp_etype) { 542 /* Reset the session ID */ 543 if (sav->tdb_cryptoid != 0) 544 sav->tdb_cryptoid = crp->crp_sid; 545 546 ESP_STATINC(ESP_STAT_NOXFORM); 547 DPRINTF("crypto error %d\n", crp->crp_etype); 548 goto bad; 549 } 550 551 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 552 553 /* If authentication was performed, check now. */ 554 if (esph != NULL) { 555 /* 556 * If we have a tag, it means an IPsec-aware NIC did 557 * the verification for us. Otherwise we need to 558 * check the authentication calculation. 559 */ 560 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 561 /* Copy the authenticator from the packet */ 562 m_copydata(m, m->m_pkthdr.len - esph->authsize, 563 esph->authsize, aalg); 564 565 ptr = (tc + 1); 566 567 /* Verify authenticator */ 568 if (!consttime_memequal(ptr, aalg, esph->authsize)) { 569 DPRINTF("authentication hash mismatch " 570 "for packet in SA %s/%08lx\n", 571 ipsec_address(&saidx->dst, buf, 572 sizeof(buf)), (u_long) ntohl(sav->spi)); 573 ESP_STATINC(ESP_STAT_BADAUTH); 574 goto bad; 575 } 576 577 /* Remove trailing authenticator */ 578 m_adj(m, -(esph->authsize)); 579 } 580 581 /* Release the crypto descriptors */ 582 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 583 tc = NULL; 584 crypto_freereq(crp); 585 crp = NULL; 586 587 /* 588 * Packet is now decrypted. 589 */ 590 m->m_flags |= M_DECRYPTED; 591 592 /* 593 * Update replay sequence number, if appropriate. 594 */ 595 if (sav->replay) { 596 uint32_t seq; 597 598 m_copydata(m, skip + offsetof(struct newesp, esp_seq), 599 sizeof(seq), &seq); 600 if (ipsec_updatereplay(ntohl(seq), sav)) { 601 char logbuf[IPSEC_LOGSASTRLEN]; 602 DPRINTF("packet replay check for %s\n", 603 ipsec_logsastr(sav, logbuf, sizeof(logbuf))); 604 ESP_STATINC(ESP_STAT_REPLAY); 605 goto bad; 606 } 607 } 608 609 /* Determine the ESP header length */ 610 if (sav->flags & SADB_X_EXT_OLD) 611 hlen = sizeof(struct esp) + sav->ivlen; 612 else 613 hlen = sizeof(struct newesp) + sav->ivlen; 614 615 /* Remove the ESP header and IV from the mbuf. */ 616 if (m_striphdr(m, skip, hlen) != 0) { 617 ESP_STATINC(ESP_STAT_HDROPS); 618 DPRINTF("bad mbuf chain, SA %s/%08lx\n", 619 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 620 (u_long) ntohl(sav->spi)); 621 goto bad; 622 } 623 624 /* Save the last three bytes of decrypted data */ 625 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree); 626 627 /* Verify pad length */ 628 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) { 629 ESP_STATINC(ESP_STAT_BADILEN); 630 DPRINTF("invalid padding length %d " 631 "for %u byte packet in SA %s/%08lx\n", 632 lastthree[1], m->m_pkthdr.len - skip, 633 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 634 (u_long) ntohl(sav->spi)); 635 goto bad; 636 } 637 638 /* Verify correct decryption by checking the last padding bytes */ 639 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) { 640 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) { 641 ESP_STATINC(ESP_STAT_BADENC); 642 DPRINTF("decryption failed for packet in SA " 643 "%s/%08lx\n", 644 ipsec_address(&sav->sah->saidx.dst, buf, 645 sizeof(buf)), (u_long) ntohl(sav->spi)); 646 DPRINTF("%x %x\n", lastthree[0], 647 lastthree[1]); 648 goto bad; 649 } 650 } 651 652 /* Trim the mbuf chain to remove trailing authenticator and padding */ 653 m_adj(m, -(lastthree[1] + 2)); 654 655 /* Restore the Next Protocol field */ 656 m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2); 657 658 IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff); 659 660 KEY_SA_UNREF(&sav); 661 IPSEC_RELEASE_GLOBAL_LOCKS(); 662 return; 663 bad: 664 if (sav) 665 KEY_SA_UNREF(&sav); 666 IPSEC_RELEASE_GLOBAL_LOCKS(); 667 m_freem(m); 668 if (tc != NULL) 669 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 670 if (crp != NULL) 671 crypto_freereq(crp); 672 } 673 674 /* 675 * ESP output routine, called by ipsec[46]_process_packet(). 676 */ 677 static int 678 esp_output(struct mbuf *m, const struct ipsecrequest *isr, struct secasvar *sav, 679 int skip, int protoff, int flags) 680 { 681 char buf[IPSEC_ADDRSTRLEN]; 682 const struct enc_xform *espx; 683 const struct auth_hash *esph; 684 int hlen, rlen, tlen, padlen, blks, alen, i, roff; 685 struct mbuf *mo = NULL; 686 struct tdb_crypto *tc; 687 struct secasindex *saidx; 688 unsigned char *tail; 689 uint8_t prot; 690 int error, maxpacketsize; 691 struct esptail *esptail; 692 struct cryptodesc *crde, *crda; 693 struct cryptop *crp; 694 695 esph = sav->tdb_authalgxform; 696 espx = sav->tdb_encalgxform; 697 KASSERT(espx != NULL); 698 699 /* Determine the ESP header length */ 700 if (sav->flags & SADB_X_EXT_OLD) 701 hlen = sizeof(struct esp) + sav->ivlen; 702 else 703 hlen = sizeof(struct newesp) + sav->ivlen; 704 /* Authenticator hash size */ 705 alen = esph ? esph->authsize : 0; 706 707 /* 708 * NB: The null encoding transform has a blocksize of 4 709 * so that headers are properly aligned. 710 */ 711 blks = espx->blocksize; /* IV blocksize */ 712 713 /* Raw payload length. */ 714 rlen = m->m_pkthdr.len - skip; 715 716 /* Encryption padding. */ 717 padlen = ((blks - ((rlen + sizeof(struct esptail)) % blks)) % blks); 718 719 /* Length of what we append (tail). */ 720 tlen = padlen + sizeof(struct esptail) + alen; 721 722 ESP_STATINC(ESP_STAT_OUTPUT); 723 724 saidx = &sav->sah->saidx; 725 /* Check for maximum packet size violations. */ 726 switch (saidx->dst.sa.sa_family) { 727 #ifdef INET 728 case AF_INET: 729 maxpacketsize = IP_MAXPACKET; 730 break; 731 #endif 732 #ifdef INET6 733 case AF_INET6: 734 maxpacketsize = IPV6_MAXPACKET; 735 break; 736 #endif 737 default: 738 DPRINTF("unknown/unsupported protocol family %d, " 739 "SA %s/%08lx\n", saidx->dst.sa.sa_family, 740 ipsec_address(&saidx->dst, buf, sizeof(buf)), 741 (u_long)ntohl(sav->spi)); 742 ESP_STATINC(ESP_STAT_NOPF); 743 error = EPFNOSUPPORT; 744 goto bad; 745 } 746 if (skip + hlen + rlen + tlen > maxpacketsize) { 747 DPRINTF("packet in SA %s/%08lx got too big (len %u, " 748 "max len %u)\n", 749 ipsec_address(&saidx->dst, buf, sizeof(buf)), 750 (u_long) ntohl(sav->spi), 751 skip + hlen + rlen + tlen, maxpacketsize); 752 ESP_STATINC(ESP_STAT_TOOBIG); 753 error = EMSGSIZE; 754 goto bad; 755 } 756 757 /* Update the counters. */ 758 ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip); 759 760 m = m_clone(m); 761 if (m == NULL) { 762 DPRINTF("cannot clone mbuf chain, SA %s/%08lx\n", 763 ipsec_address(&saidx->dst, buf, sizeof(buf)), 764 (u_long) ntohl(sav->spi)); 765 ESP_STATINC(ESP_STAT_HDROPS); 766 error = ENOBUFS; 767 goto bad; 768 } 769 770 /* Inject ESP header. */ 771 mo = m_makespace(m, skip, hlen, &roff); 772 if (mo == NULL) { 773 DPRINTF("failed to inject %u byte ESP hdr for SA " 774 "%s/%08lx\n", hlen, 775 ipsec_address(&saidx->dst, buf, sizeof(buf)), 776 (u_long) ntohl(sav->spi)); 777 ESP_STATINC(ESP_STAT_HDROPS); 778 error = ENOBUFS; 779 goto bad; 780 } 781 782 /* Initialize ESP header. */ 783 memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t)); 784 if (sav->replay) { 785 uint32_t replay; 786 787 #ifdef IPSEC_DEBUG 788 /* Emulate replay attack when ipsec_replay is TRUE. */ 789 if (ipsec_replay) 790 replay = htonl(sav->replay->count); 791 else 792 #endif 793 replay = htonl(atomic_inc_32_nv(&sav->replay->count)); 794 795 memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay, 796 sizeof(uint32_t)); 797 } 798 799 /* 800 * Grow the mbuf, we will append data at the tail. 801 */ 802 tail = m_pad(m, tlen); 803 if (tail == NULL) { 804 DPRINTF("m_pad failed for SA %s/%08lx\n", 805 ipsec_address(&saidx->dst, buf, sizeof(buf)), 806 (u_long) ntohl(sav->spi)); 807 m = NULL; 808 error = ENOBUFS; 809 goto bad; 810 } 811 812 /* 813 * Add padding: random, zero, or self-describing. 814 */ 815 switch (sav->flags & SADB_X_EXT_PMASK) { 816 case SADB_X_EXT_PSEQ: 817 for (i = 0; i < padlen; i++) 818 tail[i] = i + 1; 819 break; 820 case SADB_X_EXT_PRAND: 821 (void)cprng_fast(tail, padlen); 822 break; 823 case SADB_X_EXT_PZERO: 824 default: 825 memset(tail, 0, padlen); 826 break; 827 } 828 829 /* Build the ESP Trailer. */ 830 esptail = (struct esptail *)&tail[padlen]; 831 esptail->esp_padlen = padlen; 832 m_copydata(m, protoff, sizeof(uint8_t), &esptail->esp_nxt); 833 834 /* Fix Next Protocol in IPv4/IPv6 header. */ 835 prot = IPPROTO_ESP; 836 m_copyback(m, protoff, sizeof(uint8_t), &prot); 837 838 /* Get crypto descriptors. */ 839 crp = crypto_getreq(esph ? 2 : 1); 840 if (crp == NULL) { 841 DPRINTF("failed to acquire crypto descriptors\n"); 842 ESP_STATINC(ESP_STAT_CRYPTO); 843 error = ENOBUFS; 844 goto bad; 845 } 846 847 /* Get the descriptors. */ 848 crde = crp->crp_desc; 849 crda = crde->crd_next; 850 851 /* Encryption descriptor. */ 852 crde->crd_skip = skip + hlen; 853 if (espx->type == CRYPTO_AES_GMAC) 854 crde->crd_len = 0; 855 else 856 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 857 crde->crd_flags = CRD_F_ENCRYPT; 858 crde->crd_inject = skip + hlen - sav->ivlen; 859 crde->crd_alg = espx->type; 860 crde->crd_key = _KEYBUF(sav->key_enc); 861 crde->crd_klen = _KEYBITS(sav->key_enc); 862 /* XXX Rounds ? */ 863 864 /* IPsec-specific opaque crypto info. */ 865 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 866 if (tc == NULL) { 867 crypto_freereq(crp); 868 DPRINTF("failed to allocate tdb_crypto\n"); 869 ESP_STATINC(ESP_STAT_CRYPTO); 870 error = ENOBUFS; 871 goto bad; 872 } 873 874 { 875 int s = pserialize_read_enter(); 876 877 /* 878 * Take another reference to the SP and the SA for opencrypto callback. 879 */ 880 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD || 881 sav->state == SADB_SASTATE_DEAD)) { 882 pserialize_read_exit(s); 883 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 884 crypto_freereq(crp); 885 ESP_STATINC(ESP_STAT_NOTDB); 886 error = ENOENT; 887 goto bad; 888 } 889 KEY_SP_REF(isr->sp); 890 KEY_SA_REF(sav); 891 pserialize_read_exit(s); 892 } 893 894 /* Callback parameters */ 895 tc->tc_isr = isr; 896 tc->tc_spi = sav->spi; 897 tc->tc_dst = saidx->dst; 898 tc->tc_proto = saidx->proto; 899 tc->tc_flags = flags; 900 tc->tc_sav = sav; 901 902 /* Crypto operation descriptor. */ 903 crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ 904 crp->crp_flags = CRYPTO_F_IMBUF; 905 crp->crp_buf = m; 906 crp->crp_callback = esp_output_cb; 907 crp->crp_opaque = tc; 908 crp->crp_sid = sav->tdb_cryptoid; 909 910 if (esph) { 911 /* Authentication descriptor. */ 912 crda->crd_skip = skip; 913 if (espx->type == CRYPTO_AES_GCM_16) 914 crda->crd_len = hlen - sav->ivlen; 915 else 916 crda->crd_len = m->m_pkthdr.len - (skip + alen); 917 crda->crd_inject = m->m_pkthdr.len - alen; 918 919 /* Authentication operation. */ 920 crda->crd_alg = esph->type; 921 if (espx->type == CRYPTO_AES_GCM_16 || 922 espx->type == CRYPTO_AES_GMAC) { 923 crda->crd_key = _KEYBUF(sav->key_enc); 924 crda->crd_klen = _KEYBITS(sav->key_enc); 925 } else { 926 crda->crd_key = _KEYBUF(sav->key_auth); 927 crda->crd_klen = _KEYBITS(sav->key_auth); 928 } 929 } 930 931 crypto_dispatch(crp); 932 return 0; 933 934 bad: 935 m_freem(m); 936 return error; 937 } 938 939 /* 940 * ESP output callback from the crypto driver. 941 */ 942 static void 943 esp_output_cb(struct cryptop *crp) 944 { 945 struct tdb_crypto *tc; 946 const struct ipsecrequest *isr; 947 struct secasvar *sav; 948 struct mbuf *m; 949 int flags; 950 IPSEC_DECLARE_LOCK_VARIABLE; 951 952 KASSERT(crp->crp_opaque != NULL); 953 tc = crp->crp_opaque; 954 m = crp->crp_buf; 955 956 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 957 958 isr = tc->tc_isr; 959 sav = tc->tc_sav; 960 961 /* Check for crypto errors. */ 962 if (crp->crp_etype) { 963 /* Reset session ID. */ 964 if (sav->tdb_cryptoid != 0) 965 sav->tdb_cryptoid = crp->crp_sid; 966 967 ESP_STATINC(ESP_STAT_NOXFORM); 968 DPRINTF("crypto error %d\n", crp->crp_etype); 969 goto bad; 970 } 971 972 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 973 if (sav->tdb_authalgxform != NULL) 974 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 975 976 flags = tc->tc_flags; 977 /* Release crypto descriptors. */ 978 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 979 crypto_freereq(crp); 980 981 #ifdef IPSEC_DEBUG 982 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */ 983 if (ipsec_integrity) { 984 static unsigned char ipseczeroes[AH_ALEN_MAX]; 985 const struct auth_hash *esph; 986 987 /* 988 * Corrupt HMAC if we want to test integrity verification of 989 * the other side. 990 */ 991 esph = sav->tdb_authalgxform; 992 if (esph != NULL) { 993 m_copyback(m, m->m_pkthdr.len - esph->authsize, 994 esph->authsize, ipseczeroes); 995 } 996 } 997 #endif 998 999 /* NB: m is reclaimed by ipsec_process_done. */ 1000 (void)ipsec_process_done(m, isr, sav, flags); 1001 KEY_SA_UNREF(&sav); 1002 KEY_SP_UNREF(&isr->sp); 1003 IPSEC_RELEASE_GLOBAL_LOCKS(); 1004 return; 1005 1006 bad: 1007 if (sav) 1008 KEY_SA_UNREF(&sav); 1009 KEY_SP_UNREF(&isr->sp); 1010 IPSEC_RELEASE_GLOBAL_LOCKS(); 1011 m_freem(m); 1012 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 1013 crypto_freereq(crp); 1014 } 1015 1016 static struct xformsw esp_xformsw = { 1017 .xf_type = XF_ESP, 1018 .xf_flags = XFT_CONF|XFT_AUTH, 1019 .xf_name = "IPsec ESP", 1020 .xf_init = esp_init, 1021 .xf_zeroize = esp_zeroize, 1022 .xf_input = esp_input, 1023 .xf_output = esp_output, 1024 .xf_next = NULL, 1025 }; 1026 1027 void 1028 esp_attach(void) 1029 { 1030 1031 espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS); 1032 1033 extern int ah_max_authsize; 1034 KASSERT(ah_max_authsize != 0); 1035 esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize; 1036 esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size, 1037 coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET, 1038 NULL, NULL, NULL); 1039 1040 #define MAXIV(xform) \ 1041 if (xform.ivsize > esp_max_ivlen) \ 1042 esp_max_ivlen = xform.ivsize \ 1043 1044 esp_max_ivlen = 0; 1045 MAXIV(enc_xform_des); /* SADB_EALG_DESCBC */ 1046 MAXIV(enc_xform_3des); /* SADB_EALG_3DESCBC */ 1047 MAXIV(enc_xform_aes); /* SADB_X_EALG_AES */ 1048 MAXIV(enc_xform_blf); /* SADB_X_EALG_BLOWFISHCBC */ 1049 MAXIV(enc_xform_cast5); /* SADB_X_EALG_CAST128CBC */ 1050 MAXIV(enc_xform_skipjack); /* SADB_X_EALG_SKIPJACK */ 1051 MAXIV(enc_xform_camellia); /* SADB_X_EALG_CAMELLIACBC */ 1052 MAXIV(enc_xform_aes_ctr); /* SADB_X_EALG_AESCTR */ 1053 MAXIV(enc_xform_null); /* SADB_EALG_NULL */ 1054 1055 xform_register(&esp_xformsw); 1056 #undef MAXIV 1057 } 1058
Indexes created Sat Sep 20 22:09:52 GMT 2025