if_wg.c revision 1.76
1 /* $NetBSD: if_wg.c,v 1.76 2023/04/11 14:03:46 jakllsch Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.76 2023/04/11 14:03:46 jakllsch Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <net/bpf.h> 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/if_wg.h> 90 #include <net/pktqueue.h> 91 #include <net/route.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip.h> 97 #include <netinet/ip_var.h> 98 #include <netinet/udp.h> 99 #include <netinet/udp_var.h> 100 101 #ifdef INET6 102 #include <netinet/ip6.h> 103 #include <netinet6/in6_pcb.h> 104 #include <netinet6/in6_var.h> 105 #include <netinet6/ip6_var.h> 106 #include <netinet6/udp6_var.h> 107 #endif /* INET6 */ 108 109 #include <prop/proplib.h> 110 111 #include <crypto/blake2/blake2s.h> 112 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 113 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 114 #include <crypto/sodium/crypto_scalarmult.h> 115 116 #include "ioconf.h" 117 118 #ifdef WG_RUMPKERNEL 119 #include "wg_user.h" 120 #endif 121 122 /* 123 * Data structures 124 * - struct wg_softc is an instance of wg interfaces 125 * - It has a list of peers (struct wg_peer) 126 * - It has a threadpool job that sends/receives handshake messages and 127 * runs event handlers 128 * - It has its own two routing tables: one is for IPv4 and the other IPv6 129 * - struct wg_peer is a representative of a peer 130 * - It has a struct work to handle handshakes and timer tasks 131 * - It has a pair of session instances (struct wg_session) 132 * - It has a pair of endpoint instances (struct wg_sockaddr) 133 * - Normally one endpoint is used and the second one is used only on 134 * a peer migration (a change of peer's IP address) 135 * - It has a list of IP addresses and sub networks called allowedips 136 * (struct wg_allowedip) 137 * - A packets sent over a session is allowed if its destination matches 138 * any IP addresses or sub networks of the list 139 * - struct wg_session represents a session of a secure tunnel with a peer 140 * - Two instances of sessions belong to a peer; a stable session and a 141 * unstable session 142 * - A handshake process of a session always starts with a unstable instance 143 * - Once a session is established, its instance becomes stable and the 144 * other becomes unstable instead 145 * - Data messages are always sent via a stable session 146 * 147 * Locking notes: 148 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 149 * - Changes to the peer list are serialized by wg_lock 150 * - The peer list may be read with pserialize(9) and psref(9) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 152 * => XXX replace by pserialize when routing table is psz-safe 153 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 154 * only in thread context and serializes: 155 * - the stable and unstable session pointers 156 * - all unstable session state 157 * - Packet processing may be done in softint context: 158 * - The stable session can be read under pserialize(9) or psref(9) 159 * - The stable session is always ESTABLISHED 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 * - Lock order: wg_lock -> wgp_lock 164 */ 165 166 167 #define WGLOG(level, fmt, args...) \ 168 log(level, "%s: " fmt, __func__, ##args) 169 170 /* Debug options */ 171 #ifdef WG_DEBUG 172 /* Output debug logs */ 173 #ifndef WG_DEBUG_LOG 174 #define WG_DEBUG_LOG 175 #endif 176 /* Output trace logs */ 177 #ifndef WG_DEBUG_TRACE 178 #define WG_DEBUG_TRACE 179 #endif 180 /* Output hash values, etc. */ 181 #ifndef WG_DEBUG_DUMP 182 #define WG_DEBUG_DUMP 183 #endif 184 /* Make some internal parameters configurable for testing and debugging */ 185 #ifndef WG_DEBUG_PARAMS 186 #define WG_DEBUG_PARAMS 187 #endif 188 #endif 189 190 #ifdef WG_DEBUG_TRACE 191 #define WG_TRACE(msg) \ 192 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)) 193 #else 194 #define WG_TRACE(msg) __nothing 195 #endif 196 197 #ifdef WG_DEBUG_LOG 198 #define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args) 199 #else 200 #define WG_DLOG(fmt, args...) __nothing 201 #endif 202 203 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 204 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 205 &(wgprc)->wgprc_curpps, 1)) { \ 206 log(level, fmt, ##args); \ 207 } \ 208 } while (0) 209 210 #ifdef WG_DEBUG_PARAMS 211 static bool wg_force_underload = false; 212 #endif 213 214 #ifdef WG_DEBUG_DUMP 215 216 static char * 217 gethexdump(const char *p, size_t n) 218 { 219 char *buf; 220 size_t i; 221 222 if (n > SIZE_MAX/3 - 1) 223 return NULL; 224 buf = kmem_alloc(3*n + 1, KM_NOSLEEP); 225 if (buf == NULL) 226 return NULL; 227 for (i = 0; i < n; i++) 228 snprintf(buf + 3*i, 3 + 1, " %02hhx", p[i]); 229 return buf; 230 } 231 232 static void 233 puthexdump(char *buf, const void *p, size_t n) 234 { 235 236 if (buf == NULL) 237 return; 238 kmem_free(buf, 3*n + 1); 239 } 240 241 #ifdef WG_RUMPKERNEL 242 static void 243 wg_dump_buf(const char *func, const char *buf, const size_t size) 244 { 245 char *hex = gethexdump(buf, size); 246 247 log(LOG_DEBUG, "%s: %s\n", func, hex ? hex : "(enomem)"); 248 puthexdump(hex, buf, size); 249 } 250 #endif 251 252 static void 253 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 254 const size_t size) 255 { 256 char *hex = gethexdump(hash, size); 257 258 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex ? hex : "(enomem)"); 259 puthexdump(hex, hash, size); 260 } 261 262 #define WG_DUMP_HASH(name, hash) \ 263 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 264 #define WG_DUMP_HASH48(name, hash) \ 265 wg_dump_hash(__func__, name, hash, 48) 266 #define WG_DUMP_BUF(buf, size) \ 267 wg_dump_buf(__func__, buf, size) 268 #else 269 #define WG_DUMP_HASH(name, hash) __nothing 270 #define WG_DUMP_HASH48(name, hash) __nothing 271 #define WG_DUMP_BUF(buf, size) __nothing 272 #endif /* WG_DEBUG_DUMP */ 273 274 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 275 #define WG_MAX_PROPLEN 32766 276 277 #define WG_MTU 1420 278 #define WG_ALLOWEDIPS 16 279 280 #define CURVE25519_KEY_LEN 32 281 #define TAI64N_LEN sizeof(uint32_t) * 3 282 #define POLY1305_AUTHTAG_LEN 16 283 #define HMAC_BLOCK_LEN 64 284 285 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 286 /* [N] 4.3: Hash functions */ 287 #define NOISE_DHLEN 32 288 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 289 #define NOISE_HASHLEN 32 290 #define NOISE_BLOCKLEN 64 291 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 292 /* [N] 5.1: "k" */ 293 #define NOISE_CIPHER_KEY_LEN 32 294 /* 295 * [N] 9.2: "psk" 296 * "... psk is a 32-byte secret value provided by the application." 297 */ 298 #define NOISE_PRESHARED_KEY_LEN 32 299 300 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 301 #define WG_TIMESTAMP_LEN TAI64N_LEN 302 303 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 304 305 #define WG_COOKIE_LEN 16 306 #define WG_MAC_LEN 16 307 #define WG_RANDVAL_LEN 24 308 309 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 310 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 311 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 312 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 313 #define WG_HASH_LEN NOISE_HASHLEN 314 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 315 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 316 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 317 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 318 #define WG_DATA_KEY_LEN 32 319 #define WG_SALT_LEN 24 320 321 /* 322 * The protocol messages 323 */ 324 struct wg_msg { 325 uint32_t wgm_type; 326 } __packed; 327 328 /* [W] 5.4.2 First Message: Initiator to Responder */ 329 struct wg_msg_init { 330 uint32_t wgmi_type; 331 uint32_t wgmi_sender; 332 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 333 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 334 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 335 uint8_t wgmi_mac1[WG_MAC_LEN]; 336 uint8_t wgmi_mac2[WG_MAC_LEN]; 337 } __packed; 338 339 /* [W] 5.4.3 Second Message: Responder to Initiator */ 340 struct wg_msg_resp { 341 uint32_t wgmr_type; 342 uint32_t wgmr_sender; 343 uint32_t wgmr_receiver; 344 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 345 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 346 uint8_t wgmr_mac1[WG_MAC_LEN]; 347 uint8_t wgmr_mac2[WG_MAC_LEN]; 348 } __packed; 349 350 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 351 struct wg_msg_data { 352 uint32_t wgmd_type; 353 uint32_t wgmd_receiver; 354 uint64_t wgmd_counter; 355 uint32_t wgmd_packet[0]; 356 } __packed; 357 358 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 359 struct wg_msg_cookie { 360 uint32_t wgmc_type; 361 uint32_t wgmc_receiver; 362 uint8_t wgmc_salt[WG_SALT_LEN]; 363 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 364 } __packed; 365 366 #define WG_MSG_TYPE_INIT 1 367 #define WG_MSG_TYPE_RESP 2 368 #define WG_MSG_TYPE_COOKIE 3 369 #define WG_MSG_TYPE_DATA 4 370 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 371 372 /* Sliding windows */ 373 374 #define SLIWIN_BITS 2048u 375 #define SLIWIN_TYPE uint32_t 376 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 377 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 378 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 379 380 struct sliwin { 381 SLIWIN_TYPE B[SLIWIN_WORDS]; 382 uint64_t T; 383 }; 384 385 static void 386 sliwin_reset(struct sliwin *W) 387 { 388 389 memset(W, 0, sizeof(*W)); 390 } 391 392 static int 393 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 394 { 395 396 /* 397 * If it's more than one window older than the highest sequence 398 * number we've seen, reject. 399 */ 400 #ifdef __HAVE_ATOMIC64_LOADSTORE 401 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 402 return EAUTH; 403 #endif 404 405 /* 406 * Otherwise, we need to take the lock to decide, so don't 407 * reject just yet. Caller must serialize a call to 408 * sliwin_update in this case. 409 */ 410 return 0; 411 } 412 413 static int 414 sliwin_update(struct sliwin *W, uint64_t S) 415 { 416 unsigned word, bit; 417 418 /* 419 * If it's more than one window older than the highest sequence 420 * number we've seen, reject. 421 */ 422 if (S + SLIWIN_NPKT < W->T) 423 return EAUTH; 424 425 /* 426 * If it's higher than the highest sequence number we've seen, 427 * advance the window. 428 */ 429 if (S > W->T) { 430 uint64_t i = W->T / SLIWIN_BPW; 431 uint64_t j = S / SLIWIN_BPW; 432 unsigned k; 433 434 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 435 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 436 #ifdef __HAVE_ATOMIC64_LOADSTORE 437 atomic_store_relaxed(&W->T, S); 438 #else 439 W->T = S; 440 #endif 441 } 442 443 /* Test and set the bit -- if already set, reject. */ 444 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 445 bit = S % SLIWIN_BPW; 446 if (W->B[word] & (1UL << bit)) 447 return EAUTH; 448 W->B[word] |= 1U << bit; 449 450 /* Accept! */ 451 return 0; 452 } 453 454 struct wg_session { 455 struct wg_peer *wgs_peer; 456 struct psref_target 457 wgs_psref; 458 459 int wgs_state; 460 #define WGS_STATE_UNKNOWN 0 461 #define WGS_STATE_INIT_ACTIVE 1 462 #define WGS_STATE_INIT_PASSIVE 2 463 #define WGS_STATE_ESTABLISHED 3 464 #define WGS_STATE_DESTROYING 4 465 466 time_t wgs_time_established; 467 time_t wgs_time_last_data_sent; 468 bool wgs_is_initiator; 469 470 uint32_t wgs_local_index; 471 uint32_t wgs_remote_index; 472 #ifdef __HAVE_ATOMIC64_LOADSTORE 473 volatile uint64_t 474 wgs_send_counter; 475 #else 476 kmutex_t wgs_send_counter_lock; 477 uint64_t wgs_send_counter; 478 #endif 479 480 struct { 481 kmutex_t lock; 482 struct sliwin window; 483 } *wgs_recvwin; 484 485 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 486 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 487 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 488 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 489 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 490 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 491 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 492 }; 493 494 struct wg_sockaddr { 495 union { 496 struct sockaddr_storage _ss; 497 struct sockaddr _sa; 498 struct sockaddr_in _sin; 499 struct sockaddr_in6 _sin6; 500 }; 501 struct psref_target wgsa_psref; 502 }; 503 504 #define wgsatoss(wgsa) (&(wgsa)->_ss) 505 #define wgsatosa(wgsa) (&(wgsa)->_sa) 506 #define wgsatosin(wgsa) (&(wgsa)->_sin) 507 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 508 509 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 510 511 struct wg_peer; 512 struct wg_allowedip { 513 struct radix_node wga_nodes[2]; 514 struct wg_sockaddr _wga_sa_addr; 515 struct wg_sockaddr _wga_sa_mask; 516 #define wga_sa_addr _wga_sa_addr._sa 517 #define wga_sa_mask _wga_sa_mask._sa 518 519 int wga_family; 520 uint8_t wga_cidr; 521 union { 522 struct in_addr _ip4; 523 struct in6_addr _ip6; 524 } wga_addr; 525 #define wga_addr4 wga_addr._ip4 526 #define wga_addr6 wga_addr._ip6 527 528 struct wg_peer *wga_peer; 529 }; 530 531 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 532 533 struct wg_ppsratecheck { 534 struct timeval wgprc_lasttime; 535 int wgprc_curpps; 536 }; 537 538 struct wg_softc; 539 struct wg_peer { 540 struct wg_softc *wgp_sc; 541 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 542 struct pslist_entry wgp_peerlist_entry; 543 pserialize_t wgp_psz; 544 struct psref_target wgp_psref; 545 kmutex_t *wgp_lock; 546 kmutex_t *wgp_intr_lock; 547 548 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 549 struct wg_sockaddr *wgp_endpoint; 550 struct wg_sockaddr *wgp_endpoint0; 551 volatile unsigned wgp_endpoint_changing; 552 bool wgp_endpoint_available; 553 554 /* The preshared key (optional) */ 555 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 556 557 struct wg_session *wgp_session_stable; 558 struct wg_session *wgp_session_unstable; 559 560 /* first outgoing packet awaiting session initiation */ 561 struct mbuf *wgp_pending; 562 563 /* timestamp in big-endian */ 564 wg_timestamp_t wgp_timestamp_latest_init; 565 566 struct timespec wgp_last_handshake_time; 567 568 callout_t wgp_rekey_timer; 569 callout_t wgp_handshake_timeout_timer; 570 callout_t wgp_session_dtor_timer; 571 572 time_t wgp_handshake_start_time; 573 574 int wgp_n_allowedips; 575 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 576 577 time_t wgp_latest_cookie_time; 578 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 579 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 580 bool wgp_last_sent_mac1_valid; 581 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 582 bool wgp_last_sent_cookie_valid; 583 584 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 585 586 time_t wgp_last_genrandval_time; 587 uint32_t wgp_randval; 588 589 struct wg_ppsratecheck wgp_ppsratecheck; 590 591 struct work wgp_work; 592 unsigned int wgp_tasks; 593 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 594 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 595 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 596 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 597 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 598 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 599 }; 600 601 struct wg_ops; 602 603 struct wg_softc { 604 struct ifnet wg_if; 605 LIST_ENTRY(wg_softc) wg_list; 606 kmutex_t *wg_lock; 607 kmutex_t *wg_intr_lock; 608 krwlock_t *wg_rwlock; 609 610 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 611 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 612 613 int wg_npeers; 614 struct pslist_head wg_peers; 615 struct thmap *wg_peers_bypubkey; 616 struct thmap *wg_peers_byname; 617 struct thmap *wg_sessions_byindex; 618 uint16_t wg_listen_port; 619 620 struct threadpool *wg_threadpool; 621 622 struct threadpool_job wg_job; 623 int wg_upcalls; 624 #define WG_UPCALL_INET __BIT(0) 625 #define WG_UPCALL_INET6 __BIT(1) 626 627 #ifdef INET 628 struct socket *wg_so4; 629 struct radix_node_head *wg_rtable_ipv4; 630 #endif 631 #ifdef INET6 632 struct socket *wg_so6; 633 struct radix_node_head *wg_rtable_ipv6; 634 #endif 635 636 struct wg_ppsratecheck wg_ppsratecheck; 637 638 struct wg_ops *wg_ops; 639 640 #ifdef WG_RUMPKERNEL 641 struct wg_user *wg_user; 642 #endif 643 }; 644 645 /* [W] 6.1 Preliminaries */ 646 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 647 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 648 #define WG_REKEY_AFTER_TIME 120 649 #define WG_REJECT_AFTER_TIME 180 650 #define WG_REKEY_ATTEMPT_TIME 90 651 #define WG_REKEY_TIMEOUT 5 652 #define WG_KEEPALIVE_TIMEOUT 10 653 654 #define WG_COOKIE_TIME 120 655 #define WG_RANDVAL_TIME (2 * 60) 656 657 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 658 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 659 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 660 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 661 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 662 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 663 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 664 665 static struct mbuf * 666 wg_get_mbuf(size_t, size_t); 667 668 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 669 struct mbuf *); 670 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 671 const uint32_t, const uint8_t [], const struct sockaddr *); 672 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 673 struct wg_session *, const struct wg_msg_init *); 674 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 675 676 static struct wg_peer * 677 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 678 struct psref *); 679 static struct wg_peer * 680 wg_lookup_peer_by_pubkey(struct wg_softc *, 681 const uint8_t [], struct psref *); 682 683 static struct wg_session * 684 wg_lookup_session_by_index(struct wg_softc *, 685 const uint32_t, struct psref *); 686 687 static void wg_update_endpoint_if_necessary(struct wg_peer *, 688 const struct sockaddr *); 689 690 static void wg_schedule_rekey_timer(struct wg_peer *); 691 static void wg_schedule_session_dtor_timer(struct wg_peer *); 692 693 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 694 static void wg_calculate_keys(struct wg_session *, const bool); 695 696 static void wg_clear_states(struct wg_session *); 697 698 static void wg_get_peer(struct wg_peer *, struct psref *); 699 static void wg_put_peer(struct wg_peer *, struct psref *); 700 701 static int wg_send_so(struct wg_peer *, struct mbuf *); 702 static int wg_send_udp(struct wg_peer *, struct mbuf *); 703 static int wg_output(struct ifnet *, struct mbuf *, 704 const struct sockaddr *, const struct rtentry *); 705 static void wg_input(struct ifnet *, struct mbuf *, const int); 706 static int wg_ioctl(struct ifnet *, u_long, void *); 707 static int wg_bind_port(struct wg_softc *, const uint16_t); 708 static int wg_init(struct ifnet *); 709 #ifdef ALTQ 710 static void wg_start(struct ifnet *); 711 #endif 712 static void wg_stop(struct ifnet *, int); 713 714 static void wg_peer_work(struct work *, void *); 715 static void wg_job(struct threadpool_job *); 716 static void wgintr(void *); 717 static void wg_purge_pending_packets(struct wg_peer *); 718 719 static int wg_clone_create(struct if_clone *, int); 720 static int wg_clone_destroy(struct ifnet *); 721 722 struct wg_ops { 723 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 724 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 725 void (*input)(struct ifnet *, struct mbuf *, const int); 726 int (*bind_port)(struct wg_softc *, const uint16_t); 727 }; 728 729 struct wg_ops wg_ops_rumpkernel = { 730 .send_hs_msg = wg_send_so, 731 .send_data_msg = wg_send_udp, 732 .input = wg_input, 733 .bind_port = wg_bind_port, 734 }; 735 736 #ifdef WG_RUMPKERNEL 737 static bool wg_user_mode(struct wg_softc *); 738 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 739 740 static int wg_send_user(struct wg_peer *, struct mbuf *); 741 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 742 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 743 744 struct wg_ops wg_ops_rumpuser = { 745 .send_hs_msg = wg_send_user, 746 .send_data_msg = wg_send_user, 747 .input = wg_input_user, 748 .bind_port = wg_bind_port_user, 749 }; 750 #endif 751 752 #define WG_PEER_READER_FOREACH(wgp, wg) \ 753 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 754 wgp_peerlist_entry) 755 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 756 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 757 wgp_peerlist_entry) 758 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 759 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 760 #define WG_PEER_WRITER_REMOVE(wgp) \ 761 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 762 763 struct wg_route { 764 struct radix_node wgr_nodes[2]; 765 struct wg_peer *wgr_peer; 766 }; 767 768 static struct radix_node_head * 769 wg_rnh(struct wg_softc *wg, const int family) 770 { 771 772 switch (family) { 773 case AF_INET: 774 return wg->wg_rtable_ipv4; 775 #ifdef INET6 776 case AF_INET6: 777 return wg->wg_rtable_ipv6; 778 #endif 779 default: 780 return NULL; 781 } 782 } 783 784 785 /* 786 * Global variables 787 */ 788 static volatile unsigned wg_count __cacheline_aligned; 789 790 struct psref_class *wg_psref_class __read_mostly; 791 792 static struct if_clone wg_cloner = 793 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 794 795 static struct pktqueue *wg_pktq __read_mostly; 796 static struct workqueue *wg_wq __read_mostly; 797 798 void wgattach(int); 799 /* ARGSUSED */ 800 void 801 wgattach(int count) 802 { 803 /* 804 * Nothing to do here, initialization is handled by the 805 * module initialization code in wginit() below). 806 */ 807 } 808 809 static void 810 wginit(void) 811 { 812 813 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 814 815 if_clone_attach(&wg_cloner); 816 } 817 818 /* 819 * XXX Kludge: This should just happen in wginit, but workqueue_create 820 * cannot be run until after CPUs have been detected, and wginit runs 821 * before configure. 822 */ 823 static int 824 wginitqueues(void) 825 { 826 int error __diagused; 827 828 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 829 KASSERT(wg_pktq != NULL); 830 831 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 832 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 833 KASSERT(error == 0); 834 835 return 0; 836 } 837 838 static void 839 wg_guarantee_initialized(void) 840 { 841 static ONCE_DECL(init); 842 int error __diagused; 843 844 error = RUN_ONCE(&init, wginitqueues); 845 KASSERT(error == 0); 846 } 847 848 static int 849 wg_count_inc(void) 850 { 851 unsigned o, n; 852 853 do { 854 o = atomic_load_relaxed(&wg_count); 855 if (o == UINT_MAX) 856 return ENFILE; 857 n = o + 1; 858 } while (atomic_cas_uint(&wg_count, o, n) != o); 859 860 return 0; 861 } 862 863 static void 864 wg_count_dec(void) 865 { 866 unsigned c __diagused; 867 868 c = atomic_dec_uint_nv(&wg_count); 869 KASSERT(c != UINT_MAX); 870 } 871 872 static int 873 wgdetach(void) 874 { 875 876 /* Prevent new interface creation. */ 877 if_clone_detach(&wg_cloner); 878 879 /* Check whether there are any existing interfaces. */ 880 if (atomic_load_relaxed(&wg_count)) { 881 /* Back out -- reattach the cloner. */ 882 if_clone_attach(&wg_cloner); 883 return EBUSY; 884 } 885 886 /* No interfaces left. Nuke it. */ 887 workqueue_destroy(wg_wq); 888 pktq_destroy(wg_pktq); 889 psref_class_destroy(wg_psref_class); 890 891 return 0; 892 } 893 894 static void 895 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 896 uint8_t hash[WG_HASH_LEN]) 897 { 898 /* [W] 5.4: CONSTRUCTION */ 899 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 900 /* [W] 5.4: IDENTIFIER */ 901 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 902 struct blake2s state; 903 904 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 905 signature, strlen(signature)); 906 907 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 908 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 909 910 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 911 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 912 blake2s_update(&state, id, strlen(id)); 913 blake2s_final(&state, hash); 914 915 WG_DUMP_HASH("ckey", ckey); 916 WG_DUMP_HASH("hash", hash); 917 } 918 919 static void 920 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 921 const size_t inputsize) 922 { 923 struct blake2s state; 924 925 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 926 blake2s_update(&state, hash, WG_HASH_LEN); 927 blake2s_update(&state, input, inputsize); 928 blake2s_final(&state, hash); 929 } 930 931 static void 932 wg_algo_mac(uint8_t out[], const size_t outsize, 933 const uint8_t key[], const size_t keylen, 934 const uint8_t input1[], const size_t input1len, 935 const uint8_t input2[], const size_t input2len) 936 { 937 struct blake2s state; 938 939 blake2s_init(&state, outsize, key, keylen); 940 941 blake2s_update(&state, input1, input1len); 942 if (input2 != NULL) 943 blake2s_update(&state, input2, input2len); 944 blake2s_final(&state, out); 945 } 946 947 static void 948 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 949 const uint8_t input1[], const size_t input1len, 950 const uint8_t input2[], const size_t input2len) 951 { 952 struct blake2s state; 953 /* [W] 5.4: LABEL-MAC1 */ 954 const char *label = "mac1----"; 955 uint8_t key[WG_HASH_LEN]; 956 957 blake2s_init(&state, sizeof(key), NULL, 0); 958 blake2s_update(&state, label, strlen(label)); 959 blake2s_update(&state, input1, input1len); 960 blake2s_final(&state, key); 961 962 blake2s_init(&state, outsize, key, sizeof(key)); 963 if (input2 != NULL) 964 blake2s_update(&state, input2, input2len); 965 blake2s_final(&state, out); 966 } 967 968 static void 969 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 970 const uint8_t input1[], const size_t input1len) 971 { 972 struct blake2s state; 973 /* [W] 5.4: LABEL-COOKIE */ 974 const char *label = "cookie--"; 975 976 blake2s_init(&state, outsize, NULL, 0); 977 blake2s_update(&state, label, strlen(label)); 978 blake2s_update(&state, input1, input1len); 979 blake2s_final(&state, out); 980 } 981 982 static void 983 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 984 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 985 { 986 987 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 988 989 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 990 crypto_scalarmult_base(pubkey, privkey); 991 } 992 993 static void 994 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 995 const uint8_t privkey[WG_STATIC_KEY_LEN], 996 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 997 { 998 999 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1000 1001 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1002 KASSERT(ret == 0); 1003 } 1004 1005 static void 1006 wg_algo_hmac(uint8_t out[], const size_t outlen, 1007 const uint8_t key[], const size_t keylen, 1008 const uint8_t in[], const size_t inlen) 1009 { 1010 #define IPAD 0x36 1011 #define OPAD 0x5c 1012 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1013 uint8_t ipad[HMAC_BLOCK_LEN]; 1014 uint8_t opad[HMAC_BLOCK_LEN]; 1015 size_t i; 1016 struct blake2s state; 1017 1018 KASSERT(outlen == WG_HASH_LEN); 1019 KASSERT(keylen <= HMAC_BLOCK_LEN); 1020 1021 memcpy(hmackey, key, keylen); 1022 1023 for (i = 0; i < sizeof(hmackey); i++) { 1024 ipad[i] = hmackey[i] ^ IPAD; 1025 opad[i] = hmackey[i] ^ OPAD; 1026 } 1027 1028 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1029 blake2s_update(&state, ipad, sizeof(ipad)); 1030 blake2s_update(&state, in, inlen); 1031 blake2s_final(&state, out); 1032 1033 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1034 blake2s_update(&state, opad, sizeof(opad)); 1035 blake2s_update(&state, out, WG_HASH_LEN); 1036 blake2s_final(&state, out); 1037 #undef IPAD 1038 #undef OPAD 1039 } 1040 1041 static void 1042 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1043 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1044 const uint8_t input[], const size_t inputlen) 1045 { 1046 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1047 uint8_t one[1]; 1048 1049 /* 1050 * [N] 4.3: "an input_key_material byte sequence with length 1051 * either zero bytes, 32 bytes, or DHLEN bytes." 1052 */ 1053 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1054 1055 WG_DUMP_HASH("ckey", ckey); 1056 if (input != NULL) 1057 WG_DUMP_HASH("input", input); 1058 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1059 input, inputlen); 1060 WG_DUMP_HASH("tmp1", tmp1); 1061 one[0] = 1; 1062 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1063 one, sizeof(one)); 1064 WG_DUMP_HASH("out1", out1); 1065 if (out2 == NULL) 1066 return; 1067 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1068 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1069 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1070 tmp2, sizeof(tmp2)); 1071 WG_DUMP_HASH("out2", out2); 1072 if (out3 == NULL) 1073 return; 1074 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1075 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1076 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1077 tmp2, sizeof(tmp2)); 1078 WG_DUMP_HASH("out3", out3); 1079 } 1080 1081 static void __noinline 1082 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1083 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1084 const uint8_t local_key[WG_STATIC_KEY_LEN], 1085 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1086 { 1087 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1088 1089 wg_algo_dh(dhout, local_key, remote_key); 1090 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1091 1092 WG_DUMP_HASH("dhout", dhout); 1093 WG_DUMP_HASH("ckey", ckey); 1094 if (cipher_key != NULL) 1095 WG_DUMP_HASH("cipher_key", cipher_key); 1096 } 1097 1098 static void 1099 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1100 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1101 const uint8_t auth[], size_t authlen) 1102 { 1103 uint8_t nonce[(32 + 64) / 8] = {0}; 1104 long long unsigned int outsize; 1105 int error __diagused; 1106 1107 le64enc(&nonce[4], counter); 1108 1109 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1110 plainsize, auth, authlen, NULL, nonce, key); 1111 KASSERT(error == 0); 1112 KASSERT(outsize == expected_outsize); 1113 } 1114 1115 static int 1116 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1117 const uint64_t counter, const uint8_t encrypted[], 1118 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1119 { 1120 uint8_t nonce[(32 + 64) / 8] = {0}; 1121 long long unsigned int outsize; 1122 int error; 1123 1124 le64enc(&nonce[4], counter); 1125 1126 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1127 encrypted, encryptedsize, auth, authlen, nonce, key); 1128 if (error == 0) 1129 KASSERT(outsize == expected_outsize); 1130 return error; 1131 } 1132 1133 static void 1134 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1135 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1136 const uint8_t auth[], size_t authlen, 1137 const uint8_t nonce[WG_SALT_LEN]) 1138 { 1139 long long unsigned int outsize; 1140 int error __diagused; 1141 1142 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1143 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1144 plain, plainsize, auth, authlen, NULL, nonce, key); 1145 KASSERT(error == 0); 1146 KASSERT(outsize == expected_outsize); 1147 } 1148 1149 static int 1150 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1151 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1152 const uint8_t auth[], size_t authlen, 1153 const uint8_t nonce[WG_SALT_LEN]) 1154 { 1155 long long unsigned int outsize; 1156 int error; 1157 1158 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1159 encrypted, encryptedsize, auth, authlen, nonce, key); 1160 if (error == 0) 1161 KASSERT(outsize == expected_outsize); 1162 return error; 1163 } 1164 1165 static void 1166 wg_algo_tai64n(wg_timestamp_t timestamp) 1167 { 1168 struct timespec ts; 1169 1170 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1171 getnanotime(&ts); 1172 /* TAI64 label in external TAI64 format */ 1173 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1174 /* second beginning from 1970 TAI */ 1175 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1176 /* nanosecond in big-endian format */ 1177 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1178 } 1179 1180 /* 1181 * wg_get_stable_session(wgp, psref) 1182 * 1183 * Get a passive reference to the current stable session, or 1184 * return NULL if there is no current stable session. 1185 * 1186 * The pointer is always there but the session is not necessarily 1187 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1188 * the session may transition from ESTABLISHED to DESTROYING while 1189 * holding the passive reference. 1190 */ 1191 static struct wg_session * 1192 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1193 { 1194 int s; 1195 struct wg_session *wgs; 1196 1197 s = pserialize_read_enter(); 1198 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1199 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1200 wgs = NULL; 1201 else 1202 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1203 pserialize_read_exit(s); 1204 1205 return wgs; 1206 } 1207 1208 static void 1209 wg_put_session(struct wg_session *wgs, struct psref *psref) 1210 { 1211 1212 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1213 } 1214 1215 static void 1216 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1217 { 1218 struct wg_peer *wgp = wgs->wgs_peer; 1219 struct wg_session *wgs0 __diagused; 1220 void *garbage; 1221 1222 KASSERT(mutex_owned(wgp->wgp_lock)); 1223 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1224 1225 /* Remove the session from the table. */ 1226 wgs0 = thmap_del(wg->wg_sessions_byindex, 1227 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1228 KASSERT(wgs0 == wgs); 1229 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1230 1231 /* Wait for passive references to drain. */ 1232 pserialize_perform(wgp->wgp_psz); 1233 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1234 1235 /* Free memory, zero state, and transition to UNKNOWN. */ 1236 thmap_gc(wg->wg_sessions_byindex, garbage); 1237 wg_clear_states(wgs); 1238 wgs->wgs_state = WGS_STATE_UNKNOWN; 1239 } 1240 1241 /* 1242 * wg_get_session_index(wg, wgs) 1243 * 1244 * Choose a session index for wgs->wgs_local_index, and store it 1245 * in wg's table of sessions by index. 1246 * 1247 * wgs must be the unstable session of its peer, and must be 1248 * transitioning out of the UNKNOWN state. 1249 */ 1250 static void 1251 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1252 { 1253 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1254 struct wg_session *wgs0; 1255 uint32_t index; 1256 1257 KASSERT(mutex_owned(wgp->wgp_lock)); 1258 KASSERT(wgs == wgp->wgp_session_unstable); 1259 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1260 1261 do { 1262 /* Pick a uniform random index. */ 1263 index = cprng_strong32(); 1264 1265 /* Try to take it. */ 1266 wgs->wgs_local_index = index; 1267 wgs0 = thmap_put(wg->wg_sessions_byindex, 1268 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1269 1270 /* If someone else beat us, start over. */ 1271 } while (__predict_false(wgs0 != wgs)); 1272 } 1273 1274 /* 1275 * wg_put_session_index(wg, wgs) 1276 * 1277 * Remove wgs from the table of sessions by index, wait for any 1278 * passive references to drain, and transition the session to the 1279 * UNKNOWN state. 1280 * 1281 * wgs must be the unstable session of its peer, and must not be 1282 * UNKNOWN or ESTABLISHED. 1283 */ 1284 static void 1285 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1286 { 1287 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1288 1289 KASSERT(mutex_owned(wgp->wgp_lock)); 1290 KASSERT(wgs == wgp->wgp_session_unstable); 1291 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1292 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1293 1294 wg_destroy_session(wg, wgs); 1295 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1296 } 1297 1298 /* 1299 * Handshake patterns 1300 * 1301 * [W] 5: "These messages use the "IK" pattern from Noise" 1302 * [N] 7.5. Interactive handshake patterns (fundamental) 1303 * "The first character refers to the initiators static key:" 1304 * "I = Static key for initiator Immediately transmitted to responder, 1305 * despite reduced or absent identity hiding" 1306 * "The second character refers to the responders static key:" 1307 * "K = Static key for responder Known to initiator" 1308 * "IK: 1309 * <- s 1310 * ... 1311 * -> e, es, s, ss 1312 * <- e, ee, se" 1313 * [N] 9.4. Pattern modifiers 1314 * "IKpsk2: 1315 * <- s 1316 * ... 1317 * -> e, es, s, ss 1318 * <- e, ee, se, psk" 1319 */ 1320 static void 1321 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1322 struct wg_session *wgs, struct wg_msg_init *wgmi) 1323 { 1324 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1325 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1326 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1327 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1328 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1329 1330 KASSERT(mutex_owned(wgp->wgp_lock)); 1331 KASSERT(wgs == wgp->wgp_session_unstable); 1332 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1333 1334 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1335 wgmi->wgmi_sender = wgs->wgs_local_index; 1336 1337 /* [W] 5.4.2: First Message: Initiator to Responder */ 1338 1339 /* Ci := HASH(CONSTRUCTION) */ 1340 /* Hi := HASH(Ci || IDENTIFIER) */ 1341 wg_init_key_and_hash(ckey, hash); 1342 /* Hi := HASH(Hi || Sr^pub) */ 1343 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1344 1345 WG_DUMP_HASH("hash", hash); 1346 1347 /* [N] 2.2: "e" */ 1348 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1349 wg_algo_generate_keypair(pubkey, privkey); 1350 /* Ci := KDF1(Ci, Ei^pub) */ 1351 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1352 /* msg.ephemeral := Ei^pub */ 1353 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1354 /* Hi := HASH(Hi || msg.ephemeral) */ 1355 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1356 1357 WG_DUMP_HASH("ckey", ckey); 1358 WG_DUMP_HASH("hash", hash); 1359 1360 /* [N] 2.2: "es" */ 1361 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1362 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1363 1364 /* [N] 2.2: "s" */ 1365 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1366 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1367 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1368 hash, sizeof(hash)); 1369 /* Hi := HASH(Hi || msg.static) */ 1370 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1371 1372 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1373 1374 /* [N] 2.2: "ss" */ 1375 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1376 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1377 1378 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1379 wg_timestamp_t timestamp; 1380 wg_algo_tai64n(timestamp); 1381 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1382 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1383 /* Hi := HASH(Hi || msg.timestamp) */ 1384 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1385 1386 /* [W] 5.4.4 Cookie MACs */ 1387 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1388 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1389 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1390 /* Need mac1 to decrypt a cookie from a cookie message */ 1391 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1392 sizeof(wgp->wgp_last_sent_mac1)); 1393 wgp->wgp_last_sent_mac1_valid = true; 1394 1395 if (wgp->wgp_latest_cookie_time == 0 || 1396 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1397 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1398 else { 1399 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1400 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1401 (const uint8_t *)wgmi, 1402 offsetof(struct wg_msg_init, wgmi_mac2), 1403 NULL, 0); 1404 } 1405 1406 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1407 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1408 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1409 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1410 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1411 } 1412 1413 static void __noinline 1414 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1415 const struct sockaddr *src) 1416 { 1417 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1418 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1419 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1420 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1421 struct wg_peer *wgp; 1422 struct wg_session *wgs; 1423 int error, ret; 1424 struct psref psref_peer; 1425 uint8_t mac1[WG_MAC_LEN]; 1426 1427 WG_TRACE("init msg received"); 1428 1429 wg_algo_mac_mac1(mac1, sizeof(mac1), 1430 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1431 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1432 1433 /* 1434 * [W] 5.3: Denial of Service Mitigation & Cookies 1435 * "the responder, ..., must always reject messages with an invalid 1436 * msg.mac1" 1437 */ 1438 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1439 WG_DLOG("mac1 is invalid\n"); 1440 return; 1441 } 1442 1443 /* 1444 * [W] 5.4.2: First Message: Initiator to Responder 1445 * "When the responder receives this message, it does the same 1446 * operations so that its final state variables are identical, 1447 * replacing the operands of the DH function to produce equivalent 1448 * values." 1449 * Note that the following comments of operations are just copies of 1450 * the initiator's ones. 1451 */ 1452 1453 /* Ci := HASH(CONSTRUCTION) */ 1454 /* Hi := HASH(Ci || IDENTIFIER) */ 1455 wg_init_key_and_hash(ckey, hash); 1456 /* Hi := HASH(Hi || Sr^pub) */ 1457 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1458 1459 /* [N] 2.2: "e" */ 1460 /* Ci := KDF1(Ci, Ei^pub) */ 1461 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1462 sizeof(wgmi->wgmi_ephemeral)); 1463 /* Hi := HASH(Hi || msg.ephemeral) */ 1464 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1465 1466 WG_DUMP_HASH("ckey", ckey); 1467 1468 /* [N] 2.2: "es" */ 1469 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1470 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1471 1472 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1473 1474 /* [N] 2.2: "s" */ 1475 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1476 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1477 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1478 if (error != 0) { 1479 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1480 "%s: wg_algo_aead_dec for secret key failed\n", 1481 if_name(&wg->wg_if)); 1482 return; 1483 } 1484 /* Hi := HASH(Hi || msg.static) */ 1485 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1486 1487 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1488 if (wgp == NULL) { 1489 WG_DLOG("peer not found\n"); 1490 return; 1491 } 1492 1493 /* 1494 * Lock the peer to serialize access to cookie state. 1495 * 1496 * XXX Can we safely avoid holding the lock across DH? Take it 1497 * just to verify mac2 and then unlock/DH/lock? 1498 */ 1499 mutex_enter(wgp->wgp_lock); 1500 1501 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1502 WG_TRACE("under load"); 1503 /* 1504 * [W] 5.3: Denial of Service Mitigation & Cookies 1505 * "the responder, ..., and when under load may reject messages 1506 * with an invalid msg.mac2. If the responder receives a 1507 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1508 * and is under load, it may respond with a cookie reply 1509 * message" 1510 */ 1511 uint8_t zero[WG_MAC_LEN] = {0}; 1512 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1513 WG_TRACE("sending a cookie message: no cookie included"); 1514 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1515 wgmi->wgmi_mac1, src); 1516 goto out; 1517 } 1518 if (!wgp->wgp_last_sent_cookie_valid) { 1519 WG_TRACE("sending a cookie message: no cookie sent ever"); 1520 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1521 wgmi->wgmi_mac1, src); 1522 goto out; 1523 } 1524 uint8_t mac2[WG_MAC_LEN]; 1525 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1526 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1527 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1528 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1529 WG_DLOG("mac2 is invalid\n"); 1530 goto out; 1531 } 1532 WG_TRACE("under load, but continue to sending"); 1533 } 1534 1535 /* [N] 2.2: "ss" */ 1536 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1537 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1538 1539 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1540 wg_timestamp_t timestamp; 1541 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1542 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1543 hash, sizeof(hash)); 1544 if (error != 0) { 1545 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1546 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1547 if_name(&wg->wg_if), wgp->wgp_name); 1548 goto out; 1549 } 1550 /* Hi := HASH(Hi || msg.timestamp) */ 1551 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1552 1553 /* 1554 * [W] 5.1 "The responder keeps track of the greatest timestamp 1555 * received per peer and discards packets containing 1556 * timestamps less than or equal to it." 1557 */ 1558 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1559 sizeof(timestamp)); 1560 if (ret <= 0) { 1561 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1562 "%s: peer %s: invalid init msg: timestamp is old\n", 1563 if_name(&wg->wg_if), wgp->wgp_name); 1564 goto out; 1565 } 1566 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1567 1568 /* 1569 * Message is good -- we're committing to handle it now, unless 1570 * we were already initiating a session. 1571 */ 1572 wgs = wgp->wgp_session_unstable; 1573 switch (wgs->wgs_state) { 1574 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1575 wg_get_session_index(wg, wgs); 1576 break; 1577 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1578 WG_TRACE("Session already initializing, ignoring the message"); 1579 goto out; 1580 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1581 WG_TRACE("Session already initializing, destroying old states"); 1582 wg_clear_states(wgs); 1583 /* keep session index */ 1584 break; 1585 case WGS_STATE_ESTABLISHED: /* can't happen */ 1586 panic("unstable session can't be established"); 1587 break; 1588 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1589 WG_TRACE("Session destroying, but force to clear"); 1590 callout_stop(&wgp->wgp_session_dtor_timer); 1591 wg_clear_states(wgs); 1592 /* keep session index */ 1593 break; 1594 default: 1595 panic("invalid session state: %d", wgs->wgs_state); 1596 } 1597 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1598 1599 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1600 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1601 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1602 sizeof(wgmi->wgmi_ephemeral)); 1603 1604 wg_update_endpoint_if_necessary(wgp, src); 1605 1606 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1607 1608 wg_calculate_keys(wgs, false); 1609 wg_clear_states(wgs); 1610 1611 out: 1612 mutex_exit(wgp->wgp_lock); 1613 wg_put_peer(wgp, &psref_peer); 1614 } 1615 1616 static struct socket * 1617 wg_get_so_by_af(struct wg_softc *wg, const int af) 1618 { 1619 1620 switch (af) { 1621 #ifdef INET 1622 case AF_INET: 1623 return wg->wg_so4; 1624 #endif 1625 #ifdef INET6 1626 case AF_INET6: 1627 return wg->wg_so6; 1628 #endif 1629 default: 1630 panic("wg: no such af: %d", af); 1631 } 1632 } 1633 1634 static struct socket * 1635 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1636 { 1637 1638 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1639 } 1640 1641 static struct wg_sockaddr * 1642 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1643 { 1644 struct wg_sockaddr *wgsa; 1645 int s; 1646 1647 s = pserialize_read_enter(); 1648 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1649 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1650 pserialize_read_exit(s); 1651 1652 return wgsa; 1653 } 1654 1655 static void 1656 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1657 { 1658 1659 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1660 } 1661 1662 static int 1663 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1664 { 1665 int error; 1666 struct socket *so; 1667 struct psref psref; 1668 struct wg_sockaddr *wgsa; 1669 1670 wgsa = wg_get_endpoint_sa(wgp, &psref); 1671 so = wg_get_so_by_peer(wgp, wgsa); 1672 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1673 wg_put_sa(wgp, wgsa, &psref); 1674 1675 return error; 1676 } 1677 1678 static int 1679 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1680 { 1681 int error; 1682 struct mbuf *m; 1683 struct wg_msg_init *wgmi; 1684 struct wg_session *wgs; 1685 1686 KASSERT(mutex_owned(wgp->wgp_lock)); 1687 1688 wgs = wgp->wgp_session_unstable; 1689 /* XXX pull dispatch out into wg_task_send_init_message */ 1690 switch (wgs->wgs_state) { 1691 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1692 wg_get_session_index(wg, wgs); 1693 break; 1694 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1695 WG_TRACE("Session already initializing, skip starting new one"); 1696 return EBUSY; 1697 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1698 WG_TRACE("Session already initializing, destroying old states"); 1699 wg_clear_states(wgs); 1700 /* keep session index */ 1701 break; 1702 case WGS_STATE_ESTABLISHED: /* can't happen */ 1703 panic("unstable session can't be established"); 1704 break; 1705 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1706 WG_TRACE("Session destroying"); 1707 /* XXX should wait? */ 1708 return EBUSY; 1709 } 1710 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1711 1712 m = m_gethdr(M_WAIT, MT_DATA); 1713 if (sizeof(*wgmi) > MHLEN) { 1714 m_clget(m, M_WAIT); 1715 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1716 } 1717 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1718 wgmi = mtod(m, struct wg_msg_init *); 1719 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1720 1721 error = wg->wg_ops->send_hs_msg(wgp, m); 1722 if (error == 0) { 1723 WG_TRACE("init msg sent"); 1724 1725 if (wgp->wgp_handshake_start_time == 0) 1726 wgp->wgp_handshake_start_time = time_uptime; 1727 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1728 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1729 } else { 1730 wg_put_session_index(wg, wgs); 1731 /* Initiation failed; toss packet waiting for it if any. */ 1732 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) 1733 m_freem(m); 1734 } 1735 1736 return error; 1737 } 1738 1739 static void 1740 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1741 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1742 const struct wg_msg_init *wgmi) 1743 { 1744 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1745 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1746 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1747 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1748 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1749 1750 KASSERT(mutex_owned(wgp->wgp_lock)); 1751 KASSERT(wgs == wgp->wgp_session_unstable); 1752 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1753 1754 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1755 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1756 1757 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1758 wgmr->wgmr_sender = wgs->wgs_local_index; 1759 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1760 1761 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1762 1763 /* [N] 2.2: "e" */ 1764 /* Er^priv, Er^pub := DH-GENERATE() */ 1765 wg_algo_generate_keypair(pubkey, privkey); 1766 /* Cr := KDF1(Cr, Er^pub) */ 1767 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1768 /* msg.ephemeral := Er^pub */ 1769 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1770 /* Hr := HASH(Hr || msg.ephemeral) */ 1771 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1772 1773 WG_DUMP_HASH("ckey", ckey); 1774 WG_DUMP_HASH("hash", hash); 1775 1776 /* [N] 2.2: "ee" */ 1777 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1778 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1779 1780 /* [N] 2.2: "se" */ 1781 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1782 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1783 1784 /* [N] 9.2: "psk" */ 1785 { 1786 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1787 /* Cr, r, k := KDF3(Cr, Q) */ 1788 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1789 sizeof(wgp->wgp_psk)); 1790 /* Hr := HASH(Hr || r) */ 1791 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1792 } 1793 1794 /* msg.empty := AEAD(k, 0, e, Hr) */ 1795 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1796 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1797 /* Hr := HASH(Hr || msg.empty) */ 1798 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1799 1800 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1801 1802 /* [W] 5.4.4: Cookie MACs */ 1803 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1804 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1805 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1806 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1807 /* Need mac1 to decrypt a cookie from a cookie message */ 1808 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1809 sizeof(wgp->wgp_last_sent_mac1)); 1810 wgp->wgp_last_sent_mac1_valid = true; 1811 1812 if (wgp->wgp_latest_cookie_time == 0 || 1813 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1814 /* msg.mac2 := 0^16 */ 1815 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1816 else { 1817 /* msg.mac2 := MAC(Lm, msg_b) */ 1818 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1819 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1820 (const uint8_t *)wgmr, 1821 offsetof(struct wg_msg_resp, wgmr_mac2), 1822 NULL, 0); 1823 } 1824 1825 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1826 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1827 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1828 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1829 wgs->wgs_remote_index = wgmi->wgmi_sender; 1830 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1831 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1832 } 1833 1834 static void 1835 wg_swap_sessions(struct wg_peer *wgp) 1836 { 1837 struct wg_session *wgs, *wgs_prev; 1838 1839 KASSERT(mutex_owned(wgp->wgp_lock)); 1840 1841 wgs = wgp->wgp_session_unstable; 1842 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1843 1844 wgs_prev = wgp->wgp_session_stable; 1845 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1846 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1847 atomic_store_release(&wgp->wgp_session_stable, wgs); 1848 wgp->wgp_session_unstable = wgs_prev; 1849 } 1850 1851 static void __noinline 1852 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1853 const struct sockaddr *src) 1854 { 1855 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1856 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1857 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1858 struct wg_peer *wgp; 1859 struct wg_session *wgs; 1860 struct psref psref; 1861 int error; 1862 uint8_t mac1[WG_MAC_LEN]; 1863 struct wg_session *wgs_prev; 1864 struct mbuf *m; 1865 1866 wg_algo_mac_mac1(mac1, sizeof(mac1), 1867 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1868 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1869 1870 /* 1871 * [W] 5.3: Denial of Service Mitigation & Cookies 1872 * "the responder, ..., must always reject messages with an invalid 1873 * msg.mac1" 1874 */ 1875 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1876 WG_DLOG("mac1 is invalid\n"); 1877 return; 1878 } 1879 1880 WG_TRACE("resp msg received"); 1881 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1882 if (wgs == NULL) { 1883 WG_TRACE("No session found"); 1884 return; 1885 } 1886 1887 wgp = wgs->wgs_peer; 1888 1889 mutex_enter(wgp->wgp_lock); 1890 1891 /* If we weren't waiting for a handshake response, drop it. */ 1892 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1893 WG_TRACE("peer sent spurious handshake response, ignoring"); 1894 goto out; 1895 } 1896 1897 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1898 WG_TRACE("under load"); 1899 /* 1900 * [W] 5.3: Denial of Service Mitigation & Cookies 1901 * "the responder, ..., and when under load may reject messages 1902 * with an invalid msg.mac2. If the responder receives a 1903 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1904 * and is under load, it may respond with a cookie reply 1905 * message" 1906 */ 1907 uint8_t zero[WG_MAC_LEN] = {0}; 1908 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1909 WG_TRACE("sending a cookie message: no cookie included"); 1910 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1911 wgmr->wgmr_mac1, src); 1912 goto out; 1913 } 1914 if (!wgp->wgp_last_sent_cookie_valid) { 1915 WG_TRACE("sending a cookie message: no cookie sent ever"); 1916 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1917 wgmr->wgmr_mac1, src); 1918 goto out; 1919 } 1920 uint8_t mac2[WG_MAC_LEN]; 1921 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1922 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1923 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1924 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1925 WG_DLOG("mac2 is invalid\n"); 1926 goto out; 1927 } 1928 WG_TRACE("under load, but continue to sending"); 1929 } 1930 1931 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1932 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1933 1934 /* 1935 * [W] 5.4.3 Second Message: Responder to Initiator 1936 * "When the initiator receives this message, it does the same 1937 * operations so that its final state variables are identical, 1938 * replacing the operands of the DH function to produce equivalent 1939 * values." 1940 * Note that the following comments of operations are just copies of 1941 * the initiator's ones. 1942 */ 1943 1944 /* [N] 2.2: "e" */ 1945 /* Cr := KDF1(Cr, Er^pub) */ 1946 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1947 sizeof(wgmr->wgmr_ephemeral)); 1948 /* Hr := HASH(Hr || msg.ephemeral) */ 1949 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1950 1951 WG_DUMP_HASH("ckey", ckey); 1952 WG_DUMP_HASH("hash", hash); 1953 1954 /* [N] 2.2: "ee" */ 1955 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1956 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1957 wgmr->wgmr_ephemeral); 1958 1959 /* [N] 2.2: "se" */ 1960 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1961 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1962 1963 /* [N] 9.2: "psk" */ 1964 { 1965 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1966 /* Cr, r, k := KDF3(Cr, Q) */ 1967 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1968 sizeof(wgp->wgp_psk)); 1969 /* Hr := HASH(Hr || r) */ 1970 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1971 } 1972 1973 { 1974 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 1975 /* msg.empty := AEAD(k, 0, e, Hr) */ 1976 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 1977 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 1978 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1979 if (error != 0) { 1980 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1981 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 1982 if_name(&wg->wg_if), wgp->wgp_name); 1983 goto out; 1984 } 1985 /* Hr := HASH(Hr || msg.empty) */ 1986 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1987 } 1988 1989 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 1990 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 1991 wgs->wgs_remote_index = wgmr->wgmr_sender; 1992 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1993 1994 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1995 wgs->wgs_state = WGS_STATE_ESTABLISHED; 1996 wgs->wgs_time_established = time_uptime; 1997 wgs->wgs_time_last_data_sent = 0; 1998 wgs->wgs_is_initiator = true; 1999 wg_calculate_keys(wgs, true); 2000 wg_clear_states(wgs); 2001 WG_TRACE("WGS_STATE_ESTABLISHED"); 2002 2003 callout_stop(&wgp->wgp_handshake_timeout_timer); 2004 2005 wg_swap_sessions(wgp); 2006 KASSERT(wgs == wgp->wgp_session_stable); 2007 wgs_prev = wgp->wgp_session_unstable; 2008 getnanotime(&wgp->wgp_last_handshake_time); 2009 wgp->wgp_handshake_start_time = 0; 2010 wgp->wgp_last_sent_mac1_valid = false; 2011 wgp->wgp_last_sent_cookie_valid = false; 2012 2013 wg_schedule_rekey_timer(wgp); 2014 2015 wg_update_endpoint_if_necessary(wgp, src); 2016 2017 /* 2018 * If we had a data packet queued up, send it; otherwise send a 2019 * keepalive message -- either way we have to send something 2020 * immediately or else the responder will never answer. 2021 */ 2022 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2023 kpreempt_disable(); 2024 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2025 M_SETCTX(m, wgp); 2026 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2027 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2028 if_name(&wg->wg_if)); 2029 m_freem(m); 2030 } 2031 kpreempt_enable(); 2032 } else { 2033 wg_send_keepalive_msg(wgp, wgs); 2034 } 2035 2036 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2037 /* Wait for wg_get_stable_session to drain. */ 2038 pserialize_perform(wgp->wgp_psz); 2039 2040 /* Transition ESTABLISHED->DESTROYING. */ 2041 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2042 2043 /* We can't destroy the old session immediately */ 2044 wg_schedule_session_dtor_timer(wgp); 2045 } else { 2046 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2047 "state=%d", wgs_prev->wgs_state); 2048 } 2049 2050 out: 2051 mutex_exit(wgp->wgp_lock); 2052 wg_put_session(wgs, &psref); 2053 } 2054 2055 static int 2056 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2057 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2058 { 2059 int error; 2060 struct mbuf *m; 2061 struct wg_msg_resp *wgmr; 2062 2063 KASSERT(mutex_owned(wgp->wgp_lock)); 2064 KASSERT(wgs == wgp->wgp_session_unstable); 2065 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 2066 2067 m = m_gethdr(M_WAIT, MT_DATA); 2068 if (sizeof(*wgmr) > MHLEN) { 2069 m_clget(m, M_WAIT); 2070 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2071 } 2072 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2073 wgmr = mtod(m, struct wg_msg_resp *); 2074 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2075 2076 error = wg->wg_ops->send_hs_msg(wgp, m); 2077 if (error == 0) 2078 WG_TRACE("resp msg sent"); 2079 return error; 2080 } 2081 2082 static struct wg_peer * 2083 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2084 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2085 { 2086 struct wg_peer *wgp; 2087 2088 int s = pserialize_read_enter(); 2089 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2090 if (wgp != NULL) 2091 wg_get_peer(wgp, psref); 2092 pserialize_read_exit(s); 2093 2094 return wgp; 2095 } 2096 2097 static void 2098 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2099 struct wg_msg_cookie *wgmc, const uint32_t sender, 2100 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2101 { 2102 uint8_t cookie[WG_COOKIE_LEN]; 2103 uint8_t key[WG_HASH_LEN]; 2104 uint8_t addr[sizeof(struct in6_addr)]; 2105 size_t addrlen; 2106 uint16_t uh_sport; /* be */ 2107 2108 KASSERT(mutex_owned(wgp->wgp_lock)); 2109 2110 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2111 wgmc->wgmc_receiver = sender; 2112 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2113 2114 /* 2115 * [W] 5.4.7: Under Load: Cookie Reply Message 2116 * "The secret variable, Rm, changes every two minutes to a 2117 * random value" 2118 */ 2119 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2120 wgp->wgp_randval = cprng_strong32(); 2121 wgp->wgp_last_genrandval_time = time_uptime; 2122 } 2123 2124 switch (src->sa_family) { 2125 case AF_INET: { 2126 const struct sockaddr_in *sin = satocsin(src); 2127 addrlen = sizeof(sin->sin_addr); 2128 memcpy(addr, &sin->sin_addr, addrlen); 2129 uh_sport = sin->sin_port; 2130 break; 2131 } 2132 #ifdef INET6 2133 case AF_INET6: { 2134 const struct sockaddr_in6 *sin6 = satocsin6(src); 2135 addrlen = sizeof(sin6->sin6_addr); 2136 memcpy(addr, &sin6->sin6_addr, addrlen); 2137 uh_sport = sin6->sin6_port; 2138 break; 2139 } 2140 #endif 2141 default: 2142 panic("invalid af=%d", src->sa_family); 2143 } 2144 2145 wg_algo_mac(cookie, sizeof(cookie), 2146 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2147 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2148 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2149 sizeof(wg->wg_pubkey)); 2150 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2151 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2152 2153 /* Need to store to calculate mac2 */ 2154 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2155 wgp->wgp_last_sent_cookie_valid = true; 2156 } 2157 2158 static int 2159 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2160 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2161 const struct sockaddr *src) 2162 { 2163 int error; 2164 struct mbuf *m; 2165 struct wg_msg_cookie *wgmc; 2166 2167 KASSERT(mutex_owned(wgp->wgp_lock)); 2168 2169 m = m_gethdr(M_WAIT, MT_DATA); 2170 if (sizeof(*wgmc) > MHLEN) { 2171 m_clget(m, M_WAIT); 2172 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2173 } 2174 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2175 wgmc = mtod(m, struct wg_msg_cookie *); 2176 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2177 2178 error = wg->wg_ops->send_hs_msg(wgp, m); 2179 if (error == 0) 2180 WG_TRACE("cookie msg sent"); 2181 return error; 2182 } 2183 2184 static bool 2185 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2186 { 2187 #ifdef WG_DEBUG_PARAMS 2188 if (wg_force_underload) 2189 return true; 2190 #endif 2191 2192 /* 2193 * XXX we don't have a means of a load estimation. The purpose of 2194 * the mechanism is a DoS mitigation, so we consider frequent handshake 2195 * messages as (a kind of) load; if a message of the same type comes 2196 * to a peer within 1 second, we consider we are under load. 2197 */ 2198 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2199 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2200 return (time_uptime - last) == 0; 2201 } 2202 2203 static void 2204 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2205 { 2206 2207 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2208 2209 /* 2210 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2211 */ 2212 if (initiator) { 2213 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2214 wgs->wgs_chaining_key, NULL, 0); 2215 } else { 2216 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2217 wgs->wgs_chaining_key, NULL, 0); 2218 } 2219 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2220 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2221 } 2222 2223 static uint64_t 2224 wg_session_get_send_counter(struct wg_session *wgs) 2225 { 2226 #ifdef __HAVE_ATOMIC64_LOADSTORE 2227 return atomic_load_relaxed(&wgs->wgs_send_counter); 2228 #else 2229 uint64_t send_counter; 2230 2231 mutex_enter(&wgs->wgs_send_counter_lock); 2232 send_counter = wgs->wgs_send_counter; 2233 mutex_exit(&wgs->wgs_send_counter_lock); 2234 2235 return send_counter; 2236 #endif 2237 } 2238 2239 static uint64_t 2240 wg_session_inc_send_counter(struct wg_session *wgs) 2241 { 2242 #ifdef __HAVE_ATOMIC64_LOADSTORE 2243 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2244 #else 2245 uint64_t send_counter; 2246 2247 mutex_enter(&wgs->wgs_send_counter_lock); 2248 send_counter = wgs->wgs_send_counter++; 2249 mutex_exit(&wgs->wgs_send_counter_lock); 2250 2251 return send_counter; 2252 #endif 2253 } 2254 2255 static void 2256 wg_clear_states(struct wg_session *wgs) 2257 { 2258 2259 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2260 2261 wgs->wgs_send_counter = 0; 2262 sliwin_reset(&wgs->wgs_recvwin->window); 2263 2264 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2265 wgs_clear(handshake_hash); 2266 wgs_clear(chaining_key); 2267 wgs_clear(ephemeral_key_pub); 2268 wgs_clear(ephemeral_key_priv); 2269 wgs_clear(ephemeral_key_peer); 2270 #undef wgs_clear 2271 } 2272 2273 static struct wg_session * 2274 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2275 struct psref *psref) 2276 { 2277 struct wg_session *wgs; 2278 2279 int s = pserialize_read_enter(); 2280 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2281 if (wgs != NULL) { 2282 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2283 WGS_STATE_UNKNOWN); 2284 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2285 } 2286 pserialize_read_exit(s); 2287 2288 return wgs; 2289 } 2290 2291 static void 2292 wg_schedule_rekey_timer(struct wg_peer *wgp) 2293 { 2294 int timeout = MIN(wg_rekey_after_time, (unsigned)(INT_MAX / hz)); 2295 2296 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2297 } 2298 2299 static void 2300 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2301 { 2302 struct mbuf *m; 2303 2304 /* 2305 * [W] 6.5 Passive Keepalive 2306 * "A keepalive message is simply a transport data message with 2307 * a zero-length encapsulated encrypted inner-packet." 2308 */ 2309 m = m_gethdr(M_WAIT, MT_DATA); 2310 wg_send_data_msg(wgp, wgs, m); 2311 } 2312 2313 static bool 2314 wg_need_to_send_init_message(struct wg_session *wgs) 2315 { 2316 /* 2317 * [W] 6.2 Transport Message Limits 2318 * "if a peer is the initiator of a current secure session, 2319 * WireGuard will send a handshake initiation message to begin 2320 * a new secure session ... if after receiving a transport data 2321 * message, the current secure session is (REJECT-AFTER-TIME 2322 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2323 * not yet acted upon this event." 2324 */ 2325 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2326 (time_uptime - wgs->wgs_time_established) >= 2327 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2328 } 2329 2330 static void 2331 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2332 { 2333 2334 mutex_enter(wgp->wgp_intr_lock); 2335 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2336 if (wgp->wgp_tasks == 0) 2337 /* 2338 * XXX If the current CPU is already loaded -- e.g., if 2339 * there's already a bunch of handshakes queued up -- 2340 * consider tossing this over to another CPU to 2341 * distribute the load. 2342 */ 2343 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2344 wgp->wgp_tasks |= task; 2345 mutex_exit(wgp->wgp_intr_lock); 2346 } 2347 2348 static void 2349 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2350 { 2351 struct wg_sockaddr *wgsa_prev; 2352 2353 WG_TRACE("Changing endpoint"); 2354 2355 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2356 wgsa_prev = wgp->wgp_endpoint; 2357 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2358 wgp->wgp_endpoint0 = wgsa_prev; 2359 atomic_store_release(&wgp->wgp_endpoint_available, true); 2360 2361 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2362 } 2363 2364 static bool 2365 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2366 { 2367 uint16_t packet_len; 2368 const struct ip *ip; 2369 2370 if (__predict_false(decrypted_len < sizeof(struct ip))) 2371 return false; 2372 2373 ip = (const struct ip *)packet; 2374 if (ip->ip_v == 4) 2375 *af = AF_INET; 2376 else if (ip->ip_v == 6) 2377 *af = AF_INET6; 2378 else 2379 return false; 2380 2381 WG_DLOG("af=%d\n", *af); 2382 2383 switch (*af) { 2384 #ifdef INET 2385 case AF_INET: 2386 packet_len = ntohs(ip->ip_len); 2387 break; 2388 #endif 2389 #ifdef INET6 2390 case AF_INET6: { 2391 const struct ip6_hdr *ip6; 2392 2393 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr))) 2394 return false; 2395 2396 ip6 = (const struct ip6_hdr *)packet; 2397 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen); 2398 break; 2399 } 2400 #endif 2401 default: 2402 return false; 2403 } 2404 2405 WG_DLOG("packet_len=%u\n", packet_len); 2406 if (packet_len > decrypted_len) 2407 return false; 2408 2409 return true; 2410 } 2411 2412 static bool 2413 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2414 int af, char *packet) 2415 { 2416 struct sockaddr_storage ss; 2417 struct sockaddr *sa; 2418 struct psref psref; 2419 struct wg_peer *wgp; 2420 bool ok; 2421 2422 /* 2423 * II CRYPTOKEY ROUTING 2424 * "it will only accept it if its source IP resolves in the 2425 * table to the public key used in the secure session for 2426 * decrypting it." 2427 */ 2428 2429 if (af == AF_INET) { 2430 const struct ip *ip = (const struct ip *)packet; 2431 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2432 sockaddr_in_init(sin, &ip->ip_src, 0); 2433 sa = sintosa(sin); 2434 #ifdef INET6 2435 } else { 2436 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2437 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2438 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2439 sa = sin6tosa(sin6); 2440 #endif 2441 } 2442 2443 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2444 ok = (wgp == wgp_expected); 2445 if (wgp != NULL) 2446 wg_put_peer(wgp, &psref); 2447 2448 return ok; 2449 } 2450 2451 static void 2452 wg_session_dtor_timer(void *arg) 2453 { 2454 struct wg_peer *wgp = arg; 2455 2456 WG_TRACE("enter"); 2457 2458 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2459 } 2460 2461 static void 2462 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2463 { 2464 2465 /* 1 second grace period */ 2466 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2467 } 2468 2469 static bool 2470 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2471 { 2472 if (sa1->sa_family != sa2->sa_family) 2473 return false; 2474 2475 switch (sa1->sa_family) { 2476 #ifdef INET 2477 case AF_INET: 2478 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2479 #endif 2480 #ifdef INET6 2481 case AF_INET6: 2482 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2483 #endif 2484 default: 2485 return false; 2486 } 2487 } 2488 2489 static void 2490 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2491 const struct sockaddr *src) 2492 { 2493 struct wg_sockaddr *wgsa; 2494 struct psref psref; 2495 2496 wgsa = wg_get_endpoint_sa(wgp, &psref); 2497 2498 #ifdef WG_DEBUG_LOG 2499 char oldaddr[128], newaddr[128]; 2500 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2501 sockaddr_format(src, newaddr, sizeof(newaddr)); 2502 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2503 #endif 2504 2505 /* 2506 * III: "Since the packet has authenticated correctly, the source IP of 2507 * the outer UDP/IP packet is used to update the endpoint for peer..." 2508 */ 2509 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2510 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2511 /* XXX We can't change the endpoint twice in a short period */ 2512 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2513 wg_change_endpoint(wgp, src); 2514 } 2515 } 2516 2517 wg_put_sa(wgp, wgsa, &psref); 2518 } 2519 2520 static void __noinline 2521 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2522 const struct sockaddr *src) 2523 { 2524 struct wg_msg_data *wgmd; 2525 char *encrypted_buf = NULL, *decrypted_buf; 2526 size_t encrypted_len, decrypted_len; 2527 struct wg_session *wgs; 2528 struct wg_peer *wgp; 2529 int state; 2530 size_t mlen; 2531 struct psref psref; 2532 int error, af; 2533 bool success, free_encrypted_buf = false, ok; 2534 struct mbuf *n; 2535 2536 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2537 wgmd = mtod(m, struct wg_msg_data *); 2538 2539 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2540 WG_TRACE("data"); 2541 2542 /* Find the putative session, or drop. */ 2543 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2544 if (wgs == NULL) { 2545 WG_TRACE("No session found"); 2546 m_freem(m); 2547 return; 2548 } 2549 2550 /* 2551 * We are only ready to handle data when in INIT_PASSIVE, 2552 * ESTABLISHED, or DESTROYING. All transitions out of that 2553 * state dissociate the session index and drain psrefs. 2554 */ 2555 state = atomic_load_relaxed(&wgs->wgs_state); 2556 switch (state) { 2557 case WGS_STATE_UNKNOWN: 2558 panic("wg session %p in unknown state has session index %u", 2559 wgs, wgmd->wgmd_receiver); 2560 case WGS_STATE_INIT_ACTIVE: 2561 WG_TRACE("not yet ready for data"); 2562 goto out; 2563 case WGS_STATE_INIT_PASSIVE: 2564 case WGS_STATE_ESTABLISHED: 2565 case WGS_STATE_DESTROYING: 2566 break; 2567 } 2568 2569 /* 2570 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2571 * to update the endpoint if authentication succeeds. 2572 */ 2573 wgp = wgs->wgs_peer; 2574 2575 /* 2576 * Reject outrageously wrong sequence numbers before doing any 2577 * crypto work or taking any locks. 2578 */ 2579 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2580 le64toh(wgmd->wgmd_counter)); 2581 if (error) { 2582 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2583 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2584 if_name(&wg->wg_if), wgp->wgp_name, 2585 le64toh(wgmd->wgmd_counter)); 2586 goto out; 2587 } 2588 2589 /* Ensure the payload and authenticator are contiguous. */ 2590 mlen = m_length(m); 2591 encrypted_len = mlen - sizeof(*wgmd); 2592 if (encrypted_len < WG_AUTHTAG_LEN) { 2593 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2594 goto out; 2595 } 2596 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2597 if (success) { 2598 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2599 } else { 2600 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2601 if (encrypted_buf == NULL) { 2602 WG_DLOG("failed to allocate encrypted_buf\n"); 2603 goto out; 2604 } 2605 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2606 free_encrypted_buf = true; 2607 } 2608 /* m_ensure_contig may change m regardless of its result */ 2609 KASSERT(m->m_len >= sizeof(*wgmd)); 2610 wgmd = mtod(m, struct wg_msg_data *); 2611 2612 /* 2613 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2614 * a zero-length buffer (XXX). Drop if plaintext is longer 2615 * than MCLBYTES (XXX). 2616 */ 2617 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2618 if (decrypted_len > MCLBYTES) { 2619 /* FIXME handle larger data than MCLBYTES */ 2620 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2621 goto out; 2622 } 2623 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2624 if (n == NULL) { 2625 WG_DLOG("wg_get_mbuf failed\n"); 2626 goto out; 2627 } 2628 decrypted_buf = mtod(n, char *); 2629 2630 /* Decrypt and verify the packet. */ 2631 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2632 error = wg_algo_aead_dec(decrypted_buf, 2633 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2634 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2635 encrypted_len, NULL, 0); 2636 if (error != 0) { 2637 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2638 "%s: peer %s: failed to wg_algo_aead_dec\n", 2639 if_name(&wg->wg_if), wgp->wgp_name); 2640 m_freem(n); 2641 goto out; 2642 } 2643 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2644 2645 /* Packet is genuine. Reject it if a replay or just too old. */ 2646 mutex_enter(&wgs->wgs_recvwin->lock); 2647 error = sliwin_update(&wgs->wgs_recvwin->window, 2648 le64toh(wgmd->wgmd_counter)); 2649 mutex_exit(&wgs->wgs_recvwin->lock); 2650 if (error) { 2651 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2652 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2653 if_name(&wg->wg_if), wgp->wgp_name, 2654 le64toh(wgmd->wgmd_counter)); 2655 m_freem(n); 2656 goto out; 2657 } 2658 2659 /* We're done with m now; free it and chuck the pointers. */ 2660 m_freem(m); 2661 m = NULL; 2662 wgmd = NULL; 2663 2664 /* 2665 * Validate the encapsulated packet header and get the address 2666 * family, or drop. 2667 */ 2668 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2669 if (!ok) { 2670 m_freem(n); 2671 goto out; 2672 } 2673 2674 /* 2675 * The packet is genuine. Update the peer's endpoint if the 2676 * source address changed. 2677 * 2678 * XXX How to prevent DoS by replaying genuine packets from the 2679 * wrong source address? 2680 */ 2681 wg_update_endpoint_if_necessary(wgp, src); 2682 2683 /* Submit it into our network stack if routable. */ 2684 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2685 if (ok) { 2686 wg->wg_ops->input(&wg->wg_if, n, af); 2687 } else { 2688 char addrstr[INET6_ADDRSTRLEN]; 2689 memset(addrstr, 0, sizeof(addrstr)); 2690 if (af == AF_INET) { 2691 const struct ip *ip = (const struct ip *)decrypted_buf; 2692 IN_PRINT(addrstr, &ip->ip_src); 2693 #ifdef INET6 2694 } else if (af == AF_INET6) { 2695 const struct ip6_hdr *ip6 = 2696 (const struct ip6_hdr *)decrypted_buf; 2697 IN6_PRINT(addrstr, &ip6->ip6_src); 2698 #endif 2699 } 2700 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2701 "%s: peer %s: invalid source address (%s)\n", 2702 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2703 m_freem(n); 2704 /* 2705 * The inner address is invalid however the session is valid 2706 * so continue the session processing below. 2707 */ 2708 } 2709 n = NULL; 2710 2711 /* Update the state machine if necessary. */ 2712 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2713 /* 2714 * We were waiting for the initiator to send their 2715 * first data transport message, and that has happened. 2716 * Schedule a task to establish this session. 2717 */ 2718 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2719 } else { 2720 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2721 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2722 } 2723 /* 2724 * [W] 6.5 Passive Keepalive 2725 * "If a peer has received a validly-authenticated transport 2726 * data message (section 5.4.6), but does not have any packets 2727 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2728 * a keepalive message." 2729 */ 2730 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2731 (uintmax_t)time_uptime, 2732 (uintmax_t)wgs->wgs_time_last_data_sent); 2733 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2734 wg_keepalive_timeout) { 2735 WG_TRACE("Schedule sending keepalive message"); 2736 /* 2737 * We can't send a keepalive message here to avoid 2738 * a deadlock; we already hold the solock of a socket 2739 * that is used to send the message. 2740 */ 2741 wg_schedule_peer_task(wgp, 2742 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2743 } 2744 } 2745 out: 2746 wg_put_session(wgs, &psref); 2747 if (m != NULL) 2748 m_freem(m); 2749 if (free_encrypted_buf) 2750 kmem_intr_free(encrypted_buf, encrypted_len); 2751 } 2752 2753 static void __noinline 2754 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2755 { 2756 struct wg_session *wgs; 2757 struct wg_peer *wgp; 2758 struct psref psref; 2759 int error; 2760 uint8_t key[WG_HASH_LEN]; 2761 uint8_t cookie[WG_COOKIE_LEN]; 2762 2763 WG_TRACE("cookie msg received"); 2764 2765 /* Find the putative session. */ 2766 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2767 if (wgs == NULL) { 2768 WG_TRACE("No session found"); 2769 return; 2770 } 2771 2772 /* Lock the peer so we can update the cookie state. */ 2773 wgp = wgs->wgs_peer; 2774 mutex_enter(wgp->wgp_lock); 2775 2776 if (!wgp->wgp_last_sent_mac1_valid) { 2777 WG_TRACE("No valid mac1 sent (or expired)"); 2778 goto out; 2779 } 2780 2781 /* Decrypt the cookie and store it for later handshake retry. */ 2782 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2783 sizeof(wgp->wgp_pubkey)); 2784 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2785 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2786 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2787 wgmc->wgmc_salt); 2788 if (error != 0) { 2789 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2790 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 2791 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 2792 goto out; 2793 } 2794 /* 2795 * [W] 6.6: Interaction with Cookie Reply System 2796 * "it should simply store the decrypted cookie value from the cookie 2797 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2798 * timer for retrying a handshake initiation message." 2799 */ 2800 wgp->wgp_latest_cookie_time = time_uptime; 2801 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2802 out: 2803 mutex_exit(wgp->wgp_lock); 2804 wg_put_session(wgs, &psref); 2805 } 2806 2807 static struct mbuf * 2808 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2809 { 2810 struct wg_msg wgm; 2811 size_t mbuflen; 2812 size_t msglen; 2813 2814 /* 2815 * Get the mbuf chain length. It is already guaranteed, by 2816 * wg_overudp_cb, to be large enough for a struct wg_msg. 2817 */ 2818 mbuflen = m_length(m); 2819 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2820 2821 /* 2822 * Copy the message header (32-bit message type) out -- we'll 2823 * worry about contiguity and alignment later. 2824 */ 2825 m_copydata(m, 0, sizeof(wgm), &wgm); 2826 switch (le32toh(wgm.wgm_type)) { 2827 case WG_MSG_TYPE_INIT: 2828 msglen = sizeof(struct wg_msg_init); 2829 break; 2830 case WG_MSG_TYPE_RESP: 2831 msglen = sizeof(struct wg_msg_resp); 2832 break; 2833 case WG_MSG_TYPE_COOKIE: 2834 msglen = sizeof(struct wg_msg_cookie); 2835 break; 2836 case WG_MSG_TYPE_DATA: 2837 msglen = sizeof(struct wg_msg_data); 2838 break; 2839 default: 2840 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2841 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 2842 le32toh(wgm.wgm_type)); 2843 goto error; 2844 } 2845 2846 /* Verify the mbuf chain is long enough for this type of message. */ 2847 if (__predict_false(mbuflen < msglen)) { 2848 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2849 le32toh(wgm.wgm_type)); 2850 goto error; 2851 } 2852 2853 /* Make the message header contiguous if necessary. */ 2854 if (__predict_false(m->m_len < msglen)) { 2855 m = m_pullup(m, msglen); 2856 if (m == NULL) 2857 return NULL; 2858 } 2859 2860 return m; 2861 2862 error: 2863 m_freem(m); 2864 return NULL; 2865 } 2866 2867 static void 2868 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2869 const struct sockaddr *src) 2870 { 2871 struct wg_msg *wgm; 2872 2873 m = wg_validate_msg_header(wg, m); 2874 if (__predict_false(m == NULL)) 2875 return; 2876 2877 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2878 wgm = mtod(m, struct wg_msg *); 2879 switch (le32toh(wgm->wgm_type)) { 2880 case WG_MSG_TYPE_INIT: 2881 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2882 break; 2883 case WG_MSG_TYPE_RESP: 2884 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2885 break; 2886 case WG_MSG_TYPE_COOKIE: 2887 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2888 break; 2889 case WG_MSG_TYPE_DATA: 2890 wg_handle_msg_data(wg, m, src); 2891 /* wg_handle_msg_data frees m for us */ 2892 return; 2893 default: 2894 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2895 } 2896 2897 m_freem(m); 2898 } 2899 2900 static void 2901 wg_receive_packets(struct wg_softc *wg, const int af) 2902 { 2903 2904 for (;;) { 2905 int error, flags; 2906 struct socket *so; 2907 struct mbuf *m = NULL; 2908 struct uio dummy_uio; 2909 struct mbuf *paddr = NULL; 2910 struct sockaddr *src; 2911 2912 so = wg_get_so_by_af(wg, af); 2913 flags = MSG_DONTWAIT; 2914 dummy_uio.uio_resid = 1000000000; 2915 2916 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2917 &flags); 2918 if (error || m == NULL) { 2919 //if (error == EWOULDBLOCK) 2920 return; 2921 } 2922 2923 KASSERT(paddr != NULL); 2924 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2925 src = mtod(paddr, struct sockaddr *); 2926 2927 wg_handle_packet(wg, m, src); 2928 } 2929 } 2930 2931 static void 2932 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2933 { 2934 2935 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2936 } 2937 2938 static void 2939 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2940 { 2941 2942 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 2943 } 2944 2945 static void 2946 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 2947 { 2948 struct wg_session *wgs; 2949 2950 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 2951 2952 KASSERT(mutex_owned(wgp->wgp_lock)); 2953 2954 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 2955 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 2956 if_name(&wg->wg_if)); 2957 /* XXX should do something? */ 2958 return; 2959 } 2960 2961 wgs = wgp->wgp_session_stable; 2962 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 2963 /* XXX What if the unstable session is already INIT_ACTIVE? */ 2964 wg_send_handshake_msg_init(wg, wgp); 2965 } else { 2966 /* rekey */ 2967 wgs = wgp->wgp_session_unstable; 2968 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2969 wg_send_handshake_msg_init(wg, wgp); 2970 } 2971 } 2972 2973 static void 2974 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 2975 { 2976 struct wg_session *wgs; 2977 2978 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 2979 2980 KASSERT(mutex_owned(wgp->wgp_lock)); 2981 KASSERT(wgp->wgp_handshake_start_time != 0); 2982 2983 wgs = wgp->wgp_session_unstable; 2984 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2985 return; 2986 2987 /* 2988 * XXX no real need to assign a new index here, but we do need 2989 * to transition to UNKNOWN temporarily 2990 */ 2991 wg_put_session_index(wg, wgs); 2992 2993 /* [W] 6.4 Handshake Initiation Retransmission */ 2994 if ((time_uptime - wgp->wgp_handshake_start_time) > 2995 wg_rekey_attempt_time) { 2996 /* Give up handshaking */ 2997 wgp->wgp_handshake_start_time = 0; 2998 WG_TRACE("give up"); 2999 3000 /* 3001 * If a new data packet comes, handshaking will be retried 3002 * and a new session would be established at that time, 3003 * however we don't want to send pending packets then. 3004 */ 3005 wg_purge_pending_packets(wgp); 3006 return; 3007 } 3008 3009 wg_task_send_init_message(wg, wgp); 3010 } 3011 3012 static void 3013 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3014 { 3015 struct wg_session *wgs, *wgs_prev; 3016 struct mbuf *m; 3017 3018 KASSERT(mutex_owned(wgp->wgp_lock)); 3019 3020 wgs = wgp->wgp_session_unstable; 3021 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3022 /* XXX Can this happen? */ 3023 return; 3024 3025 wgs->wgs_state = WGS_STATE_ESTABLISHED; 3026 wgs->wgs_time_established = time_uptime; 3027 wgs->wgs_time_last_data_sent = 0; 3028 wgs->wgs_is_initiator = false; 3029 WG_TRACE("WGS_STATE_ESTABLISHED"); 3030 3031 wg_swap_sessions(wgp); 3032 KASSERT(wgs == wgp->wgp_session_stable); 3033 wgs_prev = wgp->wgp_session_unstable; 3034 getnanotime(&wgp->wgp_last_handshake_time); 3035 wgp->wgp_handshake_start_time = 0; 3036 wgp->wgp_last_sent_mac1_valid = false; 3037 wgp->wgp_last_sent_cookie_valid = false; 3038 3039 /* If we had a data packet queued up, send it. */ 3040 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3041 kpreempt_disable(); 3042 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3043 M_SETCTX(m, wgp); 3044 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3045 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3046 if_name(&wg->wg_if)); 3047 m_freem(m); 3048 } 3049 kpreempt_enable(); 3050 } 3051 3052 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3053 /* Wait for wg_get_stable_session to drain. */ 3054 pserialize_perform(wgp->wgp_psz); 3055 3056 /* Transition ESTABLISHED->DESTROYING. */ 3057 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 3058 3059 /* We can't destroy the old session immediately */ 3060 wg_schedule_session_dtor_timer(wgp); 3061 } else { 3062 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3063 "state=%d", wgs_prev->wgs_state); 3064 wg_clear_states(wgs_prev); 3065 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3066 } 3067 } 3068 3069 static void 3070 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3071 { 3072 3073 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3074 3075 KASSERT(mutex_owned(wgp->wgp_lock)); 3076 3077 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3078 pserialize_perform(wgp->wgp_psz); 3079 mutex_exit(wgp->wgp_lock); 3080 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3081 wg_psref_class); 3082 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3083 wg_psref_class); 3084 mutex_enter(wgp->wgp_lock); 3085 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3086 } 3087 } 3088 3089 static void 3090 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3091 { 3092 struct wg_session *wgs; 3093 3094 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3095 3096 KASSERT(mutex_owned(wgp->wgp_lock)); 3097 3098 wgs = wgp->wgp_session_stable; 3099 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3100 return; 3101 3102 wg_send_keepalive_msg(wgp, wgs); 3103 } 3104 3105 static void 3106 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3107 { 3108 struct wg_session *wgs; 3109 3110 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3111 3112 KASSERT(mutex_owned(wgp->wgp_lock)); 3113 3114 wgs = wgp->wgp_session_unstable; 3115 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 3116 wg_put_session_index(wg, wgs); 3117 } 3118 } 3119 3120 static void 3121 wg_peer_work(struct work *wk, void *cookie) 3122 { 3123 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3124 struct wg_softc *wg = wgp->wgp_sc; 3125 unsigned int tasks; 3126 3127 mutex_enter(wgp->wgp_intr_lock); 3128 while ((tasks = wgp->wgp_tasks) != 0) { 3129 wgp->wgp_tasks = 0; 3130 mutex_exit(wgp->wgp_intr_lock); 3131 3132 mutex_enter(wgp->wgp_lock); 3133 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3134 wg_task_send_init_message(wg, wgp); 3135 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3136 wg_task_retry_handshake(wg, wgp); 3137 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3138 wg_task_establish_session(wg, wgp); 3139 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3140 wg_task_endpoint_changed(wg, wgp); 3141 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3142 wg_task_send_keepalive_message(wg, wgp); 3143 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3144 wg_task_destroy_prev_session(wg, wgp); 3145 mutex_exit(wgp->wgp_lock); 3146 3147 mutex_enter(wgp->wgp_intr_lock); 3148 } 3149 mutex_exit(wgp->wgp_intr_lock); 3150 } 3151 3152 static void 3153 wg_job(struct threadpool_job *job) 3154 { 3155 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3156 int bound, upcalls; 3157 3158 mutex_enter(wg->wg_intr_lock); 3159 while ((upcalls = wg->wg_upcalls) != 0) { 3160 wg->wg_upcalls = 0; 3161 mutex_exit(wg->wg_intr_lock); 3162 bound = curlwp_bind(); 3163 if (ISSET(upcalls, WG_UPCALL_INET)) 3164 wg_receive_packets(wg, AF_INET); 3165 if (ISSET(upcalls, WG_UPCALL_INET6)) 3166 wg_receive_packets(wg, AF_INET6); 3167 curlwp_bindx(bound); 3168 mutex_enter(wg->wg_intr_lock); 3169 } 3170 threadpool_job_done(job); 3171 mutex_exit(wg->wg_intr_lock); 3172 } 3173 3174 static int 3175 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3176 { 3177 int error; 3178 uint16_t old_port = wg->wg_listen_port; 3179 3180 if (port != 0 && old_port == port) 3181 return 0; 3182 3183 struct sockaddr_in _sin, *sin = &_sin; 3184 sin->sin_len = sizeof(*sin); 3185 sin->sin_family = AF_INET; 3186 sin->sin_addr.s_addr = INADDR_ANY; 3187 sin->sin_port = htons(port); 3188 3189 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3190 if (error != 0) 3191 return error; 3192 3193 #ifdef INET6 3194 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3195 sin6->sin6_len = sizeof(*sin6); 3196 sin6->sin6_family = AF_INET6; 3197 sin6->sin6_addr = in6addr_any; 3198 sin6->sin6_port = htons(port); 3199 3200 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3201 if (error != 0) 3202 return error; 3203 #endif 3204 3205 wg->wg_listen_port = port; 3206 3207 return 0; 3208 } 3209 3210 static void 3211 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3212 { 3213 struct wg_softc *wg = cookie; 3214 int reason; 3215 3216 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3217 WG_UPCALL_INET : 3218 WG_UPCALL_INET6; 3219 3220 mutex_enter(wg->wg_intr_lock); 3221 wg->wg_upcalls |= reason; 3222 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3223 mutex_exit(wg->wg_intr_lock); 3224 } 3225 3226 static int 3227 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3228 struct sockaddr *src, void *arg) 3229 { 3230 struct wg_softc *wg = arg; 3231 struct wg_msg wgm; 3232 struct mbuf *m = *mp; 3233 3234 WG_TRACE("enter"); 3235 3236 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3237 KASSERT(offset <= m_length(m)); 3238 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3239 /* drop on the floor */ 3240 m_freem(m); 3241 return -1; 3242 } 3243 3244 /* 3245 * Copy the message header (32-bit message type) out -- we'll 3246 * worry about contiguity and alignment later. 3247 */ 3248 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3249 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3250 3251 /* 3252 * Handle DATA packets promptly as they arrive. Other packets 3253 * may require expensive public-key crypto and are not as 3254 * sensitive to latency, so defer them to the worker thread. 3255 */ 3256 switch (le32toh(wgm.wgm_type)) { 3257 case WG_MSG_TYPE_DATA: 3258 /* handle immediately */ 3259 m_adj(m, offset); 3260 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3261 m = m_pullup(m, sizeof(struct wg_msg_data)); 3262 if (m == NULL) 3263 return -1; 3264 } 3265 wg_handle_msg_data(wg, m, src); 3266 *mp = NULL; 3267 return 1; 3268 case WG_MSG_TYPE_INIT: 3269 case WG_MSG_TYPE_RESP: 3270 case WG_MSG_TYPE_COOKIE: 3271 /* pass through to so_receive in wg_receive_packets */ 3272 return 0; 3273 default: 3274 /* drop on the floor */ 3275 m_freem(m); 3276 return -1; 3277 } 3278 } 3279 3280 static int 3281 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3282 { 3283 int error; 3284 struct socket *so; 3285 3286 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3287 if (error != 0) 3288 return error; 3289 3290 solock(so); 3291 so->so_upcallarg = wg; 3292 so->so_upcall = wg_so_upcall; 3293 so->so_rcv.sb_flags |= SB_UPCALL; 3294 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3295 sounlock(so); 3296 3297 *sop = so; 3298 3299 return 0; 3300 } 3301 3302 static bool 3303 wg_session_hit_limits(struct wg_session *wgs) 3304 { 3305 3306 /* 3307 * [W] 6.2: Transport Message Limits 3308 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3309 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3310 * comes first, WireGuard will refuse to send any more transport data 3311 * messages using the current secure session, ..." 3312 */ 3313 KASSERT(wgs->wgs_time_established != 0); 3314 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3315 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3316 return true; 3317 } else if (wg_session_get_send_counter(wgs) > 3318 wg_reject_after_messages) { 3319 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3320 return true; 3321 } 3322 3323 return false; 3324 } 3325 3326 static void 3327 wgintr(void *cookie) 3328 { 3329 struct wg_peer *wgp; 3330 struct wg_session *wgs; 3331 struct mbuf *m; 3332 struct psref psref; 3333 3334 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3335 wgp = M_GETCTX(m, struct wg_peer *); 3336 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3337 WG_TRACE("no stable session"); 3338 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3339 goto next0; 3340 } 3341 if (__predict_false(wg_session_hit_limits(wgs))) { 3342 WG_TRACE("stable session hit limits"); 3343 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3344 goto next1; 3345 } 3346 wg_send_data_msg(wgp, wgs, m); 3347 m = NULL; /* consumed */ 3348 next1: wg_put_session(wgs, &psref); 3349 next0: if (m) 3350 m_freem(m); 3351 /* XXX Yield to avoid userland starvation? */ 3352 } 3353 } 3354 3355 static void 3356 wg_rekey_timer(void *arg) 3357 { 3358 struct wg_peer *wgp = arg; 3359 3360 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3361 } 3362 3363 static void 3364 wg_purge_pending_packets(struct wg_peer *wgp) 3365 { 3366 struct mbuf *m; 3367 3368 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) 3369 m_freem(m); 3370 pktq_barrier(wg_pktq); 3371 } 3372 3373 static void 3374 wg_handshake_timeout_timer(void *arg) 3375 { 3376 struct wg_peer *wgp = arg; 3377 3378 WG_TRACE("enter"); 3379 3380 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3381 } 3382 3383 static struct wg_peer * 3384 wg_alloc_peer(struct wg_softc *wg) 3385 { 3386 struct wg_peer *wgp; 3387 3388 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3389 3390 wgp->wgp_sc = wg; 3391 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3392 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3393 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3394 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3395 wg_handshake_timeout_timer, wgp); 3396 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3397 callout_setfunc(&wgp->wgp_session_dtor_timer, 3398 wg_session_dtor_timer, wgp); 3399 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3400 wgp->wgp_endpoint_changing = false; 3401 wgp->wgp_endpoint_available = false; 3402 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3403 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3404 wgp->wgp_psz = pserialize_create(); 3405 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3406 3407 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3408 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3409 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3410 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3411 3412 struct wg_session *wgs; 3413 wgp->wgp_session_stable = 3414 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3415 wgp->wgp_session_unstable = 3416 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3417 wgs = wgp->wgp_session_stable; 3418 wgs->wgs_peer = wgp; 3419 wgs->wgs_state = WGS_STATE_UNKNOWN; 3420 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3421 #ifndef __HAVE_ATOMIC64_LOADSTORE 3422 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3423 #endif 3424 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3425 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3426 3427 wgs = wgp->wgp_session_unstable; 3428 wgs->wgs_peer = wgp; 3429 wgs->wgs_state = WGS_STATE_UNKNOWN; 3430 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3431 #ifndef __HAVE_ATOMIC64_LOADSTORE 3432 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3433 #endif 3434 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3435 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3436 3437 return wgp; 3438 } 3439 3440 static void 3441 wg_destroy_peer(struct wg_peer *wgp) 3442 { 3443 struct wg_session *wgs; 3444 struct wg_softc *wg = wgp->wgp_sc; 3445 3446 /* Prevent new packets from this peer on any source address. */ 3447 rw_enter(wg->wg_rwlock, RW_WRITER); 3448 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3449 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3450 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3451 struct radix_node *rn; 3452 3453 KASSERT(rnh != NULL); 3454 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3455 &wga->wga_sa_mask, rnh); 3456 if (rn == NULL) { 3457 char addrstr[128]; 3458 sockaddr_format(&wga->wga_sa_addr, addrstr, 3459 sizeof(addrstr)); 3460 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3461 if_name(&wg->wg_if), addrstr); 3462 } 3463 } 3464 rw_exit(wg->wg_rwlock); 3465 3466 /* Purge pending packets. */ 3467 wg_purge_pending_packets(wgp); 3468 3469 /* Halt all packet processing and timeouts. */ 3470 callout_halt(&wgp->wgp_rekey_timer, NULL); 3471 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3472 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3473 3474 /* Wait for any queued work to complete. */ 3475 workqueue_wait(wg_wq, &wgp->wgp_work); 3476 3477 wgs = wgp->wgp_session_unstable; 3478 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3479 mutex_enter(wgp->wgp_lock); 3480 wg_destroy_session(wg, wgs); 3481 mutex_exit(wgp->wgp_lock); 3482 } 3483 mutex_destroy(&wgs->wgs_recvwin->lock); 3484 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3485 #ifndef __HAVE_ATOMIC64_LOADSTORE 3486 mutex_destroy(&wgs->wgs_send_counter_lock); 3487 #endif 3488 kmem_free(wgs, sizeof(*wgs)); 3489 3490 wgs = wgp->wgp_session_stable; 3491 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3492 mutex_enter(wgp->wgp_lock); 3493 wg_destroy_session(wg, wgs); 3494 mutex_exit(wgp->wgp_lock); 3495 } 3496 mutex_destroy(&wgs->wgs_recvwin->lock); 3497 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3498 #ifndef __HAVE_ATOMIC64_LOADSTORE 3499 mutex_destroy(&wgs->wgs_send_counter_lock); 3500 #endif 3501 kmem_free(wgs, sizeof(*wgs)); 3502 3503 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3504 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3505 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3506 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3507 3508 pserialize_destroy(wgp->wgp_psz); 3509 mutex_obj_free(wgp->wgp_intr_lock); 3510 mutex_obj_free(wgp->wgp_lock); 3511 3512 kmem_free(wgp, sizeof(*wgp)); 3513 } 3514 3515 static void 3516 wg_destroy_all_peers(struct wg_softc *wg) 3517 { 3518 struct wg_peer *wgp, *wgp0 __diagused; 3519 void *garbage_byname, *garbage_bypubkey; 3520 3521 restart: 3522 garbage_byname = garbage_bypubkey = NULL; 3523 mutex_enter(wg->wg_lock); 3524 WG_PEER_WRITER_FOREACH(wgp, wg) { 3525 if (wgp->wgp_name[0]) { 3526 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3527 strlen(wgp->wgp_name)); 3528 KASSERT(wgp0 == wgp); 3529 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3530 } 3531 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3532 sizeof(wgp->wgp_pubkey)); 3533 KASSERT(wgp0 == wgp); 3534 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3535 WG_PEER_WRITER_REMOVE(wgp); 3536 wg->wg_npeers--; 3537 mutex_enter(wgp->wgp_lock); 3538 pserialize_perform(wgp->wgp_psz); 3539 mutex_exit(wgp->wgp_lock); 3540 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3541 break; 3542 } 3543 mutex_exit(wg->wg_lock); 3544 3545 if (wgp == NULL) 3546 return; 3547 3548 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3549 3550 wg_destroy_peer(wgp); 3551 thmap_gc(wg->wg_peers_byname, garbage_byname); 3552 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3553 3554 goto restart; 3555 } 3556 3557 static int 3558 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3559 { 3560 struct wg_peer *wgp, *wgp0 __diagused; 3561 void *garbage_byname, *garbage_bypubkey; 3562 3563 mutex_enter(wg->wg_lock); 3564 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3565 if (wgp != NULL) { 3566 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3567 sizeof(wgp->wgp_pubkey)); 3568 KASSERT(wgp0 == wgp); 3569 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3570 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3571 WG_PEER_WRITER_REMOVE(wgp); 3572 wg->wg_npeers--; 3573 if (wg->wg_npeers == 0) 3574 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3575 mutex_enter(wgp->wgp_lock); 3576 pserialize_perform(wgp->wgp_psz); 3577 mutex_exit(wgp->wgp_lock); 3578 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3579 } 3580 mutex_exit(wg->wg_lock); 3581 3582 if (wgp == NULL) 3583 return ENOENT; 3584 3585 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3586 3587 wg_destroy_peer(wgp); 3588 thmap_gc(wg->wg_peers_byname, garbage_byname); 3589 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3590 3591 return 0; 3592 } 3593 3594 static int 3595 wg_if_attach(struct wg_softc *wg) 3596 { 3597 3598 wg->wg_if.if_addrlen = 0; 3599 wg->wg_if.if_mtu = WG_MTU; 3600 wg->wg_if.if_flags = IFF_MULTICAST; 3601 wg->wg_if.if_extflags = IFEF_MPSAFE; 3602 wg->wg_if.if_ioctl = wg_ioctl; 3603 wg->wg_if.if_output = wg_output; 3604 wg->wg_if.if_init = wg_init; 3605 #ifdef ALTQ 3606 wg->wg_if.if_start = wg_start; 3607 #endif 3608 wg->wg_if.if_stop = wg_stop; 3609 wg->wg_if.if_type = IFT_OTHER; 3610 wg->wg_if.if_dlt = DLT_NULL; 3611 wg->wg_if.if_softc = wg; 3612 #ifdef ALTQ 3613 IFQ_SET_READY(&wg->wg_if.if_snd); 3614 #endif 3615 if_initialize(&wg->wg_if); 3616 3617 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3618 if_alloc_sadl(&wg->wg_if); 3619 if_register(&wg->wg_if); 3620 3621 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3622 3623 return 0; 3624 } 3625 3626 static void 3627 wg_if_detach(struct wg_softc *wg) 3628 { 3629 struct ifnet *ifp = &wg->wg_if; 3630 3631 bpf_detach(ifp); 3632 if_detach(ifp); 3633 } 3634 3635 static int 3636 wg_clone_create(struct if_clone *ifc, int unit) 3637 { 3638 struct wg_softc *wg; 3639 int error; 3640 3641 wg_guarantee_initialized(); 3642 3643 error = wg_count_inc(); 3644 if (error) 3645 return error; 3646 3647 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3648 3649 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3650 3651 PSLIST_INIT(&wg->wg_peers); 3652 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3653 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3654 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3655 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3656 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3657 wg->wg_rwlock = rw_obj_alloc(); 3658 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3659 "%s", if_name(&wg->wg_if)); 3660 wg->wg_ops = &wg_ops_rumpkernel; 3661 3662 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3663 if (error) 3664 goto fail0; 3665 3666 #ifdef INET 3667 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3668 if (error) 3669 goto fail1; 3670 rn_inithead((void **)&wg->wg_rtable_ipv4, 3671 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3672 #endif 3673 #ifdef INET6 3674 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3675 if (error) 3676 goto fail2; 3677 rn_inithead((void **)&wg->wg_rtable_ipv6, 3678 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3679 #endif 3680 3681 error = wg_if_attach(wg); 3682 if (error) 3683 goto fail3; 3684 3685 return 0; 3686 3687 fail4: __unused 3688 wg_if_detach(wg); 3689 fail3: wg_destroy_all_peers(wg); 3690 #ifdef INET6 3691 solock(wg->wg_so6); 3692 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3693 sounlock(wg->wg_so6); 3694 #endif 3695 #ifdef INET 3696 solock(wg->wg_so4); 3697 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3698 sounlock(wg->wg_so4); 3699 #endif 3700 mutex_enter(wg->wg_intr_lock); 3701 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3702 mutex_exit(wg->wg_intr_lock); 3703 #ifdef INET6 3704 if (wg->wg_rtable_ipv6 != NULL) 3705 free(wg->wg_rtable_ipv6, M_RTABLE); 3706 soclose(wg->wg_so6); 3707 fail2: 3708 #endif 3709 #ifdef INET 3710 if (wg->wg_rtable_ipv4 != NULL) 3711 free(wg->wg_rtable_ipv4, M_RTABLE); 3712 soclose(wg->wg_so4); 3713 fail1: 3714 #endif 3715 threadpool_put(wg->wg_threadpool, PRI_NONE); 3716 fail0: threadpool_job_destroy(&wg->wg_job); 3717 rw_obj_free(wg->wg_rwlock); 3718 mutex_obj_free(wg->wg_intr_lock); 3719 mutex_obj_free(wg->wg_lock); 3720 thmap_destroy(wg->wg_sessions_byindex); 3721 thmap_destroy(wg->wg_peers_byname); 3722 thmap_destroy(wg->wg_peers_bypubkey); 3723 PSLIST_DESTROY(&wg->wg_peers); 3724 kmem_free(wg, sizeof(*wg)); 3725 wg_count_dec(); 3726 return error; 3727 } 3728 3729 static int 3730 wg_clone_destroy(struct ifnet *ifp) 3731 { 3732 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3733 3734 #ifdef WG_RUMPKERNEL 3735 if (wg_user_mode(wg)) { 3736 rumpuser_wg_destroy(wg->wg_user); 3737 wg->wg_user = NULL; 3738 } 3739 #endif 3740 3741 wg_if_detach(wg); 3742 wg_destroy_all_peers(wg); 3743 #ifdef INET6 3744 solock(wg->wg_so6); 3745 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3746 sounlock(wg->wg_so6); 3747 #endif 3748 #ifdef INET 3749 solock(wg->wg_so4); 3750 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3751 sounlock(wg->wg_so4); 3752 #endif 3753 mutex_enter(wg->wg_intr_lock); 3754 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3755 mutex_exit(wg->wg_intr_lock); 3756 #ifdef INET6 3757 if (wg->wg_rtable_ipv6 != NULL) 3758 free(wg->wg_rtable_ipv6, M_RTABLE); 3759 soclose(wg->wg_so6); 3760 #endif 3761 #ifdef INET 3762 if (wg->wg_rtable_ipv4 != NULL) 3763 free(wg->wg_rtable_ipv4, M_RTABLE); 3764 soclose(wg->wg_so4); 3765 #endif 3766 threadpool_put(wg->wg_threadpool, PRI_NONE); 3767 threadpool_job_destroy(&wg->wg_job); 3768 rw_obj_free(wg->wg_rwlock); 3769 mutex_obj_free(wg->wg_intr_lock); 3770 mutex_obj_free(wg->wg_lock); 3771 thmap_destroy(wg->wg_sessions_byindex); 3772 thmap_destroy(wg->wg_peers_byname); 3773 thmap_destroy(wg->wg_peers_bypubkey); 3774 PSLIST_DESTROY(&wg->wg_peers); 3775 kmem_free(wg, sizeof(*wg)); 3776 wg_count_dec(); 3777 3778 return 0; 3779 } 3780 3781 static struct wg_peer * 3782 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3783 struct psref *psref) 3784 { 3785 struct radix_node_head *rnh; 3786 struct radix_node *rn; 3787 struct wg_peer *wgp = NULL; 3788 struct wg_allowedip *wga; 3789 3790 #ifdef WG_DEBUG_LOG 3791 char addrstr[128]; 3792 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3793 WG_DLOG("sa=%s\n", addrstr); 3794 #endif 3795 3796 rw_enter(wg->wg_rwlock, RW_READER); 3797 3798 rnh = wg_rnh(wg, sa->sa_family); 3799 if (rnh == NULL) 3800 goto out; 3801 3802 rn = rnh->rnh_matchaddr(sa, rnh); 3803 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3804 goto out; 3805 3806 WG_TRACE("success"); 3807 3808 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3809 wgp = wga->wga_peer; 3810 wg_get_peer(wgp, psref); 3811 3812 out: 3813 rw_exit(wg->wg_rwlock); 3814 return wgp; 3815 } 3816 3817 static void 3818 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3819 struct wg_session *wgs, struct wg_msg_data *wgmd) 3820 { 3821 3822 memset(wgmd, 0, sizeof(*wgmd)); 3823 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3824 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3825 /* [W] 5.4.6: msg.counter := Nm^send */ 3826 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3827 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3828 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3829 } 3830 3831 static int 3832 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3833 const struct rtentry *rt) 3834 { 3835 struct wg_softc *wg = ifp->if_softc; 3836 struct wg_peer *wgp = NULL; 3837 struct wg_session *wgs = NULL; 3838 struct psref wgp_psref, wgs_psref; 3839 int bound; 3840 int error; 3841 3842 bound = curlwp_bind(); 3843 3844 /* TODO make the nest limit configurable via sysctl */ 3845 error = if_tunnel_check_nesting(ifp, m, 1); 3846 if (error) { 3847 WGLOG(LOG_ERR, 3848 "%s: tunneling loop detected and packet dropped\n", 3849 if_name(&wg->wg_if)); 3850 goto out0; 3851 } 3852 3853 #ifdef ALTQ 3854 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 3855 & ALTQF_ENABLED; 3856 if (altq) 3857 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3858 #endif 3859 3860 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3861 3862 m->m_flags &= ~(M_BCAST|M_MCAST); 3863 3864 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3865 if (wgp == NULL) { 3866 WG_TRACE("peer not found"); 3867 error = EHOSTUNREACH; 3868 goto out0; 3869 } 3870 3871 /* Clear checksum-offload flags. */ 3872 m->m_pkthdr.csum_flags = 0; 3873 m->m_pkthdr.csum_data = 0; 3874 3875 /* Check whether there's an established session. */ 3876 wgs = wg_get_stable_session(wgp, &wgs_psref); 3877 if (wgs == NULL) { 3878 /* 3879 * No established session. If we're the first to try 3880 * sending data, schedule a handshake and queue the 3881 * packet for when the handshake is done; otherwise 3882 * just drop the packet and let the ongoing handshake 3883 * attempt continue. We could queue more data packets 3884 * but it's not clear that's worthwhile. 3885 */ 3886 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 3887 m = NULL; /* consume */ 3888 WG_TRACE("queued first packet; init handshake"); 3889 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3890 } else { 3891 WG_TRACE("first packet already queued, dropping"); 3892 } 3893 goto out1; 3894 } 3895 3896 /* There's an established session. Toss it in the queue. */ 3897 #ifdef ALTQ 3898 if (altq) { 3899 mutex_enter(ifp->if_snd.ifq_lock); 3900 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 3901 M_SETCTX(m, wgp); 3902 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 3903 m = NULL; /* consume */ 3904 } 3905 mutex_exit(ifp->if_snd.ifq_lock); 3906 if (m == NULL) { 3907 wg_start(ifp); 3908 goto out2; 3909 } 3910 } 3911 #endif 3912 kpreempt_disable(); 3913 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3914 M_SETCTX(m, wgp); 3915 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3916 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3917 if_name(&wg->wg_if)); 3918 error = ENOBUFS; 3919 goto out3; 3920 } 3921 m = NULL; /* consumed */ 3922 error = 0; 3923 out3: kpreempt_enable(); 3924 3925 #ifdef ALTQ 3926 out2: 3927 #endif 3928 wg_put_session(wgs, &wgs_psref); 3929 out1: wg_put_peer(wgp, &wgp_psref); 3930 out0: if (m) 3931 m_freem(m); 3932 curlwp_bindx(bound); 3933 return error; 3934 } 3935 3936 static int 3937 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3938 { 3939 struct psref psref; 3940 struct wg_sockaddr *wgsa; 3941 int error; 3942 struct socket *so; 3943 3944 wgsa = wg_get_endpoint_sa(wgp, &psref); 3945 so = wg_get_so_by_peer(wgp, wgsa); 3946 solock(so); 3947 if (wgsatosa(wgsa)->sa_family == AF_INET) { 3948 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 3949 } else { 3950 #ifdef INET6 3951 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 3952 NULL, curlwp); 3953 #else 3954 m_freem(m); 3955 error = EPFNOSUPPORT; 3956 #endif 3957 } 3958 sounlock(so); 3959 wg_put_sa(wgp, wgsa, &psref); 3960 3961 return error; 3962 } 3963 3964 /* Inspired by pppoe_get_mbuf */ 3965 static struct mbuf * 3966 wg_get_mbuf(size_t leading_len, size_t len) 3967 { 3968 struct mbuf *m; 3969 3970 KASSERT(leading_len <= MCLBYTES); 3971 KASSERT(len <= MCLBYTES - leading_len); 3972 3973 m = m_gethdr(M_DONTWAIT, MT_DATA); 3974 if (m == NULL) 3975 return NULL; 3976 if (len + leading_len > MHLEN) { 3977 m_clget(m, M_DONTWAIT); 3978 if ((m->m_flags & M_EXT) == 0) { 3979 m_free(m); 3980 return NULL; 3981 } 3982 } 3983 m->m_data += leading_len; 3984 m->m_pkthdr.len = m->m_len = len; 3985 3986 return m; 3987 } 3988 3989 static int 3990 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 3991 struct mbuf *m) 3992 { 3993 struct wg_softc *wg = wgp->wgp_sc; 3994 int error; 3995 size_t inner_len, padded_len, encrypted_len; 3996 char *padded_buf = NULL; 3997 size_t mlen; 3998 struct wg_msg_data *wgmd; 3999 bool free_padded_buf = false; 4000 struct mbuf *n; 4001 size_t leading_len = max_hdr + sizeof(struct udphdr); 4002 4003 mlen = m_length(m); 4004 inner_len = mlen; 4005 padded_len = roundup(mlen, 16); 4006 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4007 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 4008 inner_len, padded_len, encrypted_len); 4009 if (mlen != 0) { 4010 bool success; 4011 success = m_ensure_contig(&m, padded_len); 4012 if (success) { 4013 padded_buf = mtod(m, char *); 4014 } else { 4015 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4016 if (padded_buf == NULL) { 4017 error = ENOBUFS; 4018 goto end; 4019 } 4020 free_padded_buf = true; 4021 m_copydata(m, 0, mlen, padded_buf); 4022 } 4023 memset(padded_buf + mlen, 0, padded_len - inner_len); 4024 } 4025 4026 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4027 if (n == NULL) { 4028 error = ENOBUFS; 4029 goto end; 4030 } 4031 KASSERT(n->m_len >= sizeof(*wgmd)); 4032 wgmd = mtod(n, struct wg_msg_data *); 4033 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4034 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4035 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4036 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4037 padded_buf, padded_len, 4038 NULL, 0); 4039 4040 error = wg->wg_ops->send_data_msg(wgp, n); 4041 if (error == 0) { 4042 struct ifnet *ifp = &wg->wg_if; 4043 if_statadd(ifp, if_obytes, mlen); 4044 if_statinc(ifp, if_opackets); 4045 if (wgs->wgs_is_initiator && 4046 wgs->wgs_time_last_data_sent == 0) { 4047 /* 4048 * [W] 6.2 Transport Message Limits 4049 * "if a peer is the initiator of a current secure 4050 * session, WireGuard will send a handshake initiation 4051 * message to begin a new secure session if, after 4052 * transmitting a transport data message, the current 4053 * secure session is REKEY-AFTER-TIME seconds old," 4054 */ 4055 wg_schedule_rekey_timer(wgp); 4056 } 4057 wgs->wgs_time_last_data_sent = time_uptime; 4058 if (wg_session_get_send_counter(wgs) >= 4059 wg_rekey_after_messages) { 4060 /* 4061 * [W] 6.2 Transport Message Limits 4062 * "WireGuard will try to create a new session, by 4063 * sending a handshake initiation message (section 4064 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4065 * transport data messages..." 4066 */ 4067 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4068 } 4069 } 4070 end: 4071 m_freem(m); 4072 if (free_padded_buf) 4073 kmem_intr_free(padded_buf, padded_len); 4074 return error; 4075 } 4076 4077 static void 4078 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4079 { 4080 pktqueue_t *pktq; 4081 size_t pktlen; 4082 4083 KASSERT(af == AF_INET || af == AF_INET6); 4084 4085 WG_TRACE(""); 4086 4087 m_set_rcvif(m, ifp); 4088 pktlen = m->m_pkthdr.len; 4089 4090 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4091 4092 switch (af) { 4093 case AF_INET: 4094 pktq = ip_pktq; 4095 break; 4096 #ifdef INET6 4097 case AF_INET6: 4098 pktq = ip6_pktq; 4099 break; 4100 #endif 4101 default: 4102 panic("invalid af=%d", af); 4103 } 4104 4105 kpreempt_disable(); 4106 const u_int h = curcpu()->ci_index; 4107 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4108 if_statadd(ifp, if_ibytes, pktlen); 4109 if_statinc(ifp, if_ipackets); 4110 } else { 4111 m_freem(m); 4112 } 4113 kpreempt_enable(); 4114 } 4115 4116 static void 4117 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4118 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4119 { 4120 4121 crypto_scalarmult_base(pubkey, privkey); 4122 } 4123 4124 static int 4125 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4126 { 4127 struct radix_node_head *rnh; 4128 struct radix_node *rn; 4129 int error = 0; 4130 4131 rw_enter(wg->wg_rwlock, RW_WRITER); 4132 rnh = wg_rnh(wg, wga->wga_family); 4133 KASSERT(rnh != NULL); 4134 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4135 wga->wga_nodes); 4136 rw_exit(wg->wg_rwlock); 4137 4138 if (rn == NULL) 4139 error = EEXIST; 4140 4141 return error; 4142 } 4143 4144 static int 4145 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4146 struct wg_peer **wgpp) 4147 { 4148 int error = 0; 4149 const void *pubkey; 4150 size_t pubkey_len; 4151 const void *psk; 4152 size_t psk_len; 4153 const char *name = NULL; 4154 4155 if (prop_dictionary_get_string(peer, "name", &name)) { 4156 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4157 error = EINVAL; 4158 goto out; 4159 } 4160 } 4161 4162 if (!prop_dictionary_get_data(peer, "public_key", 4163 &pubkey, &pubkey_len)) { 4164 error = EINVAL; 4165 goto out; 4166 } 4167 #ifdef WG_DEBUG_DUMP 4168 { 4169 char *hex = gethexdump(pubkey, pubkey_len); 4170 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n%s\n", 4171 pubkey, pubkey_len, hex); 4172 puthexdump(hex, pubkey, pubkey_len); 4173 } 4174 #endif 4175 4176 struct wg_peer *wgp = wg_alloc_peer(wg); 4177 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4178 if (name != NULL) 4179 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4180 4181 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4182 if (psk_len != sizeof(wgp->wgp_psk)) { 4183 error = EINVAL; 4184 goto out; 4185 } 4186 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4187 } 4188 4189 const void *addr; 4190 size_t addr_len; 4191 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4192 4193 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4194 goto skip_endpoint; 4195 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4196 addr_len > sizeof(*wgsatoss(wgsa))) { 4197 error = EINVAL; 4198 goto out; 4199 } 4200 memcpy(wgsatoss(wgsa), addr, addr_len); 4201 switch (wgsa_family(wgsa)) { 4202 case AF_INET: 4203 #ifdef INET6 4204 case AF_INET6: 4205 #endif 4206 break; 4207 default: 4208 error = EPFNOSUPPORT; 4209 goto out; 4210 } 4211 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4212 error = EINVAL; 4213 goto out; 4214 } 4215 { 4216 char addrstr[128]; 4217 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4218 WG_DLOG("addr=%s\n", addrstr); 4219 } 4220 wgp->wgp_endpoint_available = true; 4221 4222 prop_array_t allowedips; 4223 skip_endpoint: 4224 allowedips = prop_dictionary_get(peer, "allowedips"); 4225 if (allowedips == NULL) 4226 goto skip; 4227 4228 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4229 prop_dictionary_t prop_allowedip; 4230 int j = 0; 4231 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4232 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4233 4234 if (!prop_dictionary_get_int(prop_allowedip, "family", 4235 &wga->wga_family)) 4236 continue; 4237 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4238 &addr, &addr_len)) 4239 continue; 4240 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4241 &wga->wga_cidr)) 4242 continue; 4243 4244 switch (wga->wga_family) { 4245 case AF_INET: { 4246 struct sockaddr_in sin; 4247 char addrstr[128]; 4248 struct in_addr mask; 4249 struct sockaddr_in sin_mask; 4250 4251 if (addr_len != sizeof(struct in_addr)) 4252 return EINVAL; 4253 memcpy(&wga->wga_addr4, addr, addr_len); 4254 4255 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4256 0); 4257 sockaddr_copy(&wga->wga_sa_addr, 4258 sizeof(sin), sintosa(&sin)); 4259 4260 sockaddr_format(sintosa(&sin), 4261 addrstr, sizeof(addrstr)); 4262 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4263 4264 in_len2mask(&mask, wga->wga_cidr); 4265 sockaddr_in_init(&sin_mask, &mask, 0); 4266 sockaddr_copy(&wga->wga_sa_mask, 4267 sizeof(sin_mask), sintosa(&sin_mask)); 4268 4269 break; 4270 } 4271 #ifdef INET6 4272 case AF_INET6: { 4273 struct sockaddr_in6 sin6; 4274 char addrstr[128]; 4275 struct in6_addr mask; 4276 struct sockaddr_in6 sin6_mask; 4277 4278 if (addr_len != sizeof(struct in6_addr)) 4279 return EINVAL; 4280 memcpy(&wga->wga_addr6, addr, addr_len); 4281 4282 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4283 0, 0, 0); 4284 sockaddr_copy(&wga->wga_sa_addr, 4285 sizeof(sin6), sin6tosa(&sin6)); 4286 4287 sockaddr_format(sin6tosa(&sin6), 4288 addrstr, sizeof(addrstr)); 4289 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4290 4291 in6_prefixlen2mask(&mask, wga->wga_cidr); 4292 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4293 sockaddr_copy(&wga->wga_sa_mask, 4294 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4295 4296 break; 4297 } 4298 #endif 4299 default: 4300 error = EINVAL; 4301 goto out; 4302 } 4303 wga->wga_peer = wgp; 4304 4305 error = wg_rtable_add_route(wg, wga); 4306 if (error != 0) 4307 goto out; 4308 4309 j++; 4310 } 4311 wgp->wgp_n_allowedips = j; 4312 skip: 4313 *wgpp = wgp; 4314 out: 4315 return error; 4316 } 4317 4318 static int 4319 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4320 { 4321 int error; 4322 char *buf; 4323 4324 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4325 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4326 return E2BIG; 4327 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4328 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4329 if (error != 0) 4330 return error; 4331 buf[ifd->ifd_len] = '\0'; 4332 #ifdef WG_DEBUG_DUMP 4333 log(LOG_DEBUG, "%.*s\n", 4334 (int)MIN(INT_MAX, ifd->ifd_len), 4335 (const char *)buf); 4336 #endif 4337 *_buf = buf; 4338 return 0; 4339 } 4340 4341 static int 4342 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4343 { 4344 int error; 4345 prop_dictionary_t prop_dict; 4346 char *buf = NULL; 4347 const void *privkey; 4348 size_t privkey_len; 4349 4350 error = wg_alloc_prop_buf(&buf, ifd); 4351 if (error != 0) 4352 return error; 4353 error = EINVAL; 4354 prop_dict = prop_dictionary_internalize(buf); 4355 if (prop_dict == NULL) 4356 goto out; 4357 if (!prop_dictionary_get_data(prop_dict, "private_key", 4358 &privkey, &privkey_len)) 4359 goto out; 4360 #ifdef WG_DEBUG_DUMP 4361 { 4362 char *hex = gethexdump(privkey, privkey_len); 4363 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n%s\n", 4364 privkey, privkey_len, hex); 4365 puthexdump(hex, privkey, privkey_len); 4366 } 4367 #endif 4368 if (privkey_len != WG_STATIC_KEY_LEN) 4369 goto out; 4370 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4371 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4372 error = 0; 4373 4374 out: 4375 kmem_free(buf, ifd->ifd_len + 1); 4376 return error; 4377 } 4378 4379 static int 4380 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4381 { 4382 int error; 4383 prop_dictionary_t prop_dict; 4384 char *buf = NULL; 4385 uint16_t port; 4386 4387 error = wg_alloc_prop_buf(&buf, ifd); 4388 if (error != 0) 4389 return error; 4390 error = EINVAL; 4391 prop_dict = prop_dictionary_internalize(buf); 4392 if (prop_dict == NULL) 4393 goto out; 4394 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4395 goto out; 4396 4397 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4398 4399 out: 4400 kmem_free(buf, ifd->ifd_len + 1); 4401 return error; 4402 } 4403 4404 static int 4405 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4406 { 4407 int error; 4408 prop_dictionary_t prop_dict; 4409 char *buf = NULL; 4410 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4411 4412 error = wg_alloc_prop_buf(&buf, ifd); 4413 if (error != 0) 4414 return error; 4415 error = EINVAL; 4416 prop_dict = prop_dictionary_internalize(buf); 4417 if (prop_dict == NULL) 4418 goto out; 4419 4420 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4421 if (error != 0) 4422 goto out; 4423 4424 mutex_enter(wg->wg_lock); 4425 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4426 sizeof(wgp->wgp_pubkey)) != NULL || 4427 (wgp->wgp_name[0] && 4428 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4429 strlen(wgp->wgp_name)) != NULL)) { 4430 mutex_exit(wg->wg_lock); 4431 wg_destroy_peer(wgp); 4432 error = EEXIST; 4433 goto out; 4434 } 4435 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4436 sizeof(wgp->wgp_pubkey), wgp); 4437 KASSERT(wgp0 == wgp); 4438 if (wgp->wgp_name[0]) { 4439 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4440 strlen(wgp->wgp_name), wgp); 4441 KASSERT(wgp0 == wgp); 4442 } 4443 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4444 wg->wg_npeers++; 4445 mutex_exit(wg->wg_lock); 4446 4447 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4448 4449 out: 4450 kmem_free(buf, ifd->ifd_len + 1); 4451 return error; 4452 } 4453 4454 static int 4455 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4456 { 4457 int error; 4458 prop_dictionary_t prop_dict; 4459 char *buf = NULL; 4460 const char *name; 4461 4462 error = wg_alloc_prop_buf(&buf, ifd); 4463 if (error != 0) 4464 return error; 4465 error = EINVAL; 4466 prop_dict = prop_dictionary_internalize(buf); 4467 if (prop_dict == NULL) 4468 goto out; 4469 4470 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4471 goto out; 4472 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4473 goto out; 4474 4475 error = wg_destroy_peer_name(wg, name); 4476 out: 4477 kmem_free(buf, ifd->ifd_len + 1); 4478 return error; 4479 } 4480 4481 static bool 4482 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4483 { 4484 int au = cmd == SIOCGDRVSPEC ? 4485 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4486 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4487 return kauth_authorize_network(kauth_cred_get(), 4488 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4489 (void *)cmd, NULL) == 0; 4490 } 4491 4492 static int 4493 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4494 { 4495 int error = ENOMEM; 4496 prop_dictionary_t prop_dict; 4497 prop_array_t peers = NULL; 4498 char *buf; 4499 struct wg_peer *wgp; 4500 int s, i; 4501 4502 prop_dict = prop_dictionary_create(); 4503 if (prop_dict == NULL) 4504 goto error; 4505 4506 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4507 if (!prop_dictionary_set_data(prop_dict, "private_key", 4508 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4509 goto error; 4510 } 4511 4512 if (wg->wg_listen_port != 0) { 4513 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4514 wg->wg_listen_port)) 4515 goto error; 4516 } 4517 4518 if (wg->wg_npeers == 0) 4519 goto skip_peers; 4520 4521 peers = prop_array_create(); 4522 if (peers == NULL) 4523 goto error; 4524 4525 s = pserialize_read_enter(); 4526 i = 0; 4527 WG_PEER_READER_FOREACH(wgp, wg) { 4528 struct wg_sockaddr *wgsa; 4529 struct psref wgp_psref, wgsa_psref; 4530 prop_dictionary_t prop_peer; 4531 4532 wg_get_peer(wgp, &wgp_psref); 4533 pserialize_read_exit(s); 4534 4535 prop_peer = prop_dictionary_create(); 4536 if (prop_peer == NULL) 4537 goto next; 4538 4539 if (strlen(wgp->wgp_name) > 0) { 4540 if (!prop_dictionary_set_string(prop_peer, "name", 4541 wgp->wgp_name)) 4542 goto next; 4543 } 4544 4545 if (!prop_dictionary_set_data(prop_peer, "public_key", 4546 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4547 goto next; 4548 4549 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4550 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4551 sizeof(wgp->wgp_psk))) { 4552 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4553 if (!prop_dictionary_set_data(prop_peer, 4554 "preshared_key", 4555 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4556 goto next; 4557 } 4558 } 4559 4560 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4561 CTASSERT(AF_UNSPEC == 0); 4562 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4563 !prop_dictionary_set_data(prop_peer, "endpoint", 4564 wgsatoss(wgsa), 4565 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4566 wg_put_sa(wgp, wgsa, &wgsa_psref); 4567 goto next; 4568 } 4569 wg_put_sa(wgp, wgsa, &wgsa_psref); 4570 4571 const struct timespec *t = &wgp->wgp_last_handshake_time; 4572 4573 if (!prop_dictionary_set_uint64(prop_peer, 4574 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4575 goto next; 4576 if (!prop_dictionary_set_uint32(prop_peer, 4577 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4578 goto next; 4579 4580 if (wgp->wgp_n_allowedips == 0) 4581 goto skip_allowedips; 4582 4583 prop_array_t allowedips = prop_array_create(); 4584 if (allowedips == NULL) 4585 goto next; 4586 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4587 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4588 prop_dictionary_t prop_allowedip; 4589 4590 prop_allowedip = prop_dictionary_create(); 4591 if (prop_allowedip == NULL) 4592 break; 4593 4594 if (!prop_dictionary_set_int(prop_allowedip, "family", 4595 wga->wga_family)) 4596 goto _next; 4597 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4598 wga->wga_cidr)) 4599 goto _next; 4600 4601 switch (wga->wga_family) { 4602 case AF_INET: 4603 if (!prop_dictionary_set_data(prop_allowedip, 4604 "ip", &wga->wga_addr4, 4605 sizeof(wga->wga_addr4))) 4606 goto _next; 4607 break; 4608 #ifdef INET6 4609 case AF_INET6: 4610 if (!prop_dictionary_set_data(prop_allowedip, 4611 "ip", &wga->wga_addr6, 4612 sizeof(wga->wga_addr6))) 4613 goto _next; 4614 break; 4615 #endif 4616 default: 4617 break; 4618 } 4619 prop_array_set(allowedips, j, prop_allowedip); 4620 _next: 4621 prop_object_release(prop_allowedip); 4622 } 4623 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4624 prop_object_release(allowedips); 4625 4626 skip_allowedips: 4627 4628 prop_array_set(peers, i, prop_peer); 4629 next: 4630 if (prop_peer) 4631 prop_object_release(prop_peer); 4632 i++; 4633 4634 s = pserialize_read_enter(); 4635 wg_put_peer(wgp, &wgp_psref); 4636 } 4637 pserialize_read_exit(s); 4638 4639 prop_dictionary_set(prop_dict, "peers", peers); 4640 prop_object_release(peers); 4641 peers = NULL; 4642 4643 skip_peers: 4644 buf = prop_dictionary_externalize(prop_dict); 4645 if (buf == NULL) 4646 goto error; 4647 if (ifd->ifd_len < (strlen(buf) + 1)) { 4648 error = EINVAL; 4649 goto error; 4650 } 4651 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4652 4653 free(buf, 0); 4654 error: 4655 if (peers != NULL) 4656 prop_object_release(peers); 4657 if (prop_dict != NULL) 4658 prop_object_release(prop_dict); 4659 4660 return error; 4661 } 4662 4663 static int 4664 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4665 { 4666 struct wg_softc *wg = ifp->if_softc; 4667 struct ifreq *ifr = data; 4668 struct ifaddr *ifa = data; 4669 struct ifdrv *ifd = data; 4670 int error = 0; 4671 4672 switch (cmd) { 4673 case SIOCINITIFADDR: 4674 if (ifa->ifa_addr->sa_family != AF_LINK && 4675 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4676 (IFF_UP | IFF_RUNNING)) { 4677 ifp->if_flags |= IFF_UP; 4678 error = if_init(ifp); 4679 } 4680 return error; 4681 case SIOCADDMULTI: 4682 case SIOCDELMULTI: 4683 switch (ifr->ifr_addr.sa_family) { 4684 case AF_INET: /* IP supports Multicast */ 4685 break; 4686 #ifdef INET6 4687 case AF_INET6: /* IP6 supports Multicast */ 4688 break; 4689 #endif 4690 default: /* Other protocols doesn't support Multicast */ 4691 error = EAFNOSUPPORT; 4692 break; 4693 } 4694 return error; 4695 case SIOCSDRVSPEC: 4696 if (!wg_is_authorized(wg, cmd)) { 4697 return EPERM; 4698 } 4699 switch (ifd->ifd_cmd) { 4700 case WG_IOCTL_SET_PRIVATE_KEY: 4701 error = wg_ioctl_set_private_key(wg, ifd); 4702 break; 4703 case WG_IOCTL_SET_LISTEN_PORT: 4704 error = wg_ioctl_set_listen_port(wg, ifd); 4705 break; 4706 case WG_IOCTL_ADD_PEER: 4707 error = wg_ioctl_add_peer(wg, ifd); 4708 break; 4709 case WG_IOCTL_DELETE_PEER: 4710 error = wg_ioctl_delete_peer(wg, ifd); 4711 break; 4712 default: 4713 error = EINVAL; 4714 break; 4715 } 4716 return error; 4717 case SIOCGDRVSPEC: 4718 return wg_ioctl_get(wg, ifd); 4719 case SIOCSIFFLAGS: 4720 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4721 break; 4722 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4723 case IFF_RUNNING: 4724 /* 4725 * If interface is marked down and it is running, 4726 * then stop and disable it. 4727 */ 4728 if_stop(ifp, 1); 4729 break; 4730 case IFF_UP: 4731 /* 4732 * If interface is marked up and it is stopped, then 4733 * start it. 4734 */ 4735 error = if_init(ifp); 4736 break; 4737 default: 4738 break; 4739 } 4740 return error; 4741 #ifdef WG_RUMPKERNEL 4742 case SIOCSLINKSTR: 4743 error = wg_ioctl_linkstr(wg, ifd); 4744 if (error == 0) 4745 wg->wg_ops = &wg_ops_rumpuser; 4746 return error; 4747 #endif 4748 default: 4749 break; 4750 } 4751 4752 error = ifioctl_common(ifp, cmd, data); 4753 4754 #ifdef WG_RUMPKERNEL 4755 if (!wg_user_mode(wg)) 4756 return error; 4757 4758 /* Do the same to the corresponding tun device on the host */ 4759 /* 4760 * XXX Actually the command has not been handled yet. It 4761 * will be handled via pr_ioctl form doifioctl later. 4762 */ 4763 switch (cmd) { 4764 case SIOCAIFADDR: 4765 case SIOCDIFADDR: { 4766 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4767 struct in_aliasreq *ifra = &_ifra; 4768 KASSERT(error == ENOTTY); 4769 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4770 IFNAMSIZ); 4771 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4772 if (error == 0) 4773 error = ENOTTY; 4774 break; 4775 } 4776 #ifdef INET6 4777 case SIOCAIFADDR_IN6: 4778 case SIOCDIFADDR_IN6: { 4779 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4780 struct in6_aliasreq *ifra = &_ifra; 4781 KASSERT(error == ENOTTY); 4782 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4783 IFNAMSIZ); 4784 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4785 if (error == 0) 4786 error = ENOTTY; 4787 break; 4788 } 4789 #endif 4790 } 4791 #endif /* WG_RUMPKERNEL */ 4792 4793 return error; 4794 } 4795 4796 static int 4797 wg_init(struct ifnet *ifp) 4798 { 4799 4800 ifp->if_flags |= IFF_RUNNING; 4801 4802 /* TODO flush pending packets. */ 4803 return 0; 4804 } 4805 4806 #ifdef ALTQ 4807 static void 4808 wg_start(struct ifnet *ifp) 4809 { 4810 struct mbuf *m; 4811 4812 for (;;) { 4813 IFQ_DEQUEUE(&ifp->if_snd, m); 4814 if (m == NULL) 4815 break; 4816 4817 kpreempt_disable(); 4818 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4819 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4820 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4821 if_name(ifp)); 4822 m_freem(m); 4823 } 4824 kpreempt_enable(); 4825 } 4826 } 4827 #endif 4828 4829 static void 4830 wg_stop(struct ifnet *ifp, int disable) 4831 { 4832 4833 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4834 ifp->if_flags &= ~IFF_RUNNING; 4835 4836 /* Need to do something? */ 4837 } 4838 4839 #ifdef WG_DEBUG_PARAMS 4840 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4841 { 4842 const struct sysctlnode *node = NULL; 4843 4844 sysctl_createv(clog, 0, NULL, &node, 4845 CTLFLAG_PERMANENT, 4846 CTLTYPE_NODE, "wg", 4847 SYSCTL_DESCR("wg(4)"), 4848 NULL, 0, NULL, 0, 4849 CTL_NET, CTL_CREATE, CTL_EOL); 4850 sysctl_createv(clog, 0, &node, NULL, 4851 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4852 CTLTYPE_QUAD, "rekey_after_messages", 4853 SYSCTL_DESCR("session liftime by messages"), 4854 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4855 sysctl_createv(clog, 0, &node, NULL, 4856 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4857 CTLTYPE_INT, "rekey_after_time", 4858 SYSCTL_DESCR("session liftime"), 4859 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4860 sysctl_createv(clog, 0, &node, NULL, 4861 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4862 CTLTYPE_INT, "rekey_timeout", 4863 SYSCTL_DESCR("session handshake retry time"), 4864 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4865 sysctl_createv(clog, 0, &node, NULL, 4866 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4867 CTLTYPE_INT, "rekey_attempt_time", 4868 SYSCTL_DESCR("session handshake timeout"), 4869 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4870 sysctl_createv(clog, 0, &node, NULL, 4871 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4872 CTLTYPE_INT, "keepalive_timeout", 4873 SYSCTL_DESCR("keepalive timeout"), 4874 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4875 sysctl_createv(clog, 0, &node, NULL, 4876 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4877 CTLTYPE_BOOL, "force_underload", 4878 SYSCTL_DESCR("force to detemine under load"), 4879 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4880 } 4881 #endif 4882 4883 #ifdef WG_RUMPKERNEL 4884 static bool 4885 wg_user_mode(struct wg_softc *wg) 4886 { 4887 4888 return wg->wg_user != NULL; 4889 } 4890 4891 static int 4892 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4893 { 4894 struct ifnet *ifp = &wg->wg_if; 4895 int error; 4896 4897 if (ifp->if_flags & IFF_UP) 4898 return EBUSY; 4899 4900 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4901 /* XXX do nothing */ 4902 return 0; 4903 } else if (ifd->ifd_cmd != 0) { 4904 return EINVAL; 4905 } else if (wg->wg_user != NULL) { 4906 return EBUSY; 4907 } 4908 4909 /* Assume \0 included */ 4910 if (ifd->ifd_len > IFNAMSIZ) { 4911 return E2BIG; 4912 } else if (ifd->ifd_len < 1) { 4913 return EINVAL; 4914 } 4915 4916 char tun_name[IFNAMSIZ]; 4917 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4918 if (error != 0) 4919 return error; 4920 4921 if (strncmp(tun_name, "tun", 3) != 0) 4922 return EINVAL; 4923 4924 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 4925 4926 return error; 4927 } 4928 4929 static int 4930 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 4931 { 4932 int error; 4933 struct psref psref; 4934 struct wg_sockaddr *wgsa; 4935 struct wg_softc *wg = wgp->wgp_sc; 4936 struct iovec iov[1]; 4937 4938 wgsa = wg_get_endpoint_sa(wgp, &psref); 4939 4940 iov[0].iov_base = mtod(m, void *); 4941 iov[0].iov_len = m->m_len; 4942 4943 /* Send messages to a peer via an ordinary socket. */ 4944 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 4945 4946 wg_put_sa(wgp, wgsa, &psref); 4947 4948 m_freem(m); 4949 4950 return error; 4951 } 4952 4953 static void 4954 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 4955 { 4956 struct wg_softc *wg = ifp->if_softc; 4957 struct iovec iov[2]; 4958 struct sockaddr_storage ss; 4959 4960 KASSERT(af == AF_INET || af == AF_INET6); 4961 4962 WG_TRACE(""); 4963 4964 if (af == AF_INET) { 4965 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 4966 struct ip *ip; 4967 4968 KASSERT(m->m_len >= sizeof(struct ip)); 4969 ip = mtod(m, struct ip *); 4970 sockaddr_in_init(sin, &ip->ip_dst, 0); 4971 } else { 4972 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 4973 struct ip6_hdr *ip6; 4974 4975 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 4976 ip6 = mtod(m, struct ip6_hdr *); 4977 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 4978 } 4979 4980 iov[0].iov_base = &ss; 4981 iov[0].iov_len = ss.ss_len; 4982 iov[1].iov_base = mtod(m, void *); 4983 iov[1].iov_len = m->m_len; 4984 4985 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4986 4987 /* Send decrypted packets to users via a tun. */ 4988 rumpuser_wg_send_user(wg->wg_user, iov, 2); 4989 4990 m_freem(m); 4991 } 4992 4993 static int 4994 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 4995 { 4996 int error; 4997 uint16_t old_port = wg->wg_listen_port; 4998 4999 if (port != 0 && old_port == port) 5000 return 0; 5001 5002 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5003 if (error == 0) 5004 wg->wg_listen_port = port; 5005 return error; 5006 } 5007 5008 /* 5009 * Receive user packets. 5010 */ 5011 void 5012 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5013 { 5014 struct ifnet *ifp = &wg->wg_if; 5015 struct mbuf *m; 5016 const struct sockaddr *dst; 5017 5018 WG_TRACE(""); 5019 5020 dst = iov[0].iov_base; 5021 5022 m = m_gethdr(M_DONTWAIT, MT_DATA); 5023 if (m == NULL) 5024 return; 5025 m->m_len = m->m_pkthdr.len = 0; 5026 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5027 5028 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5029 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5030 5031 (void)wg_output(ifp, m, dst, NULL); 5032 } 5033 5034 /* 5035 * Receive packets from a peer. 5036 */ 5037 void 5038 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5039 { 5040 struct mbuf *m; 5041 const struct sockaddr *src; 5042 5043 WG_TRACE(""); 5044 5045 src = iov[0].iov_base; 5046 5047 m = m_gethdr(M_DONTWAIT, MT_DATA); 5048 if (m == NULL) 5049 return; 5050 m->m_len = m->m_pkthdr.len = 0; 5051 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5052 5053 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5054 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5055 5056 wg_handle_packet(wg, m, src); 5057 } 5058 #endif /* WG_RUMPKERNEL */ 5059 5060 /* 5061 * Module infrastructure 5062 */ 5063 #include "if_module.h" 5064 5065 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5066
Indexes created Thu Sep 25 16:09:42 GMT 2025