if_wg.c revision 1.84
1 /* $NetBSD: if_wg.c,v 1.84 2024/07/24 23:46:13 christos Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.84 2024/07/24 23:46:13 christos Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <net/bpf.h> 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/if_wg.h> 90 #include <net/pktqueue.h> 91 #include <net/route.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip.h> 97 #include <netinet/ip_var.h> 98 #include <netinet/udp.h> 99 #include <netinet/udp_var.h> 100 101 #ifdef INET6 102 #include <netinet/ip6.h> 103 #include <netinet6/in6_pcb.h> 104 #include <netinet6/in6_var.h> 105 #include <netinet6/ip6_var.h> 106 #include <netinet6/udp6_var.h> 107 #endif /* INET6 */ 108 109 #include <prop/proplib.h> 110 111 #include <crypto/blake2/blake2s.h> 112 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 113 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 114 #include <crypto/sodium/crypto_scalarmult.h> 115 116 #include "ioconf.h" 117 118 #ifdef WG_RUMPKERNEL 119 #include "wg_user.h" 120 #endif 121 122 /* 123 * Data structures 124 * - struct wg_softc is an instance of wg interfaces 125 * - It has a list of peers (struct wg_peer) 126 * - It has a threadpool job that sends/receives handshake messages and 127 * runs event handlers 128 * - It has its own two routing tables: one is for IPv4 and the other IPv6 129 * - struct wg_peer is a representative of a peer 130 * - It has a struct work to handle handshakes and timer tasks 131 * - It has a pair of session instances (struct wg_session) 132 * - It has a pair of endpoint instances (struct wg_sockaddr) 133 * - Normally one endpoint is used and the second one is used only on 134 * a peer migration (a change of peer's IP address) 135 * - It has a list of IP addresses and sub networks called allowedips 136 * (struct wg_allowedip) 137 * - A packets sent over a session is allowed if its destination matches 138 * any IP addresses or sub networks of the list 139 * - struct wg_session represents a session of a secure tunnel with a peer 140 * - Two instances of sessions belong to a peer; a stable session and a 141 * unstable session 142 * - A handshake process of a session always starts with a unstable instance 143 * - Once a session is established, its instance becomes stable and the 144 * other becomes unstable instead 145 * - Data messages are always sent via a stable session 146 * 147 * Locking notes: 148 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 149 * - Changes to the peer list are serialized by wg_lock 150 * - The peer list may be read with pserialize(9) and psref(9) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 152 * => XXX replace by pserialize when routing table is psz-safe 153 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 154 * only in thread context and serializes: 155 * - the stable and unstable session pointers 156 * - all unstable session state 157 * - Packet processing may be done in softint context: 158 * - The stable session can be read under pserialize(9) or psref(9) 159 * - The stable session is always ESTABLISHED 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 * - Lock order: wg_lock -> wgp_lock 164 */ 165 166 167 #define WGLOG(level, fmt, args...) \ 168 log(level, "%s: " fmt, __func__, ##args) 169 170 // #define WG_DEBUG 171 172 /* Debug options */ 173 #ifdef WG_DEBUG 174 /* Output debug logs */ 175 #ifndef WG_DEBUG_LOG 176 #define WG_DEBUG_LOG 177 #endif 178 /* Output trace logs */ 179 #ifndef WG_DEBUG_TRACE 180 #define WG_DEBUG_TRACE 181 #endif 182 /* Output hash values, etc. */ 183 #ifndef WG_DEBUG_DUMP 184 #define WG_DEBUG_DUMP 185 #endif 186 /* debug packets */ 187 #ifndef WG_DEBUG_PACKET 188 #define WG_DEBUG_PACKET 189 #endif 190 /* Make some internal parameters configurable for testing and debugging */ 191 #ifndef WG_DEBUG_PARAMS 192 #define WG_DEBUG_PARAMS 193 #endif 194 #endif /* WG_DEBUG */ 195 196 #ifndef WG_DEBUG 197 # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \ 198 defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS) 199 # define WG_DEBUG 200 # endif 201 #endif 202 203 #ifdef WG_DEBUG 204 int wg_debug; 205 #define WG_DEBUG_FLAGS_LOG 1 206 #define WG_DEBUG_FLAGS_TRACE 2 207 #define WG_DEBUG_FLAGS_DUMP 4 208 #define WG_DEBUG_FLAGS_PACKET 8 209 #endif 210 211 212 #ifdef WG_DEBUG_TRACE 213 #define WG_TRACE(msg) do { \ 214 if (wg_debug & WG_DEBUG_FLAGS_TRACE) \ 215 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \ 216 } while (0) 217 #else 218 #define WG_TRACE(msg) __nothing 219 #endif 220 221 #ifdef WG_DEBUG_LOG 222 #define WG_DLOG(fmt, args...) do { \ 223 if (wg_debug & WG_DEBUG_FLAGS_LOG) \ 224 log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \ 225 } while (0) 226 #else 227 #define WG_DLOG(fmt, args...) __nothing 228 #endif 229 230 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 231 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 232 &(wgprc)->wgprc_curpps, 1)) { \ 233 log(level, fmt, ##args); \ 234 } \ 235 } while (0) 236 237 #ifdef WG_DEBUG_PARAMS 238 static bool wg_force_underload = false; 239 #endif 240 241 #ifdef WG_DEBUG_DUMP 242 243 static char * 244 gethexdump(const void *vp, size_t n) 245 { 246 char *buf; 247 const uint8_t *p = vp; 248 size_t i; 249 250 if (n > SIZE_MAX/3 - 1) 251 return NULL; 252 buf = kmem_alloc(3*n + 1, KM_NOSLEEP); 253 if (buf == NULL) 254 return NULL; 255 for (i = 0; i < n; i++) 256 snprintf(buf + 3*i, 3 + 1, " %02hhx", p[i]); 257 return buf; 258 } 259 260 static void 261 puthexdump(char *buf, const void *p, size_t n) 262 { 263 264 if (buf == NULL) 265 return; 266 kmem_free(buf, 3*n + 1); 267 } 268 269 #ifdef WG_RUMPKERNEL 270 static void 271 wg_dump_buf(const char *func, const char *buf, const size_t size) 272 { 273 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 274 return; 275 276 char *hex = gethexdump(buf, size); 277 278 log(LOG_DEBUG, "%s: %s\n", func, hex ? hex : "(enomem)"); 279 puthexdump(hex, buf, size); 280 } 281 #endif 282 283 static void 284 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 285 const size_t size) 286 { 287 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 288 return; 289 290 char *hex = gethexdump(hash, size); 291 292 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex ? hex : "(enomem)"); 293 puthexdump(hex, hash, size); 294 } 295 296 #define WG_DUMP_HASH(name, hash) \ 297 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 298 #define WG_DUMP_HASH48(name, hash) \ 299 wg_dump_hash(__func__, name, hash, 48) 300 #define WG_DUMP_BUF(buf, size) \ 301 wg_dump_buf(__func__, buf, size) 302 #else 303 #define WG_DUMP_HASH(name, hash) __nothing 304 #define WG_DUMP_HASH48(name, hash) __nothing 305 #define WG_DUMP_BUF(buf, size) __nothing 306 #endif /* WG_DEBUG_DUMP */ 307 308 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 309 #define WG_MAX_PROPLEN 32766 310 311 #define WG_MTU 1420 312 #define WG_ALLOWEDIPS 16 313 314 #define CURVE25519_KEY_LEN 32 315 #define TAI64N_LEN sizeof(uint32_t) * 3 316 #define POLY1305_AUTHTAG_LEN 16 317 #define HMAC_BLOCK_LEN 64 318 319 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 320 /* [N] 4.3: Hash functions */ 321 #define NOISE_DHLEN 32 322 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 323 #define NOISE_HASHLEN 32 324 #define NOISE_BLOCKLEN 64 325 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 326 /* [N] 5.1: "k" */ 327 #define NOISE_CIPHER_KEY_LEN 32 328 /* 329 * [N] 9.2: "psk" 330 * "... psk is a 32-byte secret value provided by the application." 331 */ 332 #define NOISE_PRESHARED_KEY_LEN 32 333 334 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 335 #define WG_TIMESTAMP_LEN TAI64N_LEN 336 337 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 338 339 #define WG_COOKIE_LEN 16 340 #define WG_MAC_LEN 16 341 #define WG_RANDVAL_LEN 24 342 343 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 344 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 345 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 346 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 347 #define WG_HASH_LEN NOISE_HASHLEN 348 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 349 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 350 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 351 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 352 #define WG_DATA_KEY_LEN 32 353 #define WG_SALT_LEN 24 354 355 /* 356 * The protocol messages 357 */ 358 struct wg_msg { 359 uint32_t wgm_type; 360 } __packed; 361 362 /* [W] 5.4.2 First Message: Initiator to Responder */ 363 struct wg_msg_init { 364 uint32_t wgmi_type; 365 uint32_t wgmi_sender; 366 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 367 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 368 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 369 uint8_t wgmi_mac1[WG_MAC_LEN]; 370 uint8_t wgmi_mac2[WG_MAC_LEN]; 371 } __packed; 372 373 /* [W] 5.4.3 Second Message: Responder to Initiator */ 374 struct wg_msg_resp { 375 uint32_t wgmr_type; 376 uint32_t wgmr_sender; 377 uint32_t wgmr_receiver; 378 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 379 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 380 uint8_t wgmr_mac1[WG_MAC_LEN]; 381 uint8_t wgmr_mac2[WG_MAC_LEN]; 382 } __packed; 383 384 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 385 struct wg_msg_data { 386 uint32_t wgmd_type; 387 uint32_t wgmd_receiver; 388 uint64_t wgmd_counter; 389 uint32_t wgmd_packet[0]; 390 } __packed; 391 392 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 393 struct wg_msg_cookie { 394 uint32_t wgmc_type; 395 uint32_t wgmc_receiver; 396 uint8_t wgmc_salt[WG_SALT_LEN]; 397 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 398 } __packed; 399 400 #define WG_MSG_TYPE_INIT 1 401 #define WG_MSG_TYPE_RESP 2 402 #define WG_MSG_TYPE_COOKIE 3 403 #define WG_MSG_TYPE_DATA 4 404 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 405 406 /* Sliding windows */ 407 408 #define SLIWIN_BITS 2048u 409 #define SLIWIN_TYPE uint32_t 410 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 411 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 412 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 413 414 struct sliwin { 415 SLIWIN_TYPE B[SLIWIN_WORDS]; 416 uint64_t T; 417 }; 418 419 static void 420 sliwin_reset(struct sliwin *W) 421 { 422 423 memset(W, 0, sizeof(*W)); 424 } 425 426 static int 427 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 428 { 429 430 /* 431 * If it's more than one window older than the highest sequence 432 * number we've seen, reject. 433 */ 434 #ifdef __HAVE_ATOMIC64_LOADSTORE 435 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 436 return EAUTH; 437 #endif 438 439 /* 440 * Otherwise, we need to take the lock to decide, so don't 441 * reject just yet. Caller must serialize a call to 442 * sliwin_update in this case. 443 */ 444 return 0; 445 } 446 447 static int 448 sliwin_update(struct sliwin *W, uint64_t S) 449 { 450 unsigned word, bit; 451 452 /* 453 * If it's more than one window older than the highest sequence 454 * number we've seen, reject. 455 */ 456 if (S + SLIWIN_NPKT < W->T) 457 return EAUTH; 458 459 /* 460 * If it's higher than the highest sequence number we've seen, 461 * advance the window. 462 */ 463 if (S > W->T) { 464 uint64_t i = W->T / SLIWIN_BPW; 465 uint64_t j = S / SLIWIN_BPW; 466 unsigned k; 467 468 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 469 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 470 #ifdef __HAVE_ATOMIC64_LOADSTORE 471 atomic_store_relaxed(&W->T, S); 472 #else 473 W->T = S; 474 #endif 475 } 476 477 /* Test and set the bit -- if already set, reject. */ 478 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 479 bit = S % SLIWIN_BPW; 480 if (W->B[word] & (1UL << bit)) 481 return EAUTH; 482 W->B[word] |= 1U << bit; 483 484 /* Accept! */ 485 return 0; 486 } 487 488 struct wg_session { 489 struct wg_peer *wgs_peer; 490 struct psref_target 491 wgs_psref; 492 493 int wgs_state; 494 #define WGS_STATE_UNKNOWN 0 495 #define WGS_STATE_INIT_ACTIVE 1 496 #define WGS_STATE_INIT_PASSIVE 2 497 #define WGS_STATE_ESTABLISHED 3 498 #define WGS_STATE_DESTROYING 4 499 500 time_t wgs_time_established; 501 time_t wgs_time_last_data_sent; 502 bool wgs_is_initiator; 503 504 uint32_t wgs_local_index; 505 uint32_t wgs_remote_index; 506 #ifdef __HAVE_ATOMIC64_LOADSTORE 507 volatile uint64_t 508 wgs_send_counter; 509 #else 510 kmutex_t wgs_send_counter_lock; 511 uint64_t wgs_send_counter; 512 #endif 513 514 struct { 515 kmutex_t lock; 516 struct sliwin window; 517 } *wgs_recvwin; 518 519 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 520 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 521 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 522 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 523 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 524 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 525 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 526 }; 527 528 struct wg_sockaddr { 529 union { 530 struct sockaddr_storage _ss; 531 struct sockaddr _sa; 532 struct sockaddr_in _sin; 533 struct sockaddr_in6 _sin6; 534 }; 535 struct psref_target wgsa_psref; 536 }; 537 538 #define wgsatoss(wgsa) (&(wgsa)->_ss) 539 #define wgsatosa(wgsa) (&(wgsa)->_sa) 540 #define wgsatosin(wgsa) (&(wgsa)->_sin) 541 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 542 543 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 544 545 struct wg_peer; 546 struct wg_allowedip { 547 struct radix_node wga_nodes[2]; 548 struct wg_sockaddr _wga_sa_addr; 549 struct wg_sockaddr _wga_sa_mask; 550 #define wga_sa_addr _wga_sa_addr._sa 551 #define wga_sa_mask _wga_sa_mask._sa 552 553 int wga_family; 554 uint8_t wga_cidr; 555 union { 556 struct in_addr _ip4; 557 struct in6_addr _ip6; 558 } wga_addr; 559 #define wga_addr4 wga_addr._ip4 560 #define wga_addr6 wga_addr._ip6 561 562 struct wg_peer *wga_peer; 563 }; 564 565 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 566 567 struct wg_ppsratecheck { 568 struct timeval wgprc_lasttime; 569 int wgprc_curpps; 570 }; 571 572 struct wg_softc; 573 struct wg_peer { 574 struct wg_softc *wgp_sc; 575 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 576 struct pslist_entry wgp_peerlist_entry; 577 pserialize_t wgp_psz; 578 struct psref_target wgp_psref; 579 kmutex_t *wgp_lock; 580 kmutex_t *wgp_intr_lock; 581 582 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 583 struct wg_sockaddr *wgp_endpoint; 584 struct wg_sockaddr *wgp_endpoint0; 585 volatile unsigned wgp_endpoint_changing; 586 bool wgp_endpoint_available; 587 588 /* The preshared key (optional) */ 589 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 590 591 struct wg_session *wgp_session_stable; 592 struct wg_session *wgp_session_unstable; 593 594 /* first outgoing packet awaiting session initiation */ 595 struct mbuf *wgp_pending; 596 597 /* timestamp in big-endian */ 598 wg_timestamp_t wgp_timestamp_latest_init; 599 600 struct timespec wgp_last_handshake_time; 601 602 callout_t wgp_rekey_timer; 603 callout_t wgp_handshake_timeout_timer; 604 callout_t wgp_session_dtor_timer; 605 606 time_t wgp_handshake_start_time; 607 608 int wgp_n_allowedips; 609 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 610 611 time_t wgp_latest_cookie_time; 612 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 613 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 614 bool wgp_last_sent_mac1_valid; 615 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 616 bool wgp_last_sent_cookie_valid; 617 618 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 619 620 time_t wgp_last_genrandval_time; 621 uint32_t wgp_randval; 622 623 struct wg_ppsratecheck wgp_ppsratecheck; 624 625 struct work wgp_work; 626 unsigned int wgp_tasks; 627 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 628 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 629 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 630 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 631 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 632 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 633 }; 634 635 struct wg_ops; 636 637 struct wg_softc { 638 struct ifnet wg_if; 639 LIST_ENTRY(wg_softc) wg_list; 640 kmutex_t *wg_lock; 641 kmutex_t *wg_intr_lock; 642 krwlock_t *wg_rwlock; 643 644 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 645 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 646 647 int wg_npeers; 648 struct pslist_head wg_peers; 649 struct thmap *wg_peers_bypubkey; 650 struct thmap *wg_peers_byname; 651 struct thmap *wg_sessions_byindex; 652 uint16_t wg_listen_port; 653 654 struct threadpool *wg_threadpool; 655 656 struct threadpool_job wg_job; 657 int wg_upcalls; 658 #define WG_UPCALL_INET __BIT(0) 659 #define WG_UPCALL_INET6 __BIT(1) 660 661 #ifdef INET 662 struct socket *wg_so4; 663 struct radix_node_head *wg_rtable_ipv4; 664 #endif 665 #ifdef INET6 666 struct socket *wg_so6; 667 struct radix_node_head *wg_rtable_ipv6; 668 #endif 669 670 struct wg_ppsratecheck wg_ppsratecheck; 671 672 struct wg_ops *wg_ops; 673 674 #ifdef WG_RUMPKERNEL 675 struct wg_user *wg_user; 676 #endif 677 }; 678 679 /* [W] 6.1 Preliminaries */ 680 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 681 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 682 #define WG_REKEY_AFTER_TIME 120 683 #define WG_REJECT_AFTER_TIME 180 684 #define WG_REKEY_ATTEMPT_TIME 90 685 #define WG_REKEY_TIMEOUT 5 686 #define WG_KEEPALIVE_TIMEOUT 10 687 688 #define WG_COOKIE_TIME 120 689 #define WG_RANDVAL_TIME (2 * 60) 690 691 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 692 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 693 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 694 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 695 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 696 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 697 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 698 699 static struct mbuf * 700 wg_get_mbuf(size_t, size_t); 701 702 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 703 struct mbuf *); 704 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 705 const uint32_t, const uint8_t [WG_MAC_LEN], 706 const struct sockaddr *); 707 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 708 struct wg_session *, const struct wg_msg_init *); 709 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 710 711 static struct wg_peer * 712 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 713 struct psref *); 714 static struct wg_peer * 715 wg_lookup_peer_by_pubkey(struct wg_softc *, 716 const uint8_t [WG_STATIC_KEY_LEN], struct psref *); 717 718 static struct wg_session * 719 wg_lookup_session_by_index(struct wg_softc *, 720 const uint32_t, struct psref *); 721 722 static void wg_update_endpoint_if_necessary(struct wg_peer *, 723 const struct sockaddr *); 724 725 static void wg_schedule_rekey_timer(struct wg_peer *); 726 static void wg_schedule_session_dtor_timer(struct wg_peer *); 727 728 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 729 static void wg_calculate_keys(struct wg_session *, const bool); 730 731 static void wg_clear_states(struct wg_session *); 732 733 static void wg_get_peer(struct wg_peer *, struct psref *); 734 static void wg_put_peer(struct wg_peer *, struct psref *); 735 736 static int wg_send_so(struct wg_peer *, struct mbuf *); 737 static int wg_send_udp(struct wg_peer *, struct mbuf *); 738 static int wg_output(struct ifnet *, struct mbuf *, 739 const struct sockaddr *, const struct rtentry *); 740 static void wg_input(struct ifnet *, struct mbuf *, const int); 741 static int wg_ioctl(struct ifnet *, u_long, void *); 742 static int wg_bind_port(struct wg_softc *, const uint16_t); 743 static int wg_init(struct ifnet *); 744 #ifdef ALTQ 745 static void wg_start(struct ifnet *); 746 #endif 747 static void wg_stop(struct ifnet *, int); 748 749 static void wg_peer_work(struct work *, void *); 750 static void wg_job(struct threadpool_job *); 751 static void wgintr(void *); 752 static void wg_purge_pending_packets(struct wg_peer *); 753 754 static int wg_clone_create(struct if_clone *, int); 755 static int wg_clone_destroy(struct ifnet *); 756 757 struct wg_ops { 758 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 759 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 760 void (*input)(struct ifnet *, struct mbuf *, const int); 761 int (*bind_port)(struct wg_softc *, const uint16_t); 762 }; 763 764 struct wg_ops wg_ops_rumpkernel = { 765 .send_hs_msg = wg_send_so, 766 .send_data_msg = wg_send_udp, 767 .input = wg_input, 768 .bind_port = wg_bind_port, 769 }; 770 771 #ifdef WG_RUMPKERNEL 772 static bool wg_user_mode(struct wg_softc *); 773 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 774 775 static int wg_send_user(struct wg_peer *, struct mbuf *); 776 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 777 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 778 779 struct wg_ops wg_ops_rumpuser = { 780 .send_hs_msg = wg_send_user, 781 .send_data_msg = wg_send_user, 782 .input = wg_input_user, 783 .bind_port = wg_bind_port_user, 784 }; 785 #endif 786 787 #define WG_PEER_READER_FOREACH(wgp, wg) \ 788 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 789 wgp_peerlist_entry) 790 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 791 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 792 wgp_peerlist_entry) 793 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 794 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 795 #define WG_PEER_WRITER_REMOVE(wgp) \ 796 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 797 798 struct wg_route { 799 struct radix_node wgr_nodes[2]; 800 struct wg_peer *wgr_peer; 801 }; 802 803 static struct radix_node_head * 804 wg_rnh(struct wg_softc *wg, const int family) 805 { 806 807 switch (family) { 808 case AF_INET: 809 return wg->wg_rtable_ipv4; 810 #ifdef INET6 811 case AF_INET6: 812 return wg->wg_rtable_ipv6; 813 #endif 814 default: 815 return NULL; 816 } 817 } 818 819 820 /* 821 * Global variables 822 */ 823 static volatile unsigned wg_count __cacheline_aligned; 824 825 struct psref_class *wg_psref_class __read_mostly; 826 827 static struct if_clone wg_cloner = 828 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 829 830 static struct pktqueue *wg_pktq __read_mostly; 831 static struct workqueue *wg_wq __read_mostly; 832 833 void wgattach(int); 834 /* ARGSUSED */ 835 void 836 wgattach(int count) 837 { 838 /* 839 * Nothing to do here, initialization is handled by the 840 * module initialization code in wginit() below). 841 */ 842 } 843 844 static void 845 wginit(void) 846 { 847 848 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 849 850 if_clone_attach(&wg_cloner); 851 } 852 853 /* 854 * XXX Kludge: This should just happen in wginit, but workqueue_create 855 * cannot be run until after CPUs have been detected, and wginit runs 856 * before configure. 857 */ 858 static int 859 wginitqueues(void) 860 { 861 int error __diagused; 862 863 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 864 KASSERT(wg_pktq != NULL); 865 866 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 867 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 868 KASSERT(error == 0); 869 870 return 0; 871 } 872 873 static void 874 wg_guarantee_initialized(void) 875 { 876 static ONCE_DECL(init); 877 int error __diagused; 878 879 error = RUN_ONCE(&init, wginitqueues); 880 KASSERT(error == 0); 881 } 882 883 static int 884 wg_count_inc(void) 885 { 886 unsigned o, n; 887 888 do { 889 o = atomic_load_relaxed(&wg_count); 890 if (o == UINT_MAX) 891 return ENFILE; 892 n = o + 1; 893 } while (atomic_cas_uint(&wg_count, o, n) != o); 894 895 return 0; 896 } 897 898 static void 899 wg_count_dec(void) 900 { 901 unsigned c __diagused; 902 903 c = atomic_dec_uint_nv(&wg_count); 904 KASSERT(c != UINT_MAX); 905 } 906 907 static int 908 wgdetach(void) 909 { 910 911 /* Prevent new interface creation. */ 912 if_clone_detach(&wg_cloner); 913 914 /* Check whether there are any existing interfaces. */ 915 if (atomic_load_relaxed(&wg_count)) { 916 /* Back out -- reattach the cloner. */ 917 if_clone_attach(&wg_cloner); 918 return EBUSY; 919 } 920 921 /* No interfaces left. Nuke it. */ 922 workqueue_destroy(wg_wq); 923 pktq_destroy(wg_pktq); 924 psref_class_destroy(wg_psref_class); 925 926 return 0; 927 } 928 929 static void 930 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 931 uint8_t hash[WG_HASH_LEN]) 932 { 933 /* [W] 5.4: CONSTRUCTION */ 934 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 935 /* [W] 5.4: IDENTIFIER */ 936 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 937 struct blake2s state; 938 939 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 940 signature, strlen(signature)); 941 942 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 943 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 944 945 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 946 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 947 blake2s_update(&state, id, strlen(id)); 948 blake2s_final(&state, hash); 949 950 WG_DUMP_HASH("ckey", ckey); 951 WG_DUMP_HASH("hash", hash); 952 } 953 954 static void 955 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 956 const size_t inputsize) 957 { 958 struct blake2s state; 959 960 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 961 blake2s_update(&state, hash, WG_HASH_LEN); 962 blake2s_update(&state, input, inputsize); 963 blake2s_final(&state, hash); 964 } 965 966 static void 967 wg_algo_mac(uint8_t out[], const size_t outsize, 968 const uint8_t key[], const size_t keylen, 969 const uint8_t input1[], const size_t input1len, 970 const uint8_t input2[], const size_t input2len) 971 { 972 struct blake2s state; 973 974 blake2s_init(&state, outsize, key, keylen); 975 976 blake2s_update(&state, input1, input1len); 977 if (input2 != NULL) 978 blake2s_update(&state, input2, input2len); 979 blake2s_final(&state, out); 980 } 981 982 static void 983 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 984 const uint8_t input1[], const size_t input1len, 985 const uint8_t input2[], const size_t input2len) 986 { 987 struct blake2s state; 988 /* [W] 5.4: LABEL-MAC1 */ 989 const char *label = "mac1----"; 990 uint8_t key[WG_HASH_LEN]; 991 992 blake2s_init(&state, sizeof(key), NULL, 0); 993 blake2s_update(&state, label, strlen(label)); 994 blake2s_update(&state, input1, input1len); 995 blake2s_final(&state, key); 996 997 blake2s_init(&state, outsize, key, sizeof(key)); 998 if (input2 != NULL) 999 blake2s_update(&state, input2, input2len); 1000 blake2s_final(&state, out); 1001 } 1002 1003 static void 1004 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 1005 const uint8_t input1[], const size_t input1len) 1006 { 1007 struct blake2s state; 1008 /* [W] 5.4: LABEL-COOKIE */ 1009 const char *label = "cookie--"; 1010 1011 blake2s_init(&state, outsize, NULL, 0); 1012 blake2s_update(&state, label, strlen(label)); 1013 blake2s_update(&state, input1, input1len); 1014 blake2s_final(&state, out); 1015 } 1016 1017 static void 1018 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 1019 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 1020 { 1021 1022 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1023 1024 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 1025 crypto_scalarmult_base(pubkey, privkey); 1026 } 1027 1028 static void 1029 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 1030 const uint8_t privkey[WG_STATIC_KEY_LEN], 1031 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 1032 { 1033 1034 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1035 1036 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1037 KASSERT(ret == 0); 1038 } 1039 1040 static void 1041 wg_algo_hmac(uint8_t out[], const size_t outlen, 1042 const uint8_t key[], const size_t keylen, 1043 const uint8_t in[], const size_t inlen) 1044 { 1045 #define IPAD 0x36 1046 #define OPAD 0x5c 1047 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1048 uint8_t ipad[HMAC_BLOCK_LEN]; 1049 uint8_t opad[HMAC_BLOCK_LEN]; 1050 size_t i; 1051 struct blake2s state; 1052 1053 KASSERT(outlen == WG_HASH_LEN); 1054 KASSERT(keylen <= HMAC_BLOCK_LEN); 1055 1056 memcpy(hmackey, key, keylen); 1057 1058 for (i = 0; i < sizeof(hmackey); i++) { 1059 ipad[i] = hmackey[i] ^ IPAD; 1060 opad[i] = hmackey[i] ^ OPAD; 1061 } 1062 1063 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1064 blake2s_update(&state, ipad, sizeof(ipad)); 1065 blake2s_update(&state, in, inlen); 1066 blake2s_final(&state, out); 1067 1068 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1069 blake2s_update(&state, opad, sizeof(opad)); 1070 blake2s_update(&state, out, WG_HASH_LEN); 1071 blake2s_final(&state, out); 1072 #undef IPAD 1073 #undef OPAD 1074 } 1075 1076 static void 1077 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1078 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1079 const uint8_t input[], const size_t inputlen) 1080 { 1081 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1082 uint8_t one[1]; 1083 1084 /* 1085 * [N] 4.3: "an input_key_material byte sequence with length 1086 * either zero bytes, 32 bytes, or DHLEN bytes." 1087 */ 1088 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1089 1090 WG_DUMP_HASH("ckey", ckey); 1091 if (input != NULL) 1092 WG_DUMP_HASH("input", input); 1093 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1094 input, inputlen); 1095 WG_DUMP_HASH("tmp1", tmp1); 1096 one[0] = 1; 1097 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1098 one, sizeof(one)); 1099 WG_DUMP_HASH("out1", out1); 1100 if (out2 == NULL) 1101 return; 1102 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1103 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1104 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1105 tmp2, sizeof(tmp2)); 1106 WG_DUMP_HASH("out2", out2); 1107 if (out3 == NULL) 1108 return; 1109 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1110 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1111 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1112 tmp2, sizeof(tmp2)); 1113 WG_DUMP_HASH("out3", out3); 1114 } 1115 1116 static void __noinline 1117 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1118 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1119 const uint8_t local_key[WG_STATIC_KEY_LEN], 1120 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1121 { 1122 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1123 1124 wg_algo_dh(dhout, local_key, remote_key); 1125 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1126 1127 WG_DUMP_HASH("dhout", dhout); 1128 WG_DUMP_HASH("ckey", ckey); 1129 if (cipher_key != NULL) 1130 WG_DUMP_HASH("cipher_key", cipher_key); 1131 } 1132 1133 static void 1134 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1135 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1136 const uint8_t auth[], size_t authlen) 1137 { 1138 uint8_t nonce[(32 + 64) / 8] = {0}; 1139 long long unsigned int outsize; 1140 int error __diagused; 1141 1142 le64enc(&nonce[4], counter); 1143 1144 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1145 plainsize, auth, authlen, NULL, nonce, key); 1146 KASSERT(error == 0); 1147 KASSERT(outsize == expected_outsize); 1148 } 1149 1150 static int 1151 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1152 const uint64_t counter, const uint8_t encrypted[], 1153 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1154 { 1155 uint8_t nonce[(32 + 64) / 8] = {0}; 1156 long long unsigned int outsize; 1157 int error; 1158 1159 le64enc(&nonce[4], counter); 1160 1161 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1162 encrypted, encryptedsize, auth, authlen, nonce, key); 1163 if (error == 0) 1164 KASSERT(outsize == expected_outsize); 1165 return error; 1166 } 1167 1168 static void 1169 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1170 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1171 const uint8_t auth[], size_t authlen, 1172 const uint8_t nonce[WG_SALT_LEN]) 1173 { 1174 long long unsigned int outsize; 1175 int error __diagused; 1176 1177 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1178 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1179 plain, plainsize, auth, authlen, NULL, nonce, key); 1180 KASSERT(error == 0); 1181 KASSERT(outsize == expected_outsize); 1182 } 1183 1184 static int 1185 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1186 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1187 const uint8_t auth[], size_t authlen, 1188 const uint8_t nonce[WG_SALT_LEN]) 1189 { 1190 long long unsigned int outsize; 1191 int error; 1192 1193 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1194 encrypted, encryptedsize, auth, authlen, nonce, key); 1195 if (error == 0) 1196 KASSERT(outsize == expected_outsize); 1197 return error; 1198 } 1199 1200 static void 1201 wg_algo_tai64n(wg_timestamp_t timestamp) 1202 { 1203 struct timespec ts; 1204 1205 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1206 getnanotime(&ts); 1207 /* TAI64 label in external TAI64 format */ 1208 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1209 /* second beginning from 1970 TAI */ 1210 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1211 /* nanosecond in big-endian format */ 1212 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1213 } 1214 1215 /* 1216 * wg_get_stable_session(wgp, psref) 1217 * 1218 * Get a passive reference to the current stable session, or 1219 * return NULL if there is no current stable session. 1220 * 1221 * The pointer is always there but the session is not necessarily 1222 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1223 * the session may transition from ESTABLISHED to DESTROYING while 1224 * holding the passive reference. 1225 */ 1226 static struct wg_session * 1227 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1228 { 1229 int s; 1230 struct wg_session *wgs; 1231 1232 s = pserialize_read_enter(); 1233 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1234 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1235 wgs = NULL; 1236 else 1237 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1238 pserialize_read_exit(s); 1239 1240 return wgs; 1241 } 1242 1243 static void 1244 wg_put_session(struct wg_session *wgs, struct psref *psref) 1245 { 1246 1247 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1248 } 1249 1250 static void 1251 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1252 { 1253 struct wg_peer *wgp = wgs->wgs_peer; 1254 struct wg_session *wgs0 __diagused; 1255 void *garbage; 1256 1257 KASSERT(mutex_owned(wgp->wgp_lock)); 1258 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1259 1260 /* Remove the session from the table. */ 1261 wgs0 = thmap_del(wg->wg_sessions_byindex, 1262 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1263 KASSERT(wgs0 == wgs); 1264 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1265 1266 /* Wait for passive references to drain. */ 1267 pserialize_perform(wgp->wgp_psz); 1268 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1269 1270 /* Free memory, zero state, and transition to UNKNOWN. */ 1271 thmap_gc(wg->wg_sessions_byindex, garbage); 1272 wg_clear_states(wgs); 1273 wgs->wgs_state = WGS_STATE_UNKNOWN; 1274 } 1275 1276 /* 1277 * wg_get_session_index(wg, wgs) 1278 * 1279 * Choose a session index for wgs->wgs_local_index, and store it 1280 * in wg's table of sessions by index. 1281 * 1282 * wgs must be the unstable session of its peer, and must be 1283 * transitioning out of the UNKNOWN state. 1284 */ 1285 static void 1286 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1287 { 1288 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1289 struct wg_session *wgs0; 1290 uint32_t index; 1291 1292 KASSERT(mutex_owned(wgp->wgp_lock)); 1293 KASSERT(wgs == wgp->wgp_session_unstable); 1294 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1295 1296 do { 1297 /* Pick a uniform random index. */ 1298 index = cprng_strong32(); 1299 1300 /* Try to take it. */ 1301 wgs->wgs_local_index = index; 1302 wgs0 = thmap_put(wg->wg_sessions_byindex, 1303 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1304 1305 /* If someone else beat us, start over. */ 1306 } while (__predict_false(wgs0 != wgs)); 1307 } 1308 1309 /* 1310 * wg_put_session_index(wg, wgs) 1311 * 1312 * Remove wgs from the table of sessions by index, wait for any 1313 * passive references to drain, and transition the session to the 1314 * UNKNOWN state. 1315 * 1316 * wgs must be the unstable session of its peer, and must not be 1317 * UNKNOWN or ESTABLISHED. 1318 */ 1319 static void 1320 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1321 { 1322 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1323 1324 KASSERT(mutex_owned(wgp->wgp_lock)); 1325 KASSERT(wgs == wgp->wgp_session_unstable); 1326 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1327 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1328 1329 wg_destroy_session(wg, wgs); 1330 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1331 } 1332 1333 /* 1334 * Handshake patterns 1335 * 1336 * [W] 5: "These messages use the "IK" pattern from Noise" 1337 * [N] 7.5. Interactive handshake patterns (fundamental) 1338 * "The first character refers to the initiators static key:" 1339 * "I = Static key for initiator Immediately transmitted to responder, 1340 * despite reduced or absent identity hiding" 1341 * "The second character refers to the responders static key:" 1342 * "K = Static key for responder Known to initiator" 1343 * "IK: 1344 * <- s 1345 * ... 1346 * -> e, es, s, ss 1347 * <- e, ee, se" 1348 * [N] 9.4. Pattern modifiers 1349 * "IKpsk2: 1350 * <- s 1351 * ... 1352 * -> e, es, s, ss 1353 * <- e, ee, se, psk" 1354 */ 1355 static void 1356 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1357 struct wg_session *wgs, struct wg_msg_init *wgmi) 1358 { 1359 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1360 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1361 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1362 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1363 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1364 1365 KASSERT(mutex_owned(wgp->wgp_lock)); 1366 KASSERT(wgs == wgp->wgp_session_unstable); 1367 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1368 1369 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1370 wgmi->wgmi_sender = wgs->wgs_local_index; 1371 1372 /* [W] 5.4.2: First Message: Initiator to Responder */ 1373 1374 /* Ci := HASH(CONSTRUCTION) */ 1375 /* Hi := HASH(Ci || IDENTIFIER) */ 1376 wg_init_key_and_hash(ckey, hash); 1377 /* Hi := HASH(Hi || Sr^pub) */ 1378 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1379 1380 WG_DUMP_HASH("hash", hash); 1381 1382 /* [N] 2.2: "e" */ 1383 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1384 wg_algo_generate_keypair(pubkey, privkey); 1385 /* Ci := KDF1(Ci, Ei^pub) */ 1386 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1387 /* msg.ephemeral := Ei^pub */ 1388 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1389 /* Hi := HASH(Hi || msg.ephemeral) */ 1390 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1391 1392 WG_DUMP_HASH("ckey", ckey); 1393 WG_DUMP_HASH("hash", hash); 1394 1395 /* [N] 2.2: "es" */ 1396 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1397 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1398 1399 /* [N] 2.2: "s" */ 1400 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1401 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1402 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1403 hash, sizeof(hash)); 1404 /* Hi := HASH(Hi || msg.static) */ 1405 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1406 1407 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1408 1409 /* [N] 2.2: "ss" */ 1410 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1411 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1412 1413 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1414 wg_timestamp_t timestamp; 1415 wg_algo_tai64n(timestamp); 1416 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1417 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1418 /* Hi := HASH(Hi || msg.timestamp) */ 1419 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1420 1421 /* [W] 5.4.4 Cookie MACs */ 1422 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1423 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1424 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1425 /* Need mac1 to decrypt a cookie from a cookie message */ 1426 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1427 sizeof(wgp->wgp_last_sent_mac1)); 1428 wgp->wgp_last_sent_mac1_valid = true; 1429 1430 if (wgp->wgp_latest_cookie_time == 0 || 1431 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1432 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1433 else { 1434 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1435 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1436 (const uint8_t *)wgmi, 1437 offsetof(struct wg_msg_init, wgmi_mac2), 1438 NULL, 0); 1439 } 1440 1441 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1442 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1443 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1444 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1445 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1446 } 1447 1448 static void __noinline 1449 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1450 const struct sockaddr *src) 1451 { 1452 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1453 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1454 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1455 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1456 struct wg_peer *wgp; 1457 struct wg_session *wgs; 1458 int error, ret; 1459 struct psref psref_peer; 1460 uint8_t mac1[WG_MAC_LEN]; 1461 1462 WG_TRACE("init msg received"); 1463 1464 wg_algo_mac_mac1(mac1, sizeof(mac1), 1465 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1466 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1467 1468 /* 1469 * [W] 5.3: Denial of Service Mitigation & Cookies 1470 * "the responder, ..., must always reject messages with an invalid 1471 * msg.mac1" 1472 */ 1473 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1474 WG_DLOG("mac1 is invalid\n"); 1475 return; 1476 } 1477 1478 /* 1479 * [W] 5.4.2: First Message: Initiator to Responder 1480 * "When the responder receives this message, it does the same 1481 * operations so that its final state variables are identical, 1482 * replacing the operands of the DH function to produce equivalent 1483 * values." 1484 * Note that the following comments of operations are just copies of 1485 * the initiator's ones. 1486 */ 1487 1488 /* Ci := HASH(CONSTRUCTION) */ 1489 /* Hi := HASH(Ci || IDENTIFIER) */ 1490 wg_init_key_and_hash(ckey, hash); 1491 /* Hi := HASH(Hi || Sr^pub) */ 1492 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1493 1494 /* [N] 2.2: "e" */ 1495 /* Ci := KDF1(Ci, Ei^pub) */ 1496 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1497 sizeof(wgmi->wgmi_ephemeral)); 1498 /* Hi := HASH(Hi || msg.ephemeral) */ 1499 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1500 1501 WG_DUMP_HASH("ckey", ckey); 1502 1503 /* [N] 2.2: "es" */ 1504 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1505 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1506 1507 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1508 1509 /* [N] 2.2: "s" */ 1510 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1511 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1512 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1513 if (error != 0) { 1514 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1515 "%s: wg_algo_aead_dec for secret key failed\n", 1516 if_name(&wg->wg_if)); 1517 return; 1518 } 1519 /* Hi := HASH(Hi || msg.static) */ 1520 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1521 1522 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1523 if (wgp == NULL) { 1524 WG_DLOG("peer not found\n"); 1525 return; 1526 } 1527 1528 /* 1529 * Lock the peer to serialize access to cookie state. 1530 * 1531 * XXX Can we safely avoid holding the lock across DH? Take it 1532 * just to verify mac2 and then unlock/DH/lock? 1533 */ 1534 mutex_enter(wgp->wgp_lock); 1535 1536 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1537 WG_TRACE("under load"); 1538 /* 1539 * [W] 5.3: Denial of Service Mitigation & Cookies 1540 * "the responder, ..., and when under load may reject messages 1541 * with an invalid msg.mac2. If the responder receives a 1542 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1543 * and is under load, it may respond with a cookie reply 1544 * message" 1545 */ 1546 uint8_t zero[WG_MAC_LEN] = {0}; 1547 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1548 WG_TRACE("sending a cookie message: no cookie included"); 1549 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1550 wgmi->wgmi_mac1, src); 1551 goto out; 1552 } 1553 if (!wgp->wgp_last_sent_cookie_valid) { 1554 WG_TRACE("sending a cookie message: no cookie sent ever"); 1555 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1556 wgmi->wgmi_mac1, src); 1557 goto out; 1558 } 1559 uint8_t mac2[WG_MAC_LEN]; 1560 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1561 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1562 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1563 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1564 WG_DLOG("mac2 is invalid\n"); 1565 goto out; 1566 } 1567 WG_TRACE("under load, but continue to sending"); 1568 } 1569 1570 /* [N] 2.2: "ss" */ 1571 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1572 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1573 1574 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1575 wg_timestamp_t timestamp; 1576 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1577 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1578 hash, sizeof(hash)); 1579 if (error != 0) { 1580 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1581 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1582 if_name(&wg->wg_if), wgp->wgp_name); 1583 goto out; 1584 } 1585 /* Hi := HASH(Hi || msg.timestamp) */ 1586 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1587 1588 /* 1589 * [W] 5.1 "The responder keeps track of the greatest timestamp 1590 * received per peer and discards packets containing 1591 * timestamps less than or equal to it." 1592 */ 1593 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1594 sizeof(timestamp)); 1595 if (ret <= 0) { 1596 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1597 "%s: peer %s: invalid init msg: timestamp is old\n", 1598 if_name(&wg->wg_if), wgp->wgp_name); 1599 goto out; 1600 } 1601 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1602 1603 /* 1604 * Message is good -- we're committing to handle it now, unless 1605 * we were already initiating a session. 1606 */ 1607 wgs = wgp->wgp_session_unstable; 1608 switch (wgs->wgs_state) { 1609 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1610 wg_get_session_index(wg, wgs); 1611 break; 1612 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1613 WG_TRACE("Session already initializing, ignoring the message"); 1614 goto out; 1615 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1616 WG_TRACE("Session already initializing, destroying old states"); 1617 wg_clear_states(wgs); 1618 /* keep session index */ 1619 break; 1620 case WGS_STATE_ESTABLISHED: /* can't happen */ 1621 panic("unstable session can't be established"); 1622 break; 1623 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1624 WG_TRACE("Session destroying, but force to clear"); 1625 callout_stop(&wgp->wgp_session_dtor_timer); 1626 wg_clear_states(wgs); 1627 /* keep session index */ 1628 break; 1629 default: 1630 panic("invalid session state: %d", wgs->wgs_state); 1631 } 1632 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1633 1634 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1635 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1636 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1637 sizeof(wgmi->wgmi_ephemeral)); 1638 1639 wg_update_endpoint_if_necessary(wgp, src); 1640 1641 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1642 1643 wg_calculate_keys(wgs, false); 1644 wg_clear_states(wgs); 1645 1646 out: 1647 mutex_exit(wgp->wgp_lock); 1648 wg_put_peer(wgp, &psref_peer); 1649 } 1650 1651 static struct socket * 1652 wg_get_so_by_af(struct wg_softc *wg, const int af) 1653 { 1654 1655 switch (af) { 1656 #ifdef INET 1657 case AF_INET: 1658 return wg->wg_so4; 1659 #endif 1660 #ifdef INET6 1661 case AF_INET6: 1662 return wg->wg_so6; 1663 #endif 1664 default: 1665 panic("wg: no such af: %d", af); 1666 } 1667 } 1668 1669 static struct socket * 1670 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1671 { 1672 1673 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1674 } 1675 1676 static struct wg_sockaddr * 1677 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1678 { 1679 struct wg_sockaddr *wgsa; 1680 int s; 1681 1682 s = pserialize_read_enter(); 1683 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1684 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1685 pserialize_read_exit(s); 1686 1687 return wgsa; 1688 } 1689 1690 static void 1691 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1692 { 1693 1694 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1695 } 1696 1697 static int 1698 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1699 { 1700 int error; 1701 struct socket *so; 1702 struct psref psref; 1703 struct wg_sockaddr *wgsa; 1704 1705 wgsa = wg_get_endpoint_sa(wgp, &psref); 1706 so = wg_get_so_by_peer(wgp, wgsa); 1707 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1708 wg_put_sa(wgp, wgsa, &psref); 1709 1710 return error; 1711 } 1712 1713 static int 1714 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1715 { 1716 int error; 1717 struct mbuf *m; 1718 struct wg_msg_init *wgmi; 1719 struct wg_session *wgs; 1720 1721 KASSERT(mutex_owned(wgp->wgp_lock)); 1722 1723 wgs = wgp->wgp_session_unstable; 1724 /* XXX pull dispatch out into wg_task_send_init_message */ 1725 switch (wgs->wgs_state) { 1726 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1727 wg_get_session_index(wg, wgs); 1728 break; 1729 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1730 WG_TRACE("Session already initializing, skip starting new one"); 1731 return EBUSY; 1732 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1733 WG_TRACE("Session already initializing, destroying old states"); 1734 wg_clear_states(wgs); 1735 /* keep session index */ 1736 break; 1737 case WGS_STATE_ESTABLISHED: /* can't happen */ 1738 panic("unstable session can't be established"); 1739 break; 1740 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1741 WG_TRACE("Session destroying"); 1742 /* XXX should wait? */ 1743 return EBUSY; 1744 } 1745 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1746 1747 m = m_gethdr(M_WAIT, MT_DATA); 1748 if (sizeof(*wgmi) > MHLEN) { 1749 m_clget(m, M_WAIT); 1750 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1751 } 1752 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1753 wgmi = mtod(m, struct wg_msg_init *); 1754 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1755 1756 error = wg->wg_ops->send_hs_msg(wgp, m); 1757 if (error == 0) { 1758 WG_TRACE("init msg sent"); 1759 1760 if (wgp->wgp_handshake_start_time == 0) 1761 wgp->wgp_handshake_start_time = time_uptime; 1762 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1763 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1764 } else { 1765 wg_put_session_index(wg, wgs); 1766 /* Initiation failed; toss packet waiting for it if any. */ 1767 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 1768 m_freem(m); 1769 } 1770 1771 return error; 1772 } 1773 1774 static void 1775 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1776 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1777 const struct wg_msg_init *wgmi) 1778 { 1779 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1780 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1781 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1782 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1783 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1784 1785 KASSERT(mutex_owned(wgp->wgp_lock)); 1786 KASSERT(wgs == wgp->wgp_session_unstable); 1787 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1788 1789 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1790 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1791 1792 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1793 wgmr->wgmr_sender = wgs->wgs_local_index; 1794 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1795 1796 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1797 1798 /* [N] 2.2: "e" */ 1799 /* Er^priv, Er^pub := DH-GENERATE() */ 1800 wg_algo_generate_keypair(pubkey, privkey); 1801 /* Cr := KDF1(Cr, Er^pub) */ 1802 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1803 /* msg.ephemeral := Er^pub */ 1804 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1805 /* Hr := HASH(Hr || msg.ephemeral) */ 1806 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1807 1808 WG_DUMP_HASH("ckey", ckey); 1809 WG_DUMP_HASH("hash", hash); 1810 1811 /* [N] 2.2: "ee" */ 1812 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1813 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1814 1815 /* [N] 2.2: "se" */ 1816 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1817 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1818 1819 /* [N] 9.2: "psk" */ 1820 { 1821 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1822 /* Cr, r, k := KDF3(Cr, Q) */ 1823 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1824 sizeof(wgp->wgp_psk)); 1825 /* Hr := HASH(Hr || r) */ 1826 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1827 } 1828 1829 /* msg.empty := AEAD(k, 0, e, Hr) */ 1830 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1831 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1832 /* Hr := HASH(Hr || msg.empty) */ 1833 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1834 1835 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1836 1837 /* [W] 5.4.4: Cookie MACs */ 1838 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1839 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1840 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1841 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1842 /* Need mac1 to decrypt a cookie from a cookie message */ 1843 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1844 sizeof(wgp->wgp_last_sent_mac1)); 1845 wgp->wgp_last_sent_mac1_valid = true; 1846 1847 if (wgp->wgp_latest_cookie_time == 0 || 1848 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1849 /* msg.mac2 := 0^16 */ 1850 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1851 else { 1852 /* msg.mac2 := MAC(Lm, msg_b) */ 1853 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1854 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1855 (const uint8_t *)wgmr, 1856 offsetof(struct wg_msg_resp, wgmr_mac2), 1857 NULL, 0); 1858 } 1859 1860 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1861 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1862 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1863 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1864 wgs->wgs_remote_index = wgmi->wgmi_sender; 1865 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1866 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1867 } 1868 1869 static void 1870 wg_swap_sessions(struct wg_peer *wgp) 1871 { 1872 struct wg_session *wgs, *wgs_prev; 1873 1874 KASSERT(mutex_owned(wgp->wgp_lock)); 1875 1876 wgs = wgp->wgp_session_unstable; 1877 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1878 1879 wgs_prev = wgp->wgp_session_stable; 1880 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1881 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1882 atomic_store_release(&wgp->wgp_session_stable, wgs); 1883 wgp->wgp_session_unstable = wgs_prev; 1884 } 1885 1886 static void __noinline 1887 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1888 const struct sockaddr *src) 1889 { 1890 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1891 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1892 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1893 struct wg_peer *wgp; 1894 struct wg_session *wgs; 1895 struct psref psref; 1896 int error; 1897 uint8_t mac1[WG_MAC_LEN]; 1898 struct wg_session *wgs_prev; 1899 struct mbuf *m; 1900 1901 wg_algo_mac_mac1(mac1, sizeof(mac1), 1902 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1903 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1904 1905 /* 1906 * [W] 5.3: Denial of Service Mitigation & Cookies 1907 * "the responder, ..., must always reject messages with an invalid 1908 * msg.mac1" 1909 */ 1910 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1911 WG_DLOG("mac1 is invalid\n"); 1912 return; 1913 } 1914 1915 WG_TRACE("resp msg received"); 1916 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1917 if (wgs == NULL) { 1918 WG_TRACE("No session found"); 1919 return; 1920 } 1921 1922 wgp = wgs->wgs_peer; 1923 1924 mutex_enter(wgp->wgp_lock); 1925 1926 /* If we weren't waiting for a handshake response, drop it. */ 1927 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1928 WG_TRACE("peer sent spurious handshake response, ignoring"); 1929 goto out; 1930 } 1931 1932 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1933 WG_TRACE("under load"); 1934 /* 1935 * [W] 5.3: Denial of Service Mitigation & Cookies 1936 * "the responder, ..., and when under load may reject messages 1937 * with an invalid msg.mac2. If the responder receives a 1938 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1939 * and is under load, it may respond with a cookie reply 1940 * message" 1941 */ 1942 uint8_t zero[WG_MAC_LEN] = {0}; 1943 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1944 WG_TRACE("sending a cookie message: no cookie included"); 1945 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1946 wgmr->wgmr_mac1, src); 1947 goto out; 1948 } 1949 if (!wgp->wgp_last_sent_cookie_valid) { 1950 WG_TRACE("sending a cookie message: no cookie sent ever"); 1951 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1952 wgmr->wgmr_mac1, src); 1953 goto out; 1954 } 1955 uint8_t mac2[WG_MAC_LEN]; 1956 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1957 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1958 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1959 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1960 WG_DLOG("mac2 is invalid\n"); 1961 goto out; 1962 } 1963 WG_TRACE("under load, but continue to sending"); 1964 } 1965 1966 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1967 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1968 1969 /* 1970 * [W] 5.4.3 Second Message: Responder to Initiator 1971 * "When the initiator receives this message, it does the same 1972 * operations so that its final state variables are identical, 1973 * replacing the operands of the DH function to produce equivalent 1974 * values." 1975 * Note that the following comments of operations are just copies of 1976 * the initiator's ones. 1977 */ 1978 1979 /* [N] 2.2: "e" */ 1980 /* Cr := KDF1(Cr, Er^pub) */ 1981 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1982 sizeof(wgmr->wgmr_ephemeral)); 1983 /* Hr := HASH(Hr || msg.ephemeral) */ 1984 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1985 1986 WG_DUMP_HASH("ckey", ckey); 1987 WG_DUMP_HASH("hash", hash); 1988 1989 /* [N] 2.2: "ee" */ 1990 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1991 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1992 wgmr->wgmr_ephemeral); 1993 1994 /* [N] 2.2: "se" */ 1995 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1996 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1997 1998 /* [N] 9.2: "psk" */ 1999 { 2000 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 2001 /* Cr, r, k := KDF3(Cr, Q) */ 2002 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 2003 sizeof(wgp->wgp_psk)); 2004 /* Hr := HASH(Hr || r) */ 2005 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 2006 } 2007 2008 { 2009 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 2010 /* msg.empty := AEAD(k, 0, e, Hr) */ 2011 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 2012 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 2013 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 2014 if (error != 0) { 2015 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2016 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 2017 if_name(&wg->wg_if), wgp->wgp_name); 2018 goto out; 2019 } 2020 /* Hr := HASH(Hr || msg.empty) */ 2021 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 2022 } 2023 2024 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 2025 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 2026 wgs->wgs_remote_index = wgmr->wgmr_sender; 2027 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 2028 2029 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 2030 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2031 wgs->wgs_time_established = time_uptime; 2032 wgs->wgs_time_last_data_sent = 0; 2033 wgs->wgs_is_initiator = true; 2034 wg_calculate_keys(wgs, true); 2035 wg_clear_states(wgs); 2036 WG_TRACE("WGS_STATE_ESTABLISHED"); 2037 2038 callout_stop(&wgp->wgp_handshake_timeout_timer); 2039 2040 wg_swap_sessions(wgp); 2041 KASSERT(wgs == wgp->wgp_session_stable); 2042 wgs_prev = wgp->wgp_session_unstable; 2043 getnanotime(&wgp->wgp_last_handshake_time); 2044 wgp->wgp_handshake_start_time = 0; 2045 wgp->wgp_last_sent_mac1_valid = false; 2046 wgp->wgp_last_sent_cookie_valid = false; 2047 2048 wg_schedule_rekey_timer(wgp); 2049 2050 wg_update_endpoint_if_necessary(wgp, src); 2051 2052 /* 2053 * If we had a data packet queued up, send it; otherwise send a 2054 * keepalive message -- either way we have to send something 2055 * immediately or else the responder will never answer. 2056 */ 2057 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2058 kpreempt_disable(); 2059 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2060 M_SETCTX(m, wgp); 2061 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2062 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2063 if_name(&wg->wg_if)); 2064 m_freem(m); 2065 } 2066 kpreempt_enable(); 2067 } else { 2068 wg_send_keepalive_msg(wgp, wgs); 2069 } 2070 2071 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2072 /* Wait for wg_get_stable_session to drain. */ 2073 pserialize_perform(wgp->wgp_psz); 2074 2075 /* Transition ESTABLISHED->DESTROYING. */ 2076 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2077 2078 /* We can't destroy the old session immediately */ 2079 wg_schedule_session_dtor_timer(wgp); 2080 } else { 2081 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2082 "state=%d", wgs_prev->wgs_state); 2083 } 2084 2085 out: 2086 mutex_exit(wgp->wgp_lock); 2087 wg_put_session(wgs, &psref); 2088 } 2089 2090 static int 2091 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2092 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2093 { 2094 int error; 2095 struct mbuf *m; 2096 struct wg_msg_resp *wgmr; 2097 2098 KASSERT(mutex_owned(wgp->wgp_lock)); 2099 KASSERT(wgs == wgp->wgp_session_unstable); 2100 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 2101 2102 m = m_gethdr(M_WAIT, MT_DATA); 2103 if (sizeof(*wgmr) > MHLEN) { 2104 m_clget(m, M_WAIT); 2105 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2106 } 2107 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2108 wgmr = mtod(m, struct wg_msg_resp *); 2109 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2110 2111 error = wg->wg_ops->send_hs_msg(wgp, m); 2112 if (error == 0) 2113 WG_TRACE("resp msg sent"); 2114 return error; 2115 } 2116 2117 static struct wg_peer * 2118 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2119 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2120 { 2121 struct wg_peer *wgp; 2122 2123 int s = pserialize_read_enter(); 2124 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2125 if (wgp != NULL) 2126 wg_get_peer(wgp, psref); 2127 pserialize_read_exit(s); 2128 2129 return wgp; 2130 } 2131 2132 static void 2133 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2134 struct wg_msg_cookie *wgmc, const uint32_t sender, 2135 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2136 { 2137 uint8_t cookie[WG_COOKIE_LEN]; 2138 uint8_t key[WG_HASH_LEN]; 2139 uint8_t addr[sizeof(struct in6_addr)]; 2140 size_t addrlen; 2141 uint16_t uh_sport; /* be */ 2142 2143 KASSERT(mutex_owned(wgp->wgp_lock)); 2144 2145 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2146 wgmc->wgmc_receiver = sender; 2147 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2148 2149 /* 2150 * [W] 5.4.7: Under Load: Cookie Reply Message 2151 * "The secret variable, Rm, changes every two minutes to a 2152 * random value" 2153 */ 2154 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2155 wgp->wgp_randval = cprng_strong32(); 2156 wgp->wgp_last_genrandval_time = time_uptime; 2157 } 2158 2159 switch (src->sa_family) { 2160 case AF_INET: { 2161 const struct sockaddr_in *sin = satocsin(src); 2162 addrlen = sizeof(sin->sin_addr); 2163 memcpy(addr, &sin->sin_addr, addrlen); 2164 uh_sport = sin->sin_port; 2165 break; 2166 } 2167 #ifdef INET6 2168 case AF_INET6: { 2169 const struct sockaddr_in6 *sin6 = satocsin6(src); 2170 addrlen = sizeof(sin6->sin6_addr); 2171 memcpy(addr, &sin6->sin6_addr, addrlen); 2172 uh_sport = sin6->sin6_port; 2173 break; 2174 } 2175 #endif 2176 default: 2177 panic("invalid af=%d", src->sa_family); 2178 } 2179 2180 wg_algo_mac(cookie, sizeof(cookie), 2181 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2182 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2183 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2184 sizeof(wg->wg_pubkey)); 2185 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2186 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2187 2188 /* Need to store to calculate mac2 */ 2189 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2190 wgp->wgp_last_sent_cookie_valid = true; 2191 } 2192 2193 static int 2194 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2195 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2196 const struct sockaddr *src) 2197 { 2198 int error; 2199 struct mbuf *m; 2200 struct wg_msg_cookie *wgmc; 2201 2202 KASSERT(mutex_owned(wgp->wgp_lock)); 2203 2204 m = m_gethdr(M_WAIT, MT_DATA); 2205 if (sizeof(*wgmc) > MHLEN) { 2206 m_clget(m, M_WAIT); 2207 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2208 } 2209 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2210 wgmc = mtod(m, struct wg_msg_cookie *); 2211 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2212 2213 error = wg->wg_ops->send_hs_msg(wgp, m); 2214 if (error == 0) 2215 WG_TRACE("cookie msg sent"); 2216 return error; 2217 } 2218 2219 static bool 2220 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2221 { 2222 #ifdef WG_DEBUG_PARAMS 2223 if (wg_force_underload) 2224 return true; 2225 #endif 2226 2227 /* 2228 * XXX we don't have a means of a load estimation. The purpose of 2229 * the mechanism is a DoS mitigation, so we consider frequent handshake 2230 * messages as (a kind of) load; if a message of the same type comes 2231 * to a peer within 1 second, we consider we are under load. 2232 */ 2233 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2234 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2235 return (time_uptime - last) == 0; 2236 } 2237 2238 static void 2239 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2240 { 2241 2242 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2243 2244 /* 2245 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2246 */ 2247 if (initiator) { 2248 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2249 wgs->wgs_chaining_key, NULL, 0); 2250 } else { 2251 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2252 wgs->wgs_chaining_key, NULL, 0); 2253 } 2254 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2255 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2256 } 2257 2258 static uint64_t 2259 wg_session_get_send_counter(struct wg_session *wgs) 2260 { 2261 #ifdef __HAVE_ATOMIC64_LOADSTORE 2262 return atomic_load_relaxed(&wgs->wgs_send_counter); 2263 #else 2264 uint64_t send_counter; 2265 2266 mutex_enter(&wgs->wgs_send_counter_lock); 2267 send_counter = wgs->wgs_send_counter; 2268 mutex_exit(&wgs->wgs_send_counter_lock); 2269 2270 return send_counter; 2271 #endif 2272 } 2273 2274 static uint64_t 2275 wg_session_inc_send_counter(struct wg_session *wgs) 2276 { 2277 #ifdef __HAVE_ATOMIC64_LOADSTORE 2278 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2279 #else 2280 uint64_t send_counter; 2281 2282 mutex_enter(&wgs->wgs_send_counter_lock); 2283 send_counter = wgs->wgs_send_counter++; 2284 mutex_exit(&wgs->wgs_send_counter_lock); 2285 2286 return send_counter; 2287 #endif 2288 } 2289 2290 static void 2291 wg_clear_states(struct wg_session *wgs) 2292 { 2293 2294 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2295 2296 wgs->wgs_send_counter = 0; 2297 sliwin_reset(&wgs->wgs_recvwin->window); 2298 2299 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2300 wgs_clear(handshake_hash); 2301 wgs_clear(chaining_key); 2302 wgs_clear(ephemeral_key_pub); 2303 wgs_clear(ephemeral_key_priv); 2304 wgs_clear(ephemeral_key_peer); 2305 #undef wgs_clear 2306 } 2307 2308 static struct wg_session * 2309 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2310 struct psref *psref) 2311 { 2312 struct wg_session *wgs; 2313 2314 int s = pserialize_read_enter(); 2315 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2316 if (wgs != NULL) { 2317 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2318 WGS_STATE_UNKNOWN); 2319 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2320 } 2321 pserialize_read_exit(s); 2322 2323 return wgs; 2324 } 2325 2326 static void 2327 wg_schedule_rekey_timer(struct wg_peer *wgp) 2328 { 2329 int timeout = MIN(wg_rekey_after_time, (unsigned)(INT_MAX / hz)); 2330 2331 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2332 } 2333 2334 static void 2335 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2336 { 2337 struct mbuf *m; 2338 2339 /* 2340 * [W] 6.5 Passive Keepalive 2341 * "A keepalive message is simply a transport data message with 2342 * a zero-length encapsulated encrypted inner-packet." 2343 */ 2344 WG_TRACE(""); 2345 m = m_gethdr(M_WAIT, MT_DATA); 2346 wg_send_data_msg(wgp, wgs, m); 2347 } 2348 2349 static bool 2350 wg_need_to_send_init_message(struct wg_session *wgs) 2351 { 2352 /* 2353 * [W] 6.2 Transport Message Limits 2354 * "if a peer is the initiator of a current secure session, 2355 * WireGuard will send a handshake initiation message to begin 2356 * a new secure session ... if after receiving a transport data 2357 * message, the current secure session is (REJECT-AFTER-TIME 2358 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2359 * not yet acted upon this event." 2360 */ 2361 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2362 (time_uptime - wgs->wgs_time_established) >= 2363 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2364 } 2365 2366 static void 2367 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2368 { 2369 2370 mutex_enter(wgp->wgp_intr_lock); 2371 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2372 if (wgp->wgp_tasks == 0) 2373 /* 2374 * XXX If the current CPU is already loaded -- e.g., if 2375 * there's already a bunch of handshakes queued up -- 2376 * consider tossing this over to another CPU to 2377 * distribute the load. 2378 */ 2379 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2380 wgp->wgp_tasks |= task; 2381 mutex_exit(wgp->wgp_intr_lock); 2382 } 2383 2384 static void 2385 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2386 { 2387 struct wg_sockaddr *wgsa_prev; 2388 2389 WG_TRACE("Changing endpoint"); 2390 2391 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2392 wgsa_prev = wgp->wgp_endpoint; 2393 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2394 wgp->wgp_endpoint0 = wgsa_prev; 2395 atomic_store_release(&wgp->wgp_endpoint_available, true); 2396 2397 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2398 } 2399 2400 static bool 2401 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2402 { 2403 uint16_t packet_len; 2404 const struct ip *ip; 2405 2406 if (__predict_false(decrypted_len < sizeof(*ip))) { 2407 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2408 sizeof(*ip)); 2409 return false; 2410 } 2411 2412 ip = (const struct ip *)packet; 2413 if (ip->ip_v == 4) 2414 *af = AF_INET; 2415 else if (ip->ip_v == 6) 2416 *af = AF_INET6; 2417 else { 2418 WG_DLOG("ip_v=%d\n", ip->ip_v); 2419 return false; 2420 } 2421 2422 WG_DLOG("af=%d\n", *af); 2423 2424 switch (*af) { 2425 #ifdef INET 2426 case AF_INET: 2427 packet_len = ntohs(ip->ip_len); 2428 break; 2429 #endif 2430 #ifdef INET6 2431 case AF_INET6: { 2432 const struct ip6_hdr *ip6; 2433 2434 if (__predict_false(decrypted_len < sizeof(*ip6))) { 2435 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2436 sizeof(*ip6)); 2437 return false; 2438 } 2439 2440 ip6 = (const struct ip6_hdr *)packet; 2441 packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); 2442 break; 2443 } 2444 #endif 2445 default: 2446 return false; 2447 } 2448 2449 if (packet_len > decrypted_len) { 2450 WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len, 2451 decrypted_len); 2452 return false; 2453 } 2454 2455 return true; 2456 } 2457 2458 static bool 2459 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2460 int af, char *packet) 2461 { 2462 struct sockaddr_storage ss; 2463 struct sockaddr *sa; 2464 struct psref psref; 2465 struct wg_peer *wgp; 2466 bool ok; 2467 2468 /* 2469 * II CRYPTOKEY ROUTING 2470 * "it will only accept it if its source IP resolves in the 2471 * table to the public key used in the secure session for 2472 * decrypting it." 2473 */ 2474 2475 if (af == AF_INET) { 2476 const struct ip *ip = (const struct ip *)packet; 2477 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2478 sockaddr_in_init(sin, &ip->ip_src, 0); 2479 sa = sintosa(sin); 2480 #ifdef INET6 2481 } else { 2482 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2483 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2484 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2485 sa = sin6tosa(sin6); 2486 #endif 2487 } 2488 2489 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2490 ok = (wgp == wgp_expected); 2491 if (wgp != NULL) 2492 wg_put_peer(wgp, &psref); 2493 2494 return ok; 2495 } 2496 2497 static void 2498 wg_session_dtor_timer(void *arg) 2499 { 2500 struct wg_peer *wgp = arg; 2501 2502 WG_TRACE("enter"); 2503 2504 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2505 } 2506 2507 static void 2508 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2509 { 2510 2511 /* 1 second grace period */ 2512 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2513 } 2514 2515 static bool 2516 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2517 { 2518 if (sa1->sa_family != sa2->sa_family) 2519 return false; 2520 2521 switch (sa1->sa_family) { 2522 #ifdef INET 2523 case AF_INET: 2524 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2525 #endif 2526 #ifdef INET6 2527 case AF_INET6: 2528 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2529 #endif 2530 default: 2531 return false; 2532 } 2533 } 2534 2535 static void 2536 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2537 const struct sockaddr *src) 2538 { 2539 struct wg_sockaddr *wgsa; 2540 struct psref psref; 2541 2542 wgsa = wg_get_endpoint_sa(wgp, &psref); 2543 2544 #ifdef WG_DEBUG_LOG 2545 char oldaddr[128], newaddr[128]; 2546 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2547 sockaddr_format(src, newaddr, sizeof(newaddr)); 2548 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2549 #endif 2550 2551 /* 2552 * III: "Since the packet has authenticated correctly, the source IP of 2553 * the outer UDP/IP packet is used to update the endpoint for peer..." 2554 */ 2555 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2556 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2557 /* XXX We can't change the endpoint twice in a short period */ 2558 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2559 wg_change_endpoint(wgp, src); 2560 } 2561 } 2562 2563 wg_put_sa(wgp, wgsa, &psref); 2564 } 2565 2566 static void __noinline 2567 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2568 const struct sockaddr *src) 2569 { 2570 struct wg_msg_data *wgmd; 2571 char *encrypted_buf = NULL, *decrypted_buf; 2572 size_t encrypted_len, decrypted_len; 2573 struct wg_session *wgs; 2574 struct wg_peer *wgp; 2575 int state; 2576 size_t mlen; 2577 struct psref psref; 2578 int error, af; 2579 bool success, free_encrypted_buf = false, ok; 2580 struct mbuf *n; 2581 2582 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2583 wgmd = mtod(m, struct wg_msg_data *); 2584 2585 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2586 WG_TRACE("data"); 2587 2588 /* Find the putative session, or drop. */ 2589 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2590 if (wgs == NULL) { 2591 WG_TRACE("No session found"); 2592 m_freem(m); 2593 return; 2594 } 2595 2596 /* 2597 * We are only ready to handle data when in INIT_PASSIVE, 2598 * ESTABLISHED, or DESTROYING. All transitions out of that 2599 * state dissociate the session index and drain psrefs. 2600 */ 2601 state = atomic_load_relaxed(&wgs->wgs_state); 2602 switch (state) { 2603 case WGS_STATE_UNKNOWN: 2604 panic("wg session %p in unknown state has session index %u", 2605 wgs, wgmd->wgmd_receiver); 2606 case WGS_STATE_INIT_ACTIVE: 2607 WG_TRACE("not yet ready for data"); 2608 goto out; 2609 case WGS_STATE_INIT_PASSIVE: 2610 case WGS_STATE_ESTABLISHED: 2611 case WGS_STATE_DESTROYING: 2612 break; 2613 } 2614 2615 /* 2616 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2617 * to update the endpoint if authentication succeeds. 2618 */ 2619 wgp = wgs->wgs_peer; 2620 2621 /* 2622 * Reject outrageously wrong sequence numbers before doing any 2623 * crypto work or taking any locks. 2624 */ 2625 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2626 le64toh(wgmd->wgmd_counter)); 2627 if (error) { 2628 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2629 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2630 if_name(&wg->wg_if), wgp->wgp_name, 2631 le64toh(wgmd->wgmd_counter)); 2632 goto out; 2633 } 2634 2635 /* Ensure the payload and authenticator are contiguous. */ 2636 mlen = m_length(m); 2637 encrypted_len = mlen - sizeof(*wgmd); 2638 if (encrypted_len < WG_AUTHTAG_LEN) { 2639 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2640 goto out; 2641 } 2642 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2643 if (success) { 2644 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2645 } else { 2646 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2647 if (encrypted_buf == NULL) { 2648 WG_DLOG("failed to allocate encrypted_buf\n"); 2649 goto out; 2650 } 2651 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2652 free_encrypted_buf = true; 2653 } 2654 /* m_ensure_contig may change m regardless of its result */ 2655 KASSERT(m->m_len >= sizeof(*wgmd)); 2656 wgmd = mtod(m, struct wg_msg_data *); 2657 2658 /* 2659 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2660 * a zero-length buffer (XXX). Drop if plaintext is longer 2661 * than MCLBYTES (XXX). 2662 */ 2663 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2664 if (decrypted_len > MCLBYTES) { 2665 /* FIXME handle larger data than MCLBYTES */ 2666 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2667 goto out; 2668 } 2669 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2670 if (n == NULL) { 2671 WG_DLOG("wg_get_mbuf failed\n"); 2672 goto out; 2673 } 2674 decrypted_buf = mtod(n, char *); 2675 2676 /* Decrypt and verify the packet. */ 2677 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2678 error = wg_algo_aead_dec(decrypted_buf, 2679 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2680 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2681 encrypted_len, NULL, 0); 2682 if (error != 0) { 2683 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2684 "%s: peer %s: failed to wg_algo_aead_dec\n", 2685 if_name(&wg->wg_if), wgp->wgp_name); 2686 m_freem(n); 2687 goto out; 2688 } 2689 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2690 2691 /* Packet is genuine. Reject it if a replay or just too old. */ 2692 mutex_enter(&wgs->wgs_recvwin->lock); 2693 error = sliwin_update(&wgs->wgs_recvwin->window, 2694 le64toh(wgmd->wgmd_counter)); 2695 mutex_exit(&wgs->wgs_recvwin->lock); 2696 if (error) { 2697 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2698 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2699 if_name(&wg->wg_if), wgp->wgp_name, 2700 le64toh(wgmd->wgmd_counter)); 2701 m_freem(n); 2702 goto out; 2703 } 2704 2705 #ifdef WG_DEBUG_PACKET 2706 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2707 char *hex = gethexdump(wgmd, sizeof(*wgmd)); 2708 log(LOG_DEBUG, "wgmd=%p, sizeof(*wgmd)=%zu\n%s\n", 2709 wgmd, sizeof(*wgmd), hex); 2710 puthexdump(hex, wgmd, sizeof(*wgmd)); 2711 hex = gethexdump(decrypted_buf, decrypted_len); 2712 log(LOG_DEBUG, "decrypted_buf=%p, decrypted_len=%zu\n%s\n", 2713 decrypted_buf, decrypted_len, hex); 2714 puthexdump(hex, decrypted_buf, decrypted_len); 2715 } 2716 #endif 2717 /* We're done with m now; free it and chuck the pointers. */ 2718 m_freem(m); 2719 m = NULL; 2720 wgmd = NULL; 2721 2722 /* 2723 * Validate the encapsulated packet header and get the address 2724 * family, or drop. 2725 */ 2726 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2727 if (!ok) { 2728 m_freem(n); 2729 goto out; 2730 } 2731 2732 /* 2733 * The packet is genuine. Update the peer's endpoint if the 2734 * source address changed. 2735 * 2736 * XXX How to prevent DoS by replaying genuine packets from the 2737 * wrong source address? 2738 */ 2739 wg_update_endpoint_if_necessary(wgp, src); 2740 2741 /* Submit it into our network stack if routable. */ 2742 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2743 if (ok) { 2744 wg->wg_ops->input(&wg->wg_if, n, af); 2745 } else { 2746 char addrstr[INET6_ADDRSTRLEN]; 2747 memset(addrstr, 0, sizeof(addrstr)); 2748 if (af == AF_INET) { 2749 const struct ip *ip = (const struct ip *)decrypted_buf; 2750 IN_PRINT(addrstr, &ip->ip_src); 2751 #ifdef INET6 2752 } else if (af == AF_INET6) { 2753 const struct ip6_hdr *ip6 = 2754 (const struct ip6_hdr *)decrypted_buf; 2755 IN6_PRINT(addrstr, &ip6->ip6_src); 2756 #endif 2757 } 2758 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2759 "%s: peer %s: invalid source address (%s)\n", 2760 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2761 m_freem(n); 2762 /* 2763 * The inner address is invalid however the session is valid 2764 * so continue the session processing below. 2765 */ 2766 } 2767 n = NULL; 2768 2769 /* Update the state machine if necessary. */ 2770 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2771 /* 2772 * We were waiting for the initiator to send their 2773 * first data transport message, and that has happened. 2774 * Schedule a task to establish this session. 2775 */ 2776 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2777 } else { 2778 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2779 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2780 } 2781 /* 2782 * [W] 6.5 Passive Keepalive 2783 * "If a peer has received a validly-authenticated transport 2784 * data message (section 5.4.6), but does not have any packets 2785 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2786 * a keepalive message." 2787 */ 2788 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2789 (uintmax_t)time_uptime, 2790 (uintmax_t)wgs->wgs_time_last_data_sent); 2791 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2792 wg_keepalive_timeout) { 2793 WG_TRACE("Schedule sending keepalive message"); 2794 /* 2795 * We can't send a keepalive message here to avoid 2796 * a deadlock; we already hold the solock of a socket 2797 * that is used to send the message. 2798 */ 2799 wg_schedule_peer_task(wgp, 2800 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2801 } 2802 } 2803 out: 2804 wg_put_session(wgs, &psref); 2805 m_freem(m); 2806 if (free_encrypted_buf) 2807 kmem_intr_free(encrypted_buf, encrypted_len); 2808 } 2809 2810 static void __noinline 2811 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2812 { 2813 struct wg_session *wgs; 2814 struct wg_peer *wgp; 2815 struct psref psref; 2816 int error; 2817 uint8_t key[WG_HASH_LEN]; 2818 uint8_t cookie[WG_COOKIE_LEN]; 2819 2820 WG_TRACE("cookie msg received"); 2821 2822 /* Find the putative session. */ 2823 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2824 if (wgs == NULL) { 2825 WG_TRACE("No session found"); 2826 return; 2827 } 2828 2829 /* Lock the peer so we can update the cookie state. */ 2830 wgp = wgs->wgs_peer; 2831 mutex_enter(wgp->wgp_lock); 2832 2833 if (!wgp->wgp_last_sent_mac1_valid) { 2834 WG_TRACE("No valid mac1 sent (or expired)"); 2835 goto out; 2836 } 2837 2838 /* Decrypt the cookie and store it for later handshake retry. */ 2839 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2840 sizeof(wgp->wgp_pubkey)); 2841 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2842 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2843 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2844 wgmc->wgmc_salt); 2845 if (error != 0) { 2846 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2847 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 2848 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 2849 goto out; 2850 } 2851 /* 2852 * [W] 6.6: Interaction with Cookie Reply System 2853 * "it should simply store the decrypted cookie value from the cookie 2854 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2855 * timer for retrying a handshake initiation message." 2856 */ 2857 wgp->wgp_latest_cookie_time = time_uptime; 2858 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2859 out: 2860 mutex_exit(wgp->wgp_lock); 2861 wg_put_session(wgs, &psref); 2862 } 2863 2864 static struct mbuf * 2865 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2866 { 2867 struct wg_msg wgm; 2868 size_t mbuflen; 2869 size_t msglen; 2870 2871 /* 2872 * Get the mbuf chain length. It is already guaranteed, by 2873 * wg_overudp_cb, to be large enough for a struct wg_msg. 2874 */ 2875 mbuflen = m_length(m); 2876 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2877 2878 /* 2879 * Copy the message header (32-bit message type) out -- we'll 2880 * worry about contiguity and alignment later. 2881 */ 2882 m_copydata(m, 0, sizeof(wgm), &wgm); 2883 switch (le32toh(wgm.wgm_type)) { 2884 case WG_MSG_TYPE_INIT: 2885 msglen = sizeof(struct wg_msg_init); 2886 break; 2887 case WG_MSG_TYPE_RESP: 2888 msglen = sizeof(struct wg_msg_resp); 2889 break; 2890 case WG_MSG_TYPE_COOKIE: 2891 msglen = sizeof(struct wg_msg_cookie); 2892 break; 2893 case WG_MSG_TYPE_DATA: 2894 msglen = sizeof(struct wg_msg_data); 2895 break; 2896 default: 2897 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2898 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 2899 le32toh(wgm.wgm_type)); 2900 goto error; 2901 } 2902 2903 /* Verify the mbuf chain is long enough for this type of message. */ 2904 if (__predict_false(mbuflen < msglen)) { 2905 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2906 le32toh(wgm.wgm_type)); 2907 goto error; 2908 } 2909 2910 /* Make the message header contiguous if necessary. */ 2911 if (__predict_false(m->m_len < msglen)) { 2912 m = m_pullup(m, msglen); 2913 if (m == NULL) 2914 return NULL; 2915 } 2916 2917 return m; 2918 2919 error: 2920 m_freem(m); 2921 return NULL; 2922 } 2923 2924 static void 2925 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2926 const struct sockaddr *src) 2927 { 2928 struct wg_msg *wgm; 2929 2930 KASSERT(curlwp->l_pflag & LP_BOUND); 2931 2932 m = wg_validate_msg_header(wg, m); 2933 if (__predict_false(m == NULL)) 2934 return; 2935 2936 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2937 wgm = mtod(m, struct wg_msg *); 2938 switch (le32toh(wgm->wgm_type)) { 2939 case WG_MSG_TYPE_INIT: 2940 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2941 break; 2942 case WG_MSG_TYPE_RESP: 2943 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2944 break; 2945 case WG_MSG_TYPE_COOKIE: 2946 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2947 break; 2948 case WG_MSG_TYPE_DATA: 2949 wg_handle_msg_data(wg, m, src); 2950 /* wg_handle_msg_data frees m for us */ 2951 return; 2952 default: 2953 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2954 } 2955 2956 m_freem(m); 2957 } 2958 2959 static void 2960 wg_receive_packets(struct wg_softc *wg, const int af) 2961 { 2962 2963 for (;;) { 2964 int error, flags; 2965 struct socket *so; 2966 struct mbuf *m = NULL; 2967 struct uio dummy_uio; 2968 struct mbuf *paddr = NULL; 2969 struct sockaddr *src; 2970 2971 so = wg_get_so_by_af(wg, af); 2972 flags = MSG_DONTWAIT; 2973 dummy_uio.uio_resid = 1000000000; 2974 2975 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2976 &flags); 2977 if (error || m == NULL) { 2978 //if (error == EWOULDBLOCK) 2979 return; 2980 } 2981 2982 KASSERT(paddr != NULL); 2983 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2984 src = mtod(paddr, struct sockaddr *); 2985 2986 wg_handle_packet(wg, m, src); 2987 } 2988 } 2989 2990 static void 2991 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2992 { 2993 2994 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2995 } 2996 2997 static void 2998 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2999 { 3000 3001 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 3002 } 3003 3004 static void 3005 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 3006 { 3007 struct wg_session *wgs; 3008 3009 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 3010 3011 KASSERT(mutex_owned(wgp->wgp_lock)); 3012 3013 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 3014 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 3015 if_name(&wg->wg_if)); 3016 /* XXX should do something? */ 3017 return; 3018 } 3019 3020 wgs = wgp->wgp_session_stable; 3021 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 3022 /* XXX What if the unstable session is already INIT_ACTIVE? */ 3023 wg_send_handshake_msg_init(wg, wgp); 3024 } else { 3025 /* rekey */ 3026 wgs = wgp->wgp_session_unstable; 3027 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3028 wg_send_handshake_msg_init(wg, wgp); 3029 } 3030 } 3031 3032 static void 3033 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 3034 { 3035 struct wg_session *wgs; 3036 3037 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 3038 3039 KASSERT(mutex_owned(wgp->wgp_lock)); 3040 KASSERT(wgp->wgp_handshake_start_time != 0); 3041 3042 wgs = wgp->wgp_session_unstable; 3043 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3044 return; 3045 3046 /* 3047 * XXX no real need to assign a new index here, but we do need 3048 * to transition to UNKNOWN temporarily 3049 */ 3050 wg_put_session_index(wg, wgs); 3051 3052 /* [W] 6.4 Handshake Initiation Retransmission */ 3053 if ((time_uptime - wgp->wgp_handshake_start_time) > 3054 wg_rekey_attempt_time) { 3055 /* Give up handshaking */ 3056 wgp->wgp_handshake_start_time = 0; 3057 WG_TRACE("give up"); 3058 3059 /* 3060 * If a new data packet comes, handshaking will be retried 3061 * and a new session would be established at that time, 3062 * however we don't want to send pending packets then. 3063 */ 3064 wg_purge_pending_packets(wgp); 3065 return; 3066 } 3067 3068 wg_task_send_init_message(wg, wgp); 3069 } 3070 3071 static void 3072 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3073 { 3074 struct wg_session *wgs, *wgs_prev; 3075 struct mbuf *m; 3076 3077 KASSERT(mutex_owned(wgp->wgp_lock)); 3078 3079 wgs = wgp->wgp_session_unstable; 3080 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3081 /* XXX Can this happen? */ 3082 return; 3083 3084 wgs->wgs_state = WGS_STATE_ESTABLISHED; 3085 wgs->wgs_time_established = time_uptime; 3086 wgs->wgs_time_last_data_sent = 0; 3087 wgs->wgs_is_initiator = false; 3088 WG_TRACE("WGS_STATE_ESTABLISHED"); 3089 3090 wg_swap_sessions(wgp); 3091 KASSERT(wgs == wgp->wgp_session_stable); 3092 wgs_prev = wgp->wgp_session_unstable; 3093 getnanotime(&wgp->wgp_last_handshake_time); 3094 wgp->wgp_handshake_start_time = 0; 3095 wgp->wgp_last_sent_mac1_valid = false; 3096 wgp->wgp_last_sent_cookie_valid = false; 3097 3098 /* If we had a data packet queued up, send it. */ 3099 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3100 kpreempt_disable(); 3101 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3102 M_SETCTX(m, wgp); 3103 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3104 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3105 if_name(&wg->wg_if)); 3106 m_freem(m); 3107 } 3108 kpreempt_enable(); 3109 } 3110 3111 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3112 /* Wait for wg_get_stable_session to drain. */ 3113 pserialize_perform(wgp->wgp_psz); 3114 3115 /* Transition ESTABLISHED->DESTROYING. */ 3116 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 3117 3118 /* We can't destroy the old session immediately */ 3119 wg_schedule_session_dtor_timer(wgp); 3120 } else { 3121 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3122 "state=%d", wgs_prev->wgs_state); 3123 wg_clear_states(wgs_prev); 3124 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3125 } 3126 } 3127 3128 static void 3129 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3130 { 3131 3132 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3133 3134 KASSERT(mutex_owned(wgp->wgp_lock)); 3135 3136 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3137 pserialize_perform(wgp->wgp_psz); 3138 mutex_exit(wgp->wgp_lock); 3139 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3140 wg_psref_class); 3141 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3142 wg_psref_class); 3143 mutex_enter(wgp->wgp_lock); 3144 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3145 } 3146 } 3147 3148 static void 3149 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3150 { 3151 struct wg_session *wgs; 3152 3153 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3154 3155 KASSERT(mutex_owned(wgp->wgp_lock)); 3156 3157 wgs = wgp->wgp_session_stable; 3158 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3159 return; 3160 3161 wg_send_keepalive_msg(wgp, wgs); 3162 } 3163 3164 static void 3165 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3166 { 3167 struct wg_session *wgs; 3168 3169 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3170 3171 KASSERT(mutex_owned(wgp->wgp_lock)); 3172 3173 wgs = wgp->wgp_session_unstable; 3174 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 3175 wg_put_session_index(wg, wgs); 3176 } 3177 } 3178 3179 static void 3180 wg_peer_work(struct work *wk, void *cookie) 3181 { 3182 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3183 struct wg_softc *wg = wgp->wgp_sc; 3184 unsigned int tasks; 3185 3186 mutex_enter(wgp->wgp_intr_lock); 3187 while ((tasks = wgp->wgp_tasks) != 0) { 3188 wgp->wgp_tasks = 0; 3189 mutex_exit(wgp->wgp_intr_lock); 3190 3191 mutex_enter(wgp->wgp_lock); 3192 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3193 wg_task_send_init_message(wg, wgp); 3194 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3195 wg_task_retry_handshake(wg, wgp); 3196 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3197 wg_task_establish_session(wg, wgp); 3198 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3199 wg_task_endpoint_changed(wg, wgp); 3200 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3201 wg_task_send_keepalive_message(wg, wgp); 3202 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3203 wg_task_destroy_prev_session(wg, wgp); 3204 mutex_exit(wgp->wgp_lock); 3205 3206 mutex_enter(wgp->wgp_intr_lock); 3207 } 3208 mutex_exit(wgp->wgp_intr_lock); 3209 } 3210 3211 static void 3212 wg_job(struct threadpool_job *job) 3213 { 3214 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3215 int bound, upcalls; 3216 3217 mutex_enter(wg->wg_intr_lock); 3218 while ((upcalls = wg->wg_upcalls) != 0) { 3219 wg->wg_upcalls = 0; 3220 mutex_exit(wg->wg_intr_lock); 3221 bound = curlwp_bind(); 3222 if (ISSET(upcalls, WG_UPCALL_INET)) 3223 wg_receive_packets(wg, AF_INET); 3224 if (ISSET(upcalls, WG_UPCALL_INET6)) 3225 wg_receive_packets(wg, AF_INET6); 3226 curlwp_bindx(bound); 3227 mutex_enter(wg->wg_intr_lock); 3228 } 3229 threadpool_job_done(job); 3230 mutex_exit(wg->wg_intr_lock); 3231 } 3232 3233 static int 3234 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3235 { 3236 int error; 3237 uint16_t old_port = wg->wg_listen_port; 3238 3239 if (port != 0 && old_port == port) 3240 return 0; 3241 3242 struct sockaddr_in _sin, *sin = &_sin; 3243 sin->sin_len = sizeof(*sin); 3244 sin->sin_family = AF_INET; 3245 sin->sin_addr.s_addr = INADDR_ANY; 3246 sin->sin_port = htons(port); 3247 3248 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3249 if (error != 0) 3250 return error; 3251 3252 #ifdef INET6 3253 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3254 sin6->sin6_len = sizeof(*sin6); 3255 sin6->sin6_family = AF_INET6; 3256 sin6->sin6_addr = in6addr_any; 3257 sin6->sin6_port = htons(port); 3258 3259 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3260 if (error != 0) 3261 return error; 3262 #endif 3263 3264 wg->wg_listen_port = port; 3265 3266 return 0; 3267 } 3268 3269 static void 3270 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3271 { 3272 struct wg_softc *wg = cookie; 3273 int reason; 3274 3275 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3276 WG_UPCALL_INET : 3277 WG_UPCALL_INET6; 3278 3279 mutex_enter(wg->wg_intr_lock); 3280 wg->wg_upcalls |= reason; 3281 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3282 mutex_exit(wg->wg_intr_lock); 3283 } 3284 3285 static int 3286 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3287 struct sockaddr *src, void *arg) 3288 { 3289 struct wg_softc *wg = arg; 3290 struct wg_msg wgm; 3291 struct mbuf *m = *mp; 3292 3293 WG_TRACE("enter"); 3294 3295 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3296 KASSERT(offset <= m_length(m)); 3297 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3298 /* drop on the floor */ 3299 m_freem(m); 3300 return -1; 3301 } 3302 3303 /* 3304 * Copy the message header (32-bit message type) out -- we'll 3305 * worry about contiguity and alignment later. 3306 */ 3307 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3308 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3309 3310 /* 3311 * Handle DATA packets promptly as they arrive. Other packets 3312 * may require expensive public-key crypto and are not as 3313 * sensitive to latency, so defer them to the worker thread. 3314 */ 3315 switch (le32toh(wgm.wgm_type)) { 3316 case WG_MSG_TYPE_DATA: 3317 /* handle immediately */ 3318 m_adj(m, offset); 3319 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3320 m = m_pullup(m, sizeof(struct wg_msg_data)); 3321 if (m == NULL) 3322 return -1; 3323 } 3324 wg_handle_msg_data(wg, m, src); 3325 *mp = NULL; 3326 return 1; 3327 case WG_MSG_TYPE_INIT: 3328 case WG_MSG_TYPE_RESP: 3329 case WG_MSG_TYPE_COOKIE: 3330 /* pass through to so_receive in wg_receive_packets */ 3331 return 0; 3332 default: 3333 /* drop on the floor */ 3334 m_freem(m); 3335 return -1; 3336 } 3337 } 3338 3339 static int 3340 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3341 { 3342 int error; 3343 struct socket *so; 3344 3345 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3346 if (error != 0) 3347 return error; 3348 3349 solock(so); 3350 so->so_upcallarg = wg; 3351 so->so_upcall = wg_so_upcall; 3352 so->so_rcv.sb_flags |= SB_UPCALL; 3353 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3354 sounlock(so); 3355 3356 *sop = so; 3357 3358 return 0; 3359 } 3360 3361 static bool 3362 wg_session_hit_limits(struct wg_session *wgs) 3363 { 3364 3365 /* 3366 * [W] 6.2: Transport Message Limits 3367 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3368 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3369 * comes first, WireGuard will refuse to send any more transport data 3370 * messages using the current secure session, ..." 3371 */ 3372 KASSERT(wgs->wgs_time_established != 0); 3373 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3374 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3375 return true; 3376 } else if (wg_session_get_send_counter(wgs) > 3377 wg_reject_after_messages) { 3378 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3379 return true; 3380 } 3381 3382 return false; 3383 } 3384 3385 static void 3386 wgintr(void *cookie) 3387 { 3388 struct wg_peer *wgp; 3389 struct wg_session *wgs; 3390 struct mbuf *m; 3391 struct psref psref; 3392 3393 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3394 wgp = M_GETCTX(m, struct wg_peer *); 3395 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3396 WG_TRACE("no stable session"); 3397 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3398 goto next0; 3399 } 3400 if (__predict_false(wg_session_hit_limits(wgs))) { 3401 WG_TRACE("stable session hit limits"); 3402 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3403 goto next1; 3404 } 3405 wg_send_data_msg(wgp, wgs, m); 3406 m = NULL; /* consumed */ 3407 next1: wg_put_session(wgs, &psref); 3408 next0: m_freem(m); 3409 /* XXX Yield to avoid userland starvation? */ 3410 } 3411 } 3412 3413 static void 3414 wg_rekey_timer(void *arg) 3415 { 3416 struct wg_peer *wgp = arg; 3417 3418 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3419 } 3420 3421 static void 3422 wg_purge_pending_packets(struct wg_peer *wgp) 3423 { 3424 struct mbuf *m; 3425 3426 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 3427 m_freem(m); 3428 pktq_barrier(wg_pktq); 3429 } 3430 3431 static void 3432 wg_handshake_timeout_timer(void *arg) 3433 { 3434 struct wg_peer *wgp = arg; 3435 3436 WG_TRACE("enter"); 3437 3438 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3439 } 3440 3441 static struct wg_peer * 3442 wg_alloc_peer(struct wg_softc *wg) 3443 { 3444 struct wg_peer *wgp; 3445 3446 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3447 3448 wgp->wgp_sc = wg; 3449 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3450 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3451 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3452 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3453 wg_handshake_timeout_timer, wgp); 3454 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3455 callout_setfunc(&wgp->wgp_session_dtor_timer, 3456 wg_session_dtor_timer, wgp); 3457 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3458 wgp->wgp_endpoint_changing = false; 3459 wgp->wgp_endpoint_available = false; 3460 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3461 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3462 wgp->wgp_psz = pserialize_create(); 3463 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3464 3465 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3466 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3467 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3468 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3469 3470 struct wg_session *wgs; 3471 wgp->wgp_session_stable = 3472 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3473 wgp->wgp_session_unstable = 3474 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3475 wgs = wgp->wgp_session_stable; 3476 wgs->wgs_peer = wgp; 3477 wgs->wgs_state = WGS_STATE_UNKNOWN; 3478 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3479 #ifndef __HAVE_ATOMIC64_LOADSTORE 3480 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3481 #endif 3482 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3483 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3484 3485 wgs = wgp->wgp_session_unstable; 3486 wgs->wgs_peer = wgp; 3487 wgs->wgs_state = WGS_STATE_UNKNOWN; 3488 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3489 #ifndef __HAVE_ATOMIC64_LOADSTORE 3490 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3491 #endif 3492 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3493 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3494 3495 return wgp; 3496 } 3497 3498 static void 3499 wg_destroy_peer(struct wg_peer *wgp) 3500 { 3501 struct wg_session *wgs; 3502 struct wg_softc *wg = wgp->wgp_sc; 3503 3504 /* Prevent new packets from this peer on any source address. */ 3505 rw_enter(wg->wg_rwlock, RW_WRITER); 3506 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3507 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3508 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3509 struct radix_node *rn; 3510 3511 KASSERT(rnh != NULL); 3512 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3513 &wga->wga_sa_mask, rnh); 3514 if (rn == NULL) { 3515 char addrstr[128]; 3516 sockaddr_format(&wga->wga_sa_addr, addrstr, 3517 sizeof(addrstr)); 3518 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3519 if_name(&wg->wg_if), addrstr); 3520 } 3521 } 3522 rw_exit(wg->wg_rwlock); 3523 3524 /* Purge pending packets. */ 3525 wg_purge_pending_packets(wgp); 3526 3527 /* Halt all packet processing and timeouts. */ 3528 callout_halt(&wgp->wgp_rekey_timer, NULL); 3529 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3530 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3531 3532 /* Wait for any queued work to complete. */ 3533 workqueue_wait(wg_wq, &wgp->wgp_work); 3534 3535 wgs = wgp->wgp_session_unstable; 3536 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3537 mutex_enter(wgp->wgp_lock); 3538 wg_destroy_session(wg, wgs); 3539 mutex_exit(wgp->wgp_lock); 3540 } 3541 mutex_destroy(&wgs->wgs_recvwin->lock); 3542 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3543 #ifndef __HAVE_ATOMIC64_LOADSTORE 3544 mutex_destroy(&wgs->wgs_send_counter_lock); 3545 #endif 3546 kmem_free(wgs, sizeof(*wgs)); 3547 3548 wgs = wgp->wgp_session_stable; 3549 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3550 mutex_enter(wgp->wgp_lock); 3551 wg_destroy_session(wg, wgs); 3552 mutex_exit(wgp->wgp_lock); 3553 } 3554 mutex_destroy(&wgs->wgs_recvwin->lock); 3555 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3556 #ifndef __HAVE_ATOMIC64_LOADSTORE 3557 mutex_destroy(&wgs->wgs_send_counter_lock); 3558 #endif 3559 kmem_free(wgs, sizeof(*wgs)); 3560 3561 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3562 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3563 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3564 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3565 3566 pserialize_destroy(wgp->wgp_psz); 3567 mutex_obj_free(wgp->wgp_intr_lock); 3568 mutex_obj_free(wgp->wgp_lock); 3569 3570 kmem_free(wgp, sizeof(*wgp)); 3571 } 3572 3573 static void 3574 wg_destroy_all_peers(struct wg_softc *wg) 3575 { 3576 struct wg_peer *wgp, *wgp0 __diagused; 3577 void *garbage_byname, *garbage_bypubkey; 3578 3579 restart: 3580 garbage_byname = garbage_bypubkey = NULL; 3581 mutex_enter(wg->wg_lock); 3582 WG_PEER_WRITER_FOREACH(wgp, wg) { 3583 if (wgp->wgp_name[0]) { 3584 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3585 strlen(wgp->wgp_name)); 3586 KASSERT(wgp0 == wgp); 3587 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3588 } 3589 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3590 sizeof(wgp->wgp_pubkey)); 3591 KASSERT(wgp0 == wgp); 3592 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3593 WG_PEER_WRITER_REMOVE(wgp); 3594 wg->wg_npeers--; 3595 mutex_enter(wgp->wgp_lock); 3596 pserialize_perform(wgp->wgp_psz); 3597 mutex_exit(wgp->wgp_lock); 3598 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3599 break; 3600 } 3601 mutex_exit(wg->wg_lock); 3602 3603 if (wgp == NULL) 3604 return; 3605 3606 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3607 3608 wg_destroy_peer(wgp); 3609 thmap_gc(wg->wg_peers_byname, garbage_byname); 3610 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3611 3612 goto restart; 3613 } 3614 3615 static int 3616 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3617 { 3618 struct wg_peer *wgp, *wgp0 __diagused; 3619 void *garbage_byname, *garbage_bypubkey; 3620 3621 mutex_enter(wg->wg_lock); 3622 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3623 if (wgp != NULL) { 3624 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3625 sizeof(wgp->wgp_pubkey)); 3626 KASSERT(wgp0 == wgp); 3627 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3628 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3629 WG_PEER_WRITER_REMOVE(wgp); 3630 wg->wg_npeers--; 3631 if (wg->wg_npeers == 0) 3632 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3633 mutex_enter(wgp->wgp_lock); 3634 pserialize_perform(wgp->wgp_psz); 3635 mutex_exit(wgp->wgp_lock); 3636 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3637 } 3638 mutex_exit(wg->wg_lock); 3639 3640 if (wgp == NULL) 3641 return ENOENT; 3642 3643 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3644 3645 wg_destroy_peer(wgp); 3646 thmap_gc(wg->wg_peers_byname, garbage_byname); 3647 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3648 3649 return 0; 3650 } 3651 3652 static int 3653 wg_if_attach(struct wg_softc *wg) 3654 { 3655 3656 wg->wg_if.if_addrlen = 0; 3657 wg->wg_if.if_mtu = WG_MTU; 3658 wg->wg_if.if_flags = IFF_MULTICAST; 3659 wg->wg_if.if_extflags = IFEF_MPSAFE; 3660 wg->wg_if.if_ioctl = wg_ioctl; 3661 wg->wg_if.if_output = wg_output; 3662 wg->wg_if.if_init = wg_init; 3663 #ifdef ALTQ 3664 wg->wg_if.if_start = wg_start; 3665 #endif 3666 wg->wg_if.if_stop = wg_stop; 3667 wg->wg_if.if_type = IFT_OTHER; 3668 wg->wg_if.if_dlt = DLT_NULL; 3669 wg->wg_if.if_softc = wg; 3670 #ifdef ALTQ 3671 IFQ_SET_READY(&wg->wg_if.if_snd); 3672 #endif 3673 if_initialize(&wg->wg_if); 3674 3675 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3676 if_alloc_sadl(&wg->wg_if); 3677 if_register(&wg->wg_if); 3678 3679 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3680 3681 return 0; 3682 } 3683 3684 static void 3685 wg_if_detach(struct wg_softc *wg) 3686 { 3687 struct ifnet *ifp = &wg->wg_if; 3688 3689 bpf_detach(ifp); 3690 if_detach(ifp); 3691 } 3692 3693 static int 3694 wg_clone_create(struct if_clone *ifc, int unit) 3695 { 3696 struct wg_softc *wg; 3697 int error; 3698 3699 wg_guarantee_initialized(); 3700 3701 error = wg_count_inc(); 3702 if (error) 3703 return error; 3704 3705 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3706 3707 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3708 3709 PSLIST_INIT(&wg->wg_peers); 3710 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3711 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3712 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3713 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3714 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3715 wg->wg_rwlock = rw_obj_alloc(); 3716 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3717 "%s", if_name(&wg->wg_if)); 3718 wg->wg_ops = &wg_ops_rumpkernel; 3719 3720 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3721 if (error) 3722 goto fail0; 3723 3724 #ifdef INET 3725 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3726 if (error) 3727 goto fail1; 3728 rn_inithead((void **)&wg->wg_rtable_ipv4, 3729 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3730 #endif 3731 #ifdef INET6 3732 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3733 if (error) 3734 goto fail2; 3735 rn_inithead((void **)&wg->wg_rtable_ipv6, 3736 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3737 #endif 3738 3739 error = wg_if_attach(wg); 3740 if (error) 3741 goto fail3; 3742 3743 return 0; 3744 3745 fail4: __unused 3746 wg_if_detach(wg); 3747 fail3: wg_destroy_all_peers(wg); 3748 #ifdef INET6 3749 solock(wg->wg_so6); 3750 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3751 sounlock(wg->wg_so6); 3752 #endif 3753 #ifdef INET 3754 solock(wg->wg_so4); 3755 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3756 sounlock(wg->wg_so4); 3757 #endif 3758 mutex_enter(wg->wg_intr_lock); 3759 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3760 mutex_exit(wg->wg_intr_lock); 3761 #ifdef INET6 3762 if (wg->wg_rtable_ipv6 != NULL) 3763 free(wg->wg_rtable_ipv6, M_RTABLE); 3764 soclose(wg->wg_so6); 3765 fail2: 3766 #endif 3767 #ifdef INET 3768 if (wg->wg_rtable_ipv4 != NULL) 3769 free(wg->wg_rtable_ipv4, M_RTABLE); 3770 soclose(wg->wg_so4); 3771 fail1: 3772 #endif 3773 threadpool_put(wg->wg_threadpool, PRI_NONE); 3774 fail0: threadpool_job_destroy(&wg->wg_job); 3775 rw_obj_free(wg->wg_rwlock); 3776 mutex_obj_free(wg->wg_intr_lock); 3777 mutex_obj_free(wg->wg_lock); 3778 thmap_destroy(wg->wg_sessions_byindex); 3779 thmap_destroy(wg->wg_peers_byname); 3780 thmap_destroy(wg->wg_peers_bypubkey); 3781 PSLIST_DESTROY(&wg->wg_peers); 3782 kmem_free(wg, sizeof(*wg)); 3783 wg_count_dec(); 3784 return error; 3785 } 3786 3787 static int 3788 wg_clone_destroy(struct ifnet *ifp) 3789 { 3790 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3791 3792 #ifdef WG_RUMPKERNEL 3793 if (wg_user_mode(wg)) { 3794 rumpuser_wg_destroy(wg->wg_user); 3795 wg->wg_user = NULL; 3796 } 3797 #endif 3798 3799 wg_if_detach(wg); 3800 wg_destroy_all_peers(wg); 3801 #ifdef INET6 3802 solock(wg->wg_so6); 3803 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3804 sounlock(wg->wg_so6); 3805 #endif 3806 #ifdef INET 3807 solock(wg->wg_so4); 3808 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3809 sounlock(wg->wg_so4); 3810 #endif 3811 mutex_enter(wg->wg_intr_lock); 3812 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3813 mutex_exit(wg->wg_intr_lock); 3814 #ifdef INET6 3815 if (wg->wg_rtable_ipv6 != NULL) 3816 free(wg->wg_rtable_ipv6, M_RTABLE); 3817 soclose(wg->wg_so6); 3818 #endif 3819 #ifdef INET 3820 if (wg->wg_rtable_ipv4 != NULL) 3821 free(wg->wg_rtable_ipv4, M_RTABLE); 3822 soclose(wg->wg_so4); 3823 #endif 3824 threadpool_put(wg->wg_threadpool, PRI_NONE); 3825 threadpool_job_destroy(&wg->wg_job); 3826 rw_obj_free(wg->wg_rwlock); 3827 mutex_obj_free(wg->wg_intr_lock); 3828 mutex_obj_free(wg->wg_lock); 3829 thmap_destroy(wg->wg_sessions_byindex); 3830 thmap_destroy(wg->wg_peers_byname); 3831 thmap_destroy(wg->wg_peers_bypubkey); 3832 PSLIST_DESTROY(&wg->wg_peers); 3833 kmem_free(wg, sizeof(*wg)); 3834 wg_count_dec(); 3835 3836 return 0; 3837 } 3838 3839 static struct wg_peer * 3840 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3841 struct psref *psref) 3842 { 3843 struct radix_node_head *rnh; 3844 struct radix_node *rn; 3845 struct wg_peer *wgp = NULL; 3846 struct wg_allowedip *wga; 3847 3848 #ifdef WG_DEBUG_LOG 3849 char addrstr[128]; 3850 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3851 WG_DLOG("sa=%s\n", addrstr); 3852 #endif 3853 3854 rw_enter(wg->wg_rwlock, RW_READER); 3855 3856 rnh = wg_rnh(wg, sa->sa_family); 3857 if (rnh == NULL) 3858 goto out; 3859 3860 rn = rnh->rnh_matchaddr(sa, rnh); 3861 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3862 goto out; 3863 3864 WG_TRACE("success"); 3865 3866 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3867 wgp = wga->wga_peer; 3868 wg_get_peer(wgp, psref); 3869 3870 out: 3871 rw_exit(wg->wg_rwlock); 3872 return wgp; 3873 } 3874 3875 static void 3876 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3877 struct wg_session *wgs, struct wg_msg_data *wgmd) 3878 { 3879 3880 memset(wgmd, 0, sizeof(*wgmd)); 3881 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3882 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3883 /* [W] 5.4.6: msg.counter := Nm^send */ 3884 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3885 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3886 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3887 } 3888 3889 static int 3890 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3891 const struct rtentry *rt) 3892 { 3893 struct wg_softc *wg = ifp->if_softc; 3894 struct wg_peer *wgp = NULL; 3895 struct wg_session *wgs = NULL; 3896 struct psref wgp_psref, wgs_psref; 3897 int bound; 3898 int error; 3899 3900 bound = curlwp_bind(); 3901 3902 /* TODO make the nest limit configurable via sysctl */ 3903 error = if_tunnel_check_nesting(ifp, m, 1); 3904 if (error) { 3905 WGLOG(LOG_ERR, 3906 "%s: tunneling loop detected and packet dropped\n", 3907 if_name(&wg->wg_if)); 3908 goto out0; 3909 } 3910 3911 #ifdef ALTQ 3912 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 3913 & ALTQF_ENABLED; 3914 if (altq) 3915 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3916 #endif 3917 3918 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3919 3920 m->m_flags &= ~(M_BCAST|M_MCAST); 3921 3922 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3923 if (wgp == NULL) { 3924 WG_TRACE("peer not found"); 3925 error = EHOSTUNREACH; 3926 goto out0; 3927 } 3928 3929 /* Clear checksum-offload flags. */ 3930 m->m_pkthdr.csum_flags = 0; 3931 m->m_pkthdr.csum_data = 0; 3932 3933 /* Check whether there's an established session. */ 3934 wgs = wg_get_stable_session(wgp, &wgs_psref); 3935 if (wgs == NULL) { 3936 /* 3937 * No established session. If we're the first to try 3938 * sending data, schedule a handshake and queue the 3939 * packet for when the handshake is done; otherwise 3940 * just drop the packet and let the ongoing handshake 3941 * attempt continue. We could queue more data packets 3942 * but it's not clear that's worthwhile. 3943 */ 3944 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 3945 m = NULL; /* consume */ 3946 WG_TRACE("queued first packet; init handshake"); 3947 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3948 } else { 3949 WG_TRACE("first packet already queued, dropping"); 3950 } 3951 goto out1; 3952 } 3953 3954 /* There's an established session. Toss it in the queue. */ 3955 #ifdef ALTQ 3956 if (altq) { 3957 mutex_enter(ifp->if_snd.ifq_lock); 3958 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 3959 M_SETCTX(m, wgp); 3960 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 3961 m = NULL; /* consume */ 3962 } 3963 mutex_exit(ifp->if_snd.ifq_lock); 3964 if (m == NULL) { 3965 wg_start(ifp); 3966 goto out2; 3967 } 3968 } 3969 #endif 3970 kpreempt_disable(); 3971 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3972 M_SETCTX(m, wgp); 3973 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3974 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3975 if_name(&wg->wg_if)); 3976 error = ENOBUFS; 3977 goto out3; 3978 } 3979 m = NULL; /* consumed */ 3980 error = 0; 3981 out3: kpreempt_enable(); 3982 3983 #ifdef ALTQ 3984 out2: 3985 #endif 3986 wg_put_session(wgs, &wgs_psref); 3987 out1: wg_put_peer(wgp, &wgp_psref); 3988 out0: m_freem(m); 3989 curlwp_bindx(bound); 3990 return error; 3991 } 3992 3993 static int 3994 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3995 { 3996 struct psref psref; 3997 struct wg_sockaddr *wgsa; 3998 int error; 3999 struct socket *so; 4000 4001 wgsa = wg_get_endpoint_sa(wgp, &psref); 4002 so = wg_get_so_by_peer(wgp, wgsa); 4003 solock(so); 4004 if (wgsatosa(wgsa)->sa_family == AF_INET) { 4005 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 4006 } else { 4007 #ifdef INET6 4008 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 4009 NULL, curlwp); 4010 #else 4011 m_freem(m); 4012 error = EPFNOSUPPORT; 4013 #endif 4014 } 4015 sounlock(so); 4016 wg_put_sa(wgp, wgsa, &psref); 4017 4018 return error; 4019 } 4020 4021 /* Inspired by pppoe_get_mbuf */ 4022 static struct mbuf * 4023 wg_get_mbuf(size_t leading_len, size_t len) 4024 { 4025 struct mbuf *m; 4026 4027 KASSERT(leading_len <= MCLBYTES); 4028 KASSERT(len <= MCLBYTES - leading_len); 4029 4030 m = m_gethdr(M_DONTWAIT, MT_DATA); 4031 if (m == NULL) 4032 return NULL; 4033 if (len + leading_len > MHLEN) { 4034 m_clget(m, M_DONTWAIT); 4035 if ((m->m_flags & M_EXT) == 0) { 4036 m_free(m); 4037 return NULL; 4038 } 4039 } 4040 m->m_data += leading_len; 4041 m->m_pkthdr.len = m->m_len = len; 4042 4043 return m; 4044 } 4045 4046 static int 4047 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 4048 struct mbuf *m) 4049 { 4050 struct wg_softc *wg = wgp->wgp_sc; 4051 int error; 4052 size_t inner_len, padded_len, encrypted_len; 4053 char *padded_buf = NULL; 4054 size_t mlen; 4055 struct wg_msg_data *wgmd; 4056 bool free_padded_buf = false; 4057 struct mbuf *n; 4058 size_t leading_len = max_hdr + sizeof(struct udphdr); 4059 4060 mlen = m_length(m); 4061 inner_len = mlen; 4062 padded_len = roundup(mlen, 16); 4063 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4064 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 4065 inner_len, padded_len, encrypted_len); 4066 if (mlen != 0) { 4067 bool success; 4068 success = m_ensure_contig(&m, padded_len); 4069 if (success) { 4070 padded_buf = mtod(m, char *); 4071 } else { 4072 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4073 if (padded_buf == NULL) { 4074 error = ENOBUFS; 4075 goto end; 4076 } 4077 free_padded_buf = true; 4078 m_copydata(m, 0, mlen, padded_buf); 4079 } 4080 memset(padded_buf + mlen, 0, padded_len - inner_len); 4081 } 4082 4083 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4084 if (n == NULL) { 4085 error = ENOBUFS; 4086 goto end; 4087 } 4088 KASSERT(n->m_len >= sizeof(*wgmd)); 4089 wgmd = mtod(n, struct wg_msg_data *); 4090 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4091 #ifdef WG_DEBUG_PACKET 4092 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4093 char *hex = gethexdump(wgmd, sizeof(*wgmd)); 4094 log(LOG_DEBUG, "wgmd=%p, sizeof(*wgmd)=%zu\n%s\n", 4095 wgmd, sizeof(*wgmd), hex); 4096 puthexdump(hex, wgmd, sizeof(*wgmd)); 4097 hex = gethexdump(padded_buf, padded_len); 4098 log(LOG_DEBUG, "padded_buf=%p, padded_len=%zu\n%s\n", 4099 padded_buf, padded_len, hex); 4100 puthexdump(hex, padded_buf, padded_len); 4101 } 4102 #endif 4103 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4104 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4105 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4106 padded_buf, padded_len, 4107 NULL, 0); 4108 4109 error = wg->wg_ops->send_data_msg(wgp, n); 4110 if (error == 0) { 4111 struct ifnet *ifp = &wg->wg_if; 4112 if_statadd(ifp, if_obytes, mlen); 4113 if_statinc(ifp, if_opackets); 4114 if (wgs->wgs_is_initiator && 4115 wgs->wgs_time_last_data_sent == 0) { 4116 /* 4117 * [W] 6.2 Transport Message Limits 4118 * "if a peer is the initiator of a current secure 4119 * session, WireGuard will send a handshake initiation 4120 * message to begin a new secure session if, after 4121 * transmitting a transport data message, the current 4122 * secure session is REKEY-AFTER-TIME seconds old," 4123 */ 4124 wg_schedule_rekey_timer(wgp); 4125 } 4126 wgs->wgs_time_last_data_sent = time_uptime; 4127 if (wg_session_get_send_counter(wgs) >= 4128 wg_rekey_after_messages) { 4129 /* 4130 * [W] 6.2 Transport Message Limits 4131 * "WireGuard will try to create a new session, by 4132 * sending a handshake initiation message (section 4133 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4134 * transport data messages..." 4135 */ 4136 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4137 } 4138 } 4139 end: 4140 m_freem(m); 4141 if (free_padded_buf) 4142 kmem_intr_free(padded_buf, padded_len); 4143 return error; 4144 } 4145 4146 static void 4147 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4148 { 4149 pktqueue_t *pktq; 4150 size_t pktlen; 4151 4152 KASSERT(af == AF_INET || af == AF_INET6); 4153 4154 WG_TRACE(""); 4155 4156 m_set_rcvif(m, ifp); 4157 pktlen = m->m_pkthdr.len; 4158 4159 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4160 4161 switch (af) { 4162 case AF_INET: 4163 pktq = ip_pktq; 4164 break; 4165 #ifdef INET6 4166 case AF_INET6: 4167 pktq = ip6_pktq; 4168 break; 4169 #endif 4170 default: 4171 panic("invalid af=%d", af); 4172 } 4173 4174 kpreempt_disable(); 4175 const u_int h = curcpu()->ci_index; 4176 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4177 if_statadd(ifp, if_ibytes, pktlen); 4178 if_statinc(ifp, if_ipackets); 4179 } else { 4180 m_freem(m); 4181 } 4182 kpreempt_enable(); 4183 } 4184 4185 static void 4186 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4187 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4188 { 4189 4190 crypto_scalarmult_base(pubkey, privkey); 4191 } 4192 4193 static int 4194 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4195 { 4196 struct radix_node_head *rnh; 4197 struct radix_node *rn; 4198 int error = 0; 4199 4200 rw_enter(wg->wg_rwlock, RW_WRITER); 4201 rnh = wg_rnh(wg, wga->wga_family); 4202 KASSERT(rnh != NULL); 4203 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4204 wga->wga_nodes); 4205 rw_exit(wg->wg_rwlock); 4206 4207 if (rn == NULL) 4208 error = EEXIST; 4209 4210 return error; 4211 } 4212 4213 static int 4214 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4215 struct wg_peer **wgpp) 4216 { 4217 int error = 0; 4218 const void *pubkey; 4219 size_t pubkey_len; 4220 const void *psk; 4221 size_t psk_len; 4222 const char *name = NULL; 4223 4224 if (prop_dictionary_get_string(peer, "name", &name)) { 4225 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4226 error = EINVAL; 4227 goto out; 4228 } 4229 } 4230 4231 if (!prop_dictionary_get_data(peer, "public_key", 4232 &pubkey, &pubkey_len)) { 4233 error = EINVAL; 4234 goto out; 4235 } 4236 #ifdef WG_DEBUG_DUMP 4237 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4238 char *hex = gethexdump(pubkey, pubkey_len); 4239 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n%s\n", 4240 pubkey, pubkey_len, hex); 4241 puthexdump(hex, pubkey, pubkey_len); 4242 } 4243 #endif 4244 4245 struct wg_peer *wgp = wg_alloc_peer(wg); 4246 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4247 if (name != NULL) 4248 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4249 4250 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4251 if (psk_len != sizeof(wgp->wgp_psk)) { 4252 error = EINVAL; 4253 goto out; 4254 } 4255 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4256 } 4257 4258 const void *addr; 4259 size_t addr_len; 4260 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4261 4262 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4263 goto skip_endpoint; 4264 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4265 addr_len > sizeof(*wgsatoss(wgsa))) { 4266 error = EINVAL; 4267 goto out; 4268 } 4269 memcpy(wgsatoss(wgsa), addr, addr_len); 4270 switch (wgsa_family(wgsa)) { 4271 case AF_INET: 4272 #ifdef INET6 4273 case AF_INET6: 4274 #endif 4275 break; 4276 default: 4277 error = EPFNOSUPPORT; 4278 goto out; 4279 } 4280 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4281 error = EINVAL; 4282 goto out; 4283 } 4284 { 4285 char addrstr[128]; 4286 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4287 WG_DLOG("addr=%s\n", addrstr); 4288 } 4289 wgp->wgp_endpoint_available = true; 4290 4291 prop_array_t allowedips; 4292 skip_endpoint: 4293 allowedips = prop_dictionary_get(peer, "allowedips"); 4294 if (allowedips == NULL) 4295 goto skip; 4296 4297 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4298 prop_dictionary_t prop_allowedip; 4299 int j = 0; 4300 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4301 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4302 4303 if (!prop_dictionary_get_int(prop_allowedip, "family", 4304 &wga->wga_family)) 4305 continue; 4306 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4307 &addr, &addr_len)) 4308 continue; 4309 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4310 &wga->wga_cidr)) 4311 continue; 4312 4313 switch (wga->wga_family) { 4314 case AF_INET: { 4315 struct sockaddr_in sin; 4316 char addrstr[128]; 4317 struct in_addr mask; 4318 struct sockaddr_in sin_mask; 4319 4320 if (addr_len != sizeof(struct in_addr)) 4321 return EINVAL; 4322 memcpy(&wga->wga_addr4, addr, addr_len); 4323 4324 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4325 0); 4326 sockaddr_copy(&wga->wga_sa_addr, 4327 sizeof(sin), sintosa(&sin)); 4328 4329 sockaddr_format(sintosa(&sin), 4330 addrstr, sizeof(addrstr)); 4331 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4332 4333 in_len2mask(&mask, wga->wga_cidr); 4334 sockaddr_in_init(&sin_mask, &mask, 0); 4335 sockaddr_copy(&wga->wga_sa_mask, 4336 sizeof(sin_mask), sintosa(&sin_mask)); 4337 4338 break; 4339 } 4340 #ifdef INET6 4341 case AF_INET6: { 4342 struct sockaddr_in6 sin6; 4343 char addrstr[128]; 4344 struct in6_addr mask; 4345 struct sockaddr_in6 sin6_mask; 4346 4347 if (addr_len != sizeof(struct in6_addr)) 4348 return EINVAL; 4349 memcpy(&wga->wga_addr6, addr, addr_len); 4350 4351 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4352 0, 0, 0); 4353 sockaddr_copy(&wga->wga_sa_addr, 4354 sizeof(sin6), sin6tosa(&sin6)); 4355 4356 sockaddr_format(sin6tosa(&sin6), 4357 addrstr, sizeof(addrstr)); 4358 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4359 4360 in6_prefixlen2mask(&mask, wga->wga_cidr); 4361 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4362 sockaddr_copy(&wga->wga_sa_mask, 4363 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4364 4365 break; 4366 } 4367 #endif 4368 default: 4369 error = EINVAL; 4370 goto out; 4371 } 4372 wga->wga_peer = wgp; 4373 4374 error = wg_rtable_add_route(wg, wga); 4375 if (error != 0) 4376 goto out; 4377 4378 j++; 4379 } 4380 wgp->wgp_n_allowedips = j; 4381 skip: 4382 *wgpp = wgp; 4383 out: 4384 return error; 4385 } 4386 4387 static int 4388 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4389 { 4390 int error; 4391 char *buf; 4392 4393 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4394 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4395 return E2BIG; 4396 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4397 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4398 if (error != 0) 4399 return error; 4400 buf[ifd->ifd_len] = '\0'; 4401 #ifdef WG_DEBUG_DUMP 4402 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4403 log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len), 4404 (const char *)buf); 4405 } 4406 #endif 4407 *_buf = buf; 4408 return 0; 4409 } 4410 4411 static int 4412 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4413 { 4414 int error; 4415 prop_dictionary_t prop_dict; 4416 char *buf = NULL; 4417 const void *privkey; 4418 size_t privkey_len; 4419 4420 error = wg_alloc_prop_buf(&buf, ifd); 4421 if (error != 0) 4422 return error; 4423 error = EINVAL; 4424 prop_dict = prop_dictionary_internalize(buf); 4425 if (prop_dict == NULL) 4426 goto out; 4427 if (!prop_dictionary_get_data(prop_dict, "private_key", 4428 &privkey, &privkey_len)) 4429 goto out; 4430 #ifdef WG_DEBUG_DUMP 4431 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4432 char *hex = gethexdump(privkey, privkey_len); 4433 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n%s\n", 4434 privkey, privkey_len, hex); 4435 puthexdump(hex, privkey, privkey_len); 4436 } 4437 #endif 4438 if (privkey_len != WG_STATIC_KEY_LEN) 4439 goto out; 4440 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4441 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4442 error = 0; 4443 4444 out: 4445 kmem_free(buf, ifd->ifd_len + 1); 4446 return error; 4447 } 4448 4449 static int 4450 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4451 { 4452 int error; 4453 prop_dictionary_t prop_dict; 4454 char *buf = NULL; 4455 uint16_t port; 4456 4457 error = wg_alloc_prop_buf(&buf, ifd); 4458 if (error != 0) 4459 return error; 4460 error = EINVAL; 4461 prop_dict = prop_dictionary_internalize(buf); 4462 if (prop_dict == NULL) 4463 goto out; 4464 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4465 goto out; 4466 4467 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4468 4469 out: 4470 kmem_free(buf, ifd->ifd_len + 1); 4471 return error; 4472 } 4473 4474 static int 4475 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4476 { 4477 int error; 4478 prop_dictionary_t prop_dict; 4479 char *buf = NULL; 4480 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4481 4482 error = wg_alloc_prop_buf(&buf, ifd); 4483 if (error != 0) 4484 return error; 4485 error = EINVAL; 4486 prop_dict = prop_dictionary_internalize(buf); 4487 if (prop_dict == NULL) 4488 goto out; 4489 4490 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4491 if (error != 0) 4492 goto out; 4493 4494 mutex_enter(wg->wg_lock); 4495 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4496 sizeof(wgp->wgp_pubkey)) != NULL || 4497 (wgp->wgp_name[0] && 4498 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4499 strlen(wgp->wgp_name)) != NULL)) { 4500 mutex_exit(wg->wg_lock); 4501 wg_destroy_peer(wgp); 4502 error = EEXIST; 4503 goto out; 4504 } 4505 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4506 sizeof(wgp->wgp_pubkey), wgp); 4507 KASSERT(wgp0 == wgp); 4508 if (wgp->wgp_name[0]) { 4509 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4510 strlen(wgp->wgp_name), wgp); 4511 KASSERT(wgp0 == wgp); 4512 } 4513 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4514 wg->wg_npeers++; 4515 mutex_exit(wg->wg_lock); 4516 4517 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4518 4519 out: 4520 kmem_free(buf, ifd->ifd_len + 1); 4521 return error; 4522 } 4523 4524 static int 4525 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4526 { 4527 int error; 4528 prop_dictionary_t prop_dict; 4529 char *buf = NULL; 4530 const char *name; 4531 4532 error = wg_alloc_prop_buf(&buf, ifd); 4533 if (error != 0) 4534 return error; 4535 error = EINVAL; 4536 prop_dict = prop_dictionary_internalize(buf); 4537 if (prop_dict == NULL) 4538 goto out; 4539 4540 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4541 goto out; 4542 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4543 goto out; 4544 4545 error = wg_destroy_peer_name(wg, name); 4546 out: 4547 kmem_free(buf, ifd->ifd_len + 1); 4548 return error; 4549 } 4550 4551 static bool 4552 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4553 { 4554 int au = cmd == SIOCGDRVSPEC ? 4555 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4556 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4557 return kauth_authorize_network(kauth_cred_get(), 4558 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4559 (void *)cmd, NULL) == 0; 4560 } 4561 4562 static int 4563 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4564 { 4565 int error = ENOMEM; 4566 prop_dictionary_t prop_dict; 4567 prop_array_t peers = NULL; 4568 char *buf; 4569 struct wg_peer *wgp; 4570 int s, i; 4571 4572 prop_dict = prop_dictionary_create(); 4573 if (prop_dict == NULL) 4574 goto error; 4575 4576 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4577 if (!prop_dictionary_set_data(prop_dict, "private_key", 4578 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4579 goto error; 4580 } 4581 4582 if (wg->wg_listen_port != 0) { 4583 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4584 wg->wg_listen_port)) 4585 goto error; 4586 } 4587 4588 if (wg->wg_npeers == 0) 4589 goto skip_peers; 4590 4591 peers = prop_array_create(); 4592 if (peers == NULL) 4593 goto error; 4594 4595 s = pserialize_read_enter(); 4596 i = 0; 4597 WG_PEER_READER_FOREACH(wgp, wg) { 4598 struct wg_sockaddr *wgsa; 4599 struct psref wgp_psref, wgsa_psref; 4600 prop_dictionary_t prop_peer; 4601 4602 wg_get_peer(wgp, &wgp_psref); 4603 pserialize_read_exit(s); 4604 4605 prop_peer = prop_dictionary_create(); 4606 if (prop_peer == NULL) 4607 goto next; 4608 4609 if (strlen(wgp->wgp_name) > 0) { 4610 if (!prop_dictionary_set_string(prop_peer, "name", 4611 wgp->wgp_name)) 4612 goto next; 4613 } 4614 4615 if (!prop_dictionary_set_data(prop_peer, "public_key", 4616 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4617 goto next; 4618 4619 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4620 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4621 sizeof(wgp->wgp_psk))) { 4622 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4623 if (!prop_dictionary_set_data(prop_peer, 4624 "preshared_key", 4625 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4626 goto next; 4627 } 4628 } 4629 4630 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4631 CTASSERT(AF_UNSPEC == 0); 4632 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4633 !prop_dictionary_set_data(prop_peer, "endpoint", 4634 wgsatoss(wgsa), 4635 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4636 wg_put_sa(wgp, wgsa, &wgsa_psref); 4637 goto next; 4638 } 4639 wg_put_sa(wgp, wgsa, &wgsa_psref); 4640 4641 const struct timespec *t = &wgp->wgp_last_handshake_time; 4642 4643 if (!prop_dictionary_set_uint64(prop_peer, 4644 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4645 goto next; 4646 if (!prop_dictionary_set_uint32(prop_peer, 4647 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4648 goto next; 4649 4650 if (wgp->wgp_n_allowedips == 0) 4651 goto skip_allowedips; 4652 4653 prop_array_t allowedips = prop_array_create(); 4654 if (allowedips == NULL) 4655 goto next; 4656 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4657 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4658 prop_dictionary_t prop_allowedip; 4659 4660 prop_allowedip = prop_dictionary_create(); 4661 if (prop_allowedip == NULL) 4662 break; 4663 4664 if (!prop_dictionary_set_int(prop_allowedip, "family", 4665 wga->wga_family)) 4666 goto _next; 4667 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4668 wga->wga_cidr)) 4669 goto _next; 4670 4671 switch (wga->wga_family) { 4672 case AF_INET: 4673 if (!prop_dictionary_set_data(prop_allowedip, 4674 "ip", &wga->wga_addr4, 4675 sizeof(wga->wga_addr4))) 4676 goto _next; 4677 break; 4678 #ifdef INET6 4679 case AF_INET6: 4680 if (!prop_dictionary_set_data(prop_allowedip, 4681 "ip", &wga->wga_addr6, 4682 sizeof(wga->wga_addr6))) 4683 goto _next; 4684 break; 4685 #endif 4686 default: 4687 break; 4688 } 4689 prop_array_set(allowedips, j, prop_allowedip); 4690 _next: 4691 prop_object_release(prop_allowedip); 4692 } 4693 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4694 prop_object_release(allowedips); 4695 4696 skip_allowedips: 4697 4698 prop_array_set(peers, i, prop_peer); 4699 next: 4700 if (prop_peer) 4701 prop_object_release(prop_peer); 4702 i++; 4703 4704 s = pserialize_read_enter(); 4705 wg_put_peer(wgp, &wgp_psref); 4706 } 4707 pserialize_read_exit(s); 4708 4709 prop_dictionary_set(prop_dict, "peers", peers); 4710 prop_object_release(peers); 4711 peers = NULL; 4712 4713 skip_peers: 4714 buf = prop_dictionary_externalize(prop_dict); 4715 if (buf == NULL) 4716 goto error; 4717 if (ifd->ifd_len < (strlen(buf) + 1)) { 4718 error = EINVAL; 4719 goto error; 4720 } 4721 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4722 4723 free(buf, 0); 4724 error: 4725 if (peers != NULL) 4726 prop_object_release(peers); 4727 if (prop_dict != NULL) 4728 prop_object_release(prop_dict); 4729 4730 return error; 4731 } 4732 4733 static int 4734 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4735 { 4736 struct wg_softc *wg = ifp->if_softc; 4737 struct ifreq *ifr = data; 4738 struct ifaddr *ifa = data; 4739 struct ifdrv *ifd = data; 4740 int error = 0; 4741 4742 switch (cmd) { 4743 case SIOCINITIFADDR: 4744 if (ifa->ifa_addr->sa_family != AF_LINK && 4745 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4746 (IFF_UP | IFF_RUNNING)) { 4747 ifp->if_flags |= IFF_UP; 4748 error = if_init(ifp); 4749 } 4750 return error; 4751 case SIOCADDMULTI: 4752 case SIOCDELMULTI: 4753 switch (ifr->ifr_addr.sa_family) { 4754 case AF_INET: /* IP supports Multicast */ 4755 break; 4756 #ifdef INET6 4757 case AF_INET6: /* IP6 supports Multicast */ 4758 break; 4759 #endif 4760 default: /* Other protocols doesn't support Multicast */ 4761 error = EAFNOSUPPORT; 4762 break; 4763 } 4764 return error; 4765 case SIOCSDRVSPEC: 4766 if (!wg_is_authorized(wg, cmd)) { 4767 return EPERM; 4768 } 4769 switch (ifd->ifd_cmd) { 4770 case WG_IOCTL_SET_PRIVATE_KEY: 4771 error = wg_ioctl_set_private_key(wg, ifd); 4772 break; 4773 case WG_IOCTL_SET_LISTEN_PORT: 4774 error = wg_ioctl_set_listen_port(wg, ifd); 4775 break; 4776 case WG_IOCTL_ADD_PEER: 4777 error = wg_ioctl_add_peer(wg, ifd); 4778 break; 4779 case WG_IOCTL_DELETE_PEER: 4780 error = wg_ioctl_delete_peer(wg, ifd); 4781 break; 4782 default: 4783 error = EINVAL; 4784 break; 4785 } 4786 return error; 4787 case SIOCGDRVSPEC: 4788 return wg_ioctl_get(wg, ifd); 4789 case SIOCSIFFLAGS: 4790 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4791 break; 4792 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4793 case IFF_RUNNING: 4794 /* 4795 * If interface is marked down and it is running, 4796 * then stop and disable it. 4797 */ 4798 if_stop(ifp, 1); 4799 break; 4800 case IFF_UP: 4801 /* 4802 * If interface is marked up and it is stopped, then 4803 * start it. 4804 */ 4805 error = if_init(ifp); 4806 break; 4807 default: 4808 break; 4809 } 4810 return error; 4811 #ifdef WG_RUMPKERNEL 4812 case SIOCSLINKSTR: 4813 error = wg_ioctl_linkstr(wg, ifd); 4814 if (error == 0) 4815 wg->wg_ops = &wg_ops_rumpuser; 4816 return error; 4817 #endif 4818 default: 4819 break; 4820 } 4821 4822 error = ifioctl_common(ifp, cmd, data); 4823 4824 #ifdef WG_RUMPKERNEL 4825 if (!wg_user_mode(wg)) 4826 return error; 4827 4828 /* Do the same to the corresponding tun device on the host */ 4829 /* 4830 * XXX Actually the command has not been handled yet. It 4831 * will be handled via pr_ioctl form doifioctl later. 4832 */ 4833 switch (cmd) { 4834 case SIOCAIFADDR: 4835 case SIOCDIFADDR: { 4836 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4837 struct in_aliasreq *ifra = &_ifra; 4838 KASSERT(error == ENOTTY); 4839 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4840 IFNAMSIZ); 4841 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4842 if (error == 0) 4843 error = ENOTTY; 4844 break; 4845 } 4846 #ifdef INET6 4847 case SIOCAIFADDR_IN6: 4848 case SIOCDIFADDR_IN6: { 4849 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4850 struct in6_aliasreq *ifra = &_ifra; 4851 KASSERT(error == ENOTTY); 4852 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4853 IFNAMSIZ); 4854 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4855 if (error == 0) 4856 error = ENOTTY; 4857 break; 4858 } 4859 #endif 4860 } 4861 #endif /* WG_RUMPKERNEL */ 4862 4863 return error; 4864 } 4865 4866 static int 4867 wg_init(struct ifnet *ifp) 4868 { 4869 4870 ifp->if_flags |= IFF_RUNNING; 4871 4872 /* TODO flush pending packets. */ 4873 return 0; 4874 } 4875 4876 #ifdef ALTQ 4877 static void 4878 wg_start(struct ifnet *ifp) 4879 { 4880 struct mbuf *m; 4881 4882 for (;;) { 4883 IFQ_DEQUEUE(&ifp->if_snd, m); 4884 if (m == NULL) 4885 break; 4886 4887 kpreempt_disable(); 4888 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4889 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4890 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4891 if_name(ifp)); 4892 m_freem(m); 4893 } 4894 kpreempt_enable(); 4895 } 4896 } 4897 #endif 4898 4899 static void 4900 wg_stop(struct ifnet *ifp, int disable) 4901 { 4902 4903 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4904 ifp->if_flags &= ~IFF_RUNNING; 4905 4906 /* Need to do something? */ 4907 } 4908 4909 #ifdef WG_DEBUG_PARAMS 4910 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4911 { 4912 const struct sysctlnode *node = NULL; 4913 4914 sysctl_createv(clog, 0, NULL, &node, 4915 CTLFLAG_PERMANENT, 4916 CTLTYPE_NODE, "wg", 4917 SYSCTL_DESCR("wg(4)"), 4918 NULL, 0, NULL, 0, 4919 CTL_NET, CTL_CREATE, CTL_EOL); 4920 sysctl_createv(clog, 0, &node, NULL, 4921 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4922 CTLTYPE_QUAD, "rekey_after_messages", 4923 SYSCTL_DESCR("session liftime by messages"), 4924 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4925 sysctl_createv(clog, 0, &node, NULL, 4926 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4927 CTLTYPE_INT, "rekey_after_time", 4928 SYSCTL_DESCR("session liftime"), 4929 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4930 sysctl_createv(clog, 0, &node, NULL, 4931 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4932 CTLTYPE_INT, "rekey_timeout", 4933 SYSCTL_DESCR("session handshake retry time"), 4934 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4935 sysctl_createv(clog, 0, &node, NULL, 4936 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4937 CTLTYPE_INT, "rekey_attempt_time", 4938 SYSCTL_DESCR("session handshake timeout"), 4939 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4940 sysctl_createv(clog, 0, &node, NULL, 4941 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4942 CTLTYPE_INT, "keepalive_timeout", 4943 SYSCTL_DESCR("keepalive timeout"), 4944 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4945 sysctl_createv(clog, 0, &node, NULL, 4946 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4947 CTLTYPE_BOOL, "force_underload", 4948 SYSCTL_DESCR("force to detemine under load"), 4949 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4950 sysctl_createv(clog, 0, &node, NULL, 4951 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4952 CTLTYPE_INT, "debug", 4953 SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"), 4954 NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL); 4955 } 4956 #endif 4957 4958 #ifdef WG_RUMPKERNEL 4959 static bool 4960 wg_user_mode(struct wg_softc *wg) 4961 { 4962 4963 return wg->wg_user != NULL; 4964 } 4965 4966 static int 4967 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4968 { 4969 struct ifnet *ifp = &wg->wg_if; 4970 int error; 4971 4972 if (ifp->if_flags & IFF_UP) 4973 return EBUSY; 4974 4975 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4976 /* XXX do nothing */ 4977 return 0; 4978 } else if (ifd->ifd_cmd != 0) { 4979 return EINVAL; 4980 } else if (wg->wg_user != NULL) { 4981 return EBUSY; 4982 } 4983 4984 /* Assume \0 included */ 4985 if (ifd->ifd_len > IFNAMSIZ) { 4986 return E2BIG; 4987 } else if (ifd->ifd_len < 1) { 4988 return EINVAL; 4989 } 4990 4991 char tun_name[IFNAMSIZ]; 4992 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4993 if (error != 0) 4994 return error; 4995 4996 if (strncmp(tun_name, "tun", 3) != 0) 4997 return EINVAL; 4998 4999 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 5000 5001 return error; 5002 } 5003 5004 static int 5005 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 5006 { 5007 int error; 5008 struct psref psref; 5009 struct wg_sockaddr *wgsa; 5010 struct wg_softc *wg = wgp->wgp_sc; 5011 struct iovec iov[1]; 5012 5013 wgsa = wg_get_endpoint_sa(wgp, &psref); 5014 5015 iov[0].iov_base = mtod(m, void *); 5016 iov[0].iov_len = m->m_len; 5017 5018 /* Send messages to a peer via an ordinary socket. */ 5019 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 5020 5021 wg_put_sa(wgp, wgsa, &psref); 5022 5023 m_freem(m); 5024 5025 return error; 5026 } 5027 5028 static void 5029 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 5030 { 5031 struct wg_softc *wg = ifp->if_softc; 5032 struct iovec iov[2]; 5033 struct sockaddr_storage ss; 5034 5035 KASSERT(af == AF_INET || af == AF_INET6); 5036 5037 WG_TRACE(""); 5038 5039 if (af == AF_INET) { 5040 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 5041 struct ip *ip; 5042 5043 KASSERT(m->m_len >= sizeof(struct ip)); 5044 ip = mtod(m, struct ip *); 5045 sockaddr_in_init(sin, &ip->ip_dst, 0); 5046 } else { 5047 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 5048 struct ip6_hdr *ip6; 5049 5050 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 5051 ip6 = mtod(m, struct ip6_hdr *); 5052 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 5053 } 5054 5055 iov[0].iov_base = &ss; 5056 iov[0].iov_len = ss.ss_len; 5057 iov[1].iov_base = mtod(m, void *); 5058 iov[1].iov_len = m->m_len; 5059 5060 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5061 5062 /* Send decrypted packets to users via a tun. */ 5063 rumpuser_wg_send_user(wg->wg_user, iov, 2); 5064 5065 m_freem(m); 5066 } 5067 5068 static int 5069 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 5070 { 5071 int error; 5072 uint16_t old_port = wg->wg_listen_port; 5073 5074 if (port != 0 && old_port == port) 5075 return 0; 5076 5077 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5078 if (error == 0) 5079 wg->wg_listen_port = port; 5080 return error; 5081 } 5082 5083 /* 5084 * Receive user packets. 5085 */ 5086 void 5087 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5088 { 5089 struct ifnet *ifp = &wg->wg_if; 5090 struct mbuf *m; 5091 const struct sockaddr *dst; 5092 5093 WG_TRACE(""); 5094 5095 dst = iov[0].iov_base; 5096 5097 m = m_gethdr(M_DONTWAIT, MT_DATA); 5098 if (m == NULL) 5099 return; 5100 m->m_len = m->m_pkthdr.len = 0; 5101 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5102 5103 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5104 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5105 5106 (void)wg_output(ifp, m, dst, NULL); 5107 } 5108 5109 /* 5110 * Receive packets from a peer. 5111 */ 5112 void 5113 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5114 { 5115 struct mbuf *m; 5116 const struct sockaddr *src; 5117 int bound; 5118 5119 WG_TRACE(""); 5120 5121 src = iov[0].iov_base; 5122 5123 m = m_gethdr(M_DONTWAIT, MT_DATA); 5124 if (m == NULL) 5125 return; 5126 m->m_len = m->m_pkthdr.len = 0; 5127 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5128 5129 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5130 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5131 5132 bound = curlwp_bind(); 5133 wg_handle_packet(wg, m, src); 5134 curlwp_bindx(bound); 5135 } 5136 #endif /* WG_RUMPKERNEL */ 5137 5138 /* 5139 * Module infrastructure 5140 */ 5141 #include "if_module.h" 5142 5143 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5144
Indexes created Thu Sep 25 16:09:42 GMT 2025