if_wg.c revision 1.106
1 /* $NetBSD: if_wg.c,v 1.106 2024/07/28 14:48:13 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.106 2024/07/28 14:48:13 riastradh Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <lib/libkern/libkern.h> 87 88 #include <net/bpf.h> 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/if_wg.h> 92 #include <net/pktqueue.h> 93 #include <net/route.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip.h> 99 #include <netinet/ip_var.h> 100 #include <netinet/udp.h> 101 #include <netinet/udp_var.h> 102 103 #ifdef INET6 104 #include <netinet/ip6.h> 105 #include <netinet6/in6_pcb.h> 106 #include <netinet6/in6_var.h> 107 #include <netinet6/ip6_var.h> 108 #include <netinet6/udp6_var.h> 109 #endif /* INET6 */ 110 111 #include <prop/proplib.h> 112 113 #include <crypto/blake2/blake2s.h> 114 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 115 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 116 #include <crypto/sodium/crypto_scalarmult.h> 117 118 #include "ioconf.h" 119 120 #ifdef WG_RUMPKERNEL 121 #include "wg_user.h" 122 #endif 123 124 #ifndef time_uptime32 125 #define time_uptime32 ((uint32_t)time_uptime) 126 #endif 127 128 /* 129 * Data structures 130 * - struct wg_softc is an instance of wg interfaces 131 * - It has a list of peers (struct wg_peer) 132 * - It has a threadpool job that sends/receives handshake messages and 133 * runs event handlers 134 * - It has its own two routing tables: one is for IPv4 and the other IPv6 135 * - struct wg_peer is a representative of a peer 136 * - It has a struct work to handle handshakes and timer tasks 137 * - It has a pair of session instances (struct wg_session) 138 * - It has a pair of endpoint instances (struct wg_sockaddr) 139 * - Normally one endpoint is used and the second one is used only on 140 * a peer migration (a change of peer's IP address) 141 * - It has a list of IP addresses and sub networks called allowedips 142 * (struct wg_allowedip) 143 * - A packets sent over a session is allowed if its destination matches 144 * any IP addresses or sub networks of the list 145 * - struct wg_session represents a session of a secure tunnel with a peer 146 * - Two instances of sessions belong to a peer; a stable session and a 147 * unstable session 148 * - A handshake process of a session always starts with a unstable instance 149 * - Once a session is established, its instance becomes stable and the 150 * other becomes unstable instead 151 * - Data messages are always sent via a stable session 152 * 153 * Locking notes: 154 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 155 * - Changes to the peer list are serialized by wg_lock 156 * - The peer list may be read with pserialize(9) and psref(9) 157 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 158 * => XXX replace by pserialize when routing table is psz-safe 159 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 160 * only in thread context and serializes: 161 * - the stable and unstable session pointers 162 * - all unstable session state 163 * - Packet processing may be done in softint context: 164 * - The stable session can be read under pserialize(9) or psref(9) 165 * - The stable session is always ESTABLISHED 166 * - On a session swap, we must wait for all readers to release a 167 * reference to a stable session before changing wgs_state and 168 * session states 169 * - Lock order: wg_lock -> wgp_lock 170 */ 171 172 173 #define WGLOG(level, fmt, args...) \ 174 log(level, "%s: " fmt, __func__, ##args) 175 176 #define WG_DEBUG 177 178 /* Debug options */ 179 #ifdef WG_DEBUG 180 /* Output debug logs */ 181 #ifndef WG_DEBUG_LOG 182 #define WG_DEBUG_LOG 183 #endif 184 /* Output trace logs */ 185 #ifndef WG_DEBUG_TRACE 186 #define WG_DEBUG_TRACE 187 #endif 188 /* Output hash values, etc. */ 189 #ifndef WG_DEBUG_DUMP 190 #define WG_DEBUG_DUMP 191 #endif 192 /* debug packets */ 193 #ifndef WG_DEBUG_PACKET 194 #define WG_DEBUG_PACKET 195 #endif 196 /* Make some internal parameters configurable for testing and debugging */ 197 #ifndef WG_DEBUG_PARAMS 198 #define WG_DEBUG_PARAMS 199 #endif 200 #endif /* WG_DEBUG */ 201 202 #ifndef WG_DEBUG 203 # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \ 204 defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS) || \ 205 defined(WG_DEBUG_PACKET) 206 # define WG_DEBUG 207 # endif 208 #endif 209 210 #ifdef WG_DEBUG 211 int wg_debug; 212 #define WG_DEBUG_FLAGS_LOG 1 213 #define WG_DEBUG_FLAGS_TRACE 2 214 #define WG_DEBUG_FLAGS_DUMP 4 215 #define WG_DEBUG_FLAGS_PACKET 8 216 #endif 217 218 219 #ifdef WG_DEBUG_TRACE 220 #define WG_TRACE(msg) do { \ 221 if (wg_debug & WG_DEBUG_FLAGS_TRACE) \ 222 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \ 223 } while (0) 224 #else 225 #define WG_TRACE(msg) __nothing 226 #endif 227 228 #ifdef WG_DEBUG_LOG 229 #define WG_DLOG(fmt, args...) do { \ 230 if (wg_debug & WG_DEBUG_FLAGS_LOG) \ 231 log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \ 232 } while (0) 233 #else 234 #define WG_DLOG(fmt, args...) __nothing 235 #endif 236 237 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 238 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 239 &(wgprc)->wgprc_curpps, 1)) { \ 240 log(level, fmt, ##args); \ 241 } \ 242 } while (0) 243 244 #ifdef WG_DEBUG_PARAMS 245 static bool wg_force_underload = false; 246 #endif 247 248 #ifdef WG_DEBUG_DUMP 249 250 static char enomem[10] = "[enomem]"; 251 252 #define MAX_HDUMP_LEN 10000 /* large enough */ 253 254 255 static char * 256 gethexdump(const void *vp, size_t n) 257 { 258 char *buf; 259 const uint8_t *p = vp; 260 size_t i, alloc; 261 262 alloc = n; 263 if (n > MAX_HDUMP_LEN) 264 alloc = MAX_HDUMP_LEN; 265 buf = kmem_alloc(3 * alloc + 5, KM_NOSLEEP); 266 if (buf == NULL) 267 return enomem; 268 for (i = 0; i < alloc; i++) 269 snprintf(buf + 3 * i, 3 + 1, " %02hhx", p[i]); 270 if (alloc != n) 271 snprintf(buf + 3 * i, 4 + 1, " ..."); 272 return buf; 273 } 274 275 static void 276 puthexdump(char *buf, const void *p, size_t n) 277 { 278 279 if (buf == NULL || buf == enomem) 280 return; 281 if (n > MAX_HDUMP_LEN) 282 n = MAX_HDUMP_LEN; 283 kmem_free(buf, 3 * n + 5); 284 } 285 286 #ifdef WG_RUMPKERNEL 287 static void 288 wg_dump_buf(const char *func, const char *buf, const size_t size) 289 { 290 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 291 return; 292 293 char *hex = gethexdump(buf, size); 294 295 log(LOG_DEBUG, "%s: %s\n", func, hex); 296 puthexdump(hex, buf, size); 297 } 298 #endif 299 300 static void 301 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 302 const size_t size) 303 { 304 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 305 return; 306 307 char *hex = gethexdump(hash, size); 308 309 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex); 310 puthexdump(hex, hash, size); 311 } 312 313 #define WG_DUMP_HASH(name, hash) \ 314 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 315 #define WG_DUMP_HASH48(name, hash) \ 316 wg_dump_hash(__func__, name, hash, 48) 317 #define WG_DUMP_BUF(buf, size) \ 318 wg_dump_buf(__func__, buf, size) 319 #else 320 #define WG_DUMP_HASH(name, hash) __nothing 321 #define WG_DUMP_HASH48(name, hash) __nothing 322 #define WG_DUMP_BUF(buf, size) __nothing 323 #endif /* WG_DEBUG_DUMP */ 324 325 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 326 #define WG_MAX_PROPLEN 32766 327 328 #define WG_MTU 1420 329 #define WG_ALLOWEDIPS 16 330 331 #define CURVE25519_KEY_LEN 32 332 #define TAI64N_LEN sizeof(uint32_t) * 3 333 #define POLY1305_AUTHTAG_LEN 16 334 #define HMAC_BLOCK_LEN 64 335 336 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 337 /* [N] 4.3: Hash functions */ 338 #define NOISE_DHLEN 32 339 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 340 #define NOISE_HASHLEN 32 341 #define NOISE_BLOCKLEN 64 342 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 343 /* [N] 5.1: "k" */ 344 #define NOISE_CIPHER_KEY_LEN 32 345 /* 346 * [N] 9.2: "psk" 347 * "... psk is a 32-byte secret value provided by the application." 348 */ 349 #define NOISE_PRESHARED_KEY_LEN 32 350 351 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 352 #define WG_TIMESTAMP_LEN TAI64N_LEN 353 354 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 355 356 #define WG_COOKIE_LEN 16 357 #define WG_MAC_LEN 16 358 #define WG_COOKIESECRET_LEN 32 359 360 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 361 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 362 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 363 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 364 #define WG_HASH_LEN NOISE_HASHLEN 365 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 366 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 367 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 368 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 369 #define WG_DATA_KEY_LEN 32 370 #define WG_SALT_LEN 24 371 372 /* 373 * The protocol messages 374 */ 375 struct wg_msg { 376 uint32_t wgm_type; 377 } __packed; 378 379 /* [W] 5.4.2 First Message: Initiator to Responder */ 380 struct wg_msg_init { 381 uint32_t wgmi_type; 382 uint32_t wgmi_sender; 383 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 384 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 385 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 386 uint8_t wgmi_mac1[WG_MAC_LEN]; 387 uint8_t wgmi_mac2[WG_MAC_LEN]; 388 } __packed; 389 390 /* [W] 5.4.3 Second Message: Responder to Initiator */ 391 struct wg_msg_resp { 392 uint32_t wgmr_type; 393 uint32_t wgmr_sender; 394 uint32_t wgmr_receiver; 395 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 396 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 397 uint8_t wgmr_mac1[WG_MAC_LEN]; 398 uint8_t wgmr_mac2[WG_MAC_LEN]; 399 } __packed; 400 401 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 402 struct wg_msg_data { 403 uint32_t wgmd_type; 404 uint32_t wgmd_receiver; 405 uint64_t wgmd_counter; 406 uint32_t wgmd_packet[0]; 407 } __packed; 408 409 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 410 struct wg_msg_cookie { 411 uint32_t wgmc_type; 412 uint32_t wgmc_receiver; 413 uint8_t wgmc_salt[WG_SALT_LEN]; 414 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 415 } __packed; 416 417 #define WG_MSG_TYPE_INIT 1 418 #define WG_MSG_TYPE_RESP 2 419 #define WG_MSG_TYPE_COOKIE 3 420 #define WG_MSG_TYPE_DATA 4 421 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 422 423 /* Sliding windows */ 424 425 #define SLIWIN_BITS 2048u 426 #define SLIWIN_TYPE uint32_t 427 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 428 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 429 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 430 431 struct sliwin { 432 SLIWIN_TYPE B[SLIWIN_WORDS]; 433 uint64_t T; 434 }; 435 436 static void 437 sliwin_reset(struct sliwin *W) 438 { 439 440 memset(W, 0, sizeof(*W)); 441 } 442 443 static int 444 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 445 { 446 447 /* 448 * If it's more than one window older than the highest sequence 449 * number we've seen, reject. 450 */ 451 #ifdef __HAVE_ATOMIC64_LOADSTORE 452 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 453 return EAUTH; 454 #endif 455 456 /* 457 * Otherwise, we need to take the lock to decide, so don't 458 * reject just yet. Caller must serialize a call to 459 * sliwin_update in this case. 460 */ 461 return 0; 462 } 463 464 static int 465 sliwin_update(struct sliwin *W, uint64_t S) 466 { 467 unsigned word, bit; 468 469 /* 470 * If it's more than one window older than the highest sequence 471 * number we've seen, reject. 472 */ 473 if (S + SLIWIN_NPKT < W->T) 474 return EAUTH; 475 476 /* 477 * If it's higher than the highest sequence number we've seen, 478 * advance the window. 479 */ 480 if (S > W->T) { 481 uint64_t i = W->T / SLIWIN_BPW; 482 uint64_t j = S / SLIWIN_BPW; 483 unsigned k; 484 485 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 486 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 487 #ifdef __HAVE_ATOMIC64_LOADSTORE 488 atomic_store_relaxed(&W->T, S); 489 #else 490 W->T = S; 491 #endif 492 } 493 494 /* Test and set the bit -- if already set, reject. */ 495 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 496 bit = S % SLIWIN_BPW; 497 if (W->B[word] & (1UL << bit)) 498 return EAUTH; 499 W->B[word] |= 1U << bit; 500 501 /* Accept! */ 502 return 0; 503 } 504 505 struct wg_session { 506 struct wg_peer *wgs_peer; 507 struct psref_target 508 wgs_psref; 509 510 int wgs_state; 511 #define WGS_STATE_UNKNOWN 0 512 #define WGS_STATE_INIT_ACTIVE 1 513 #define WGS_STATE_INIT_PASSIVE 2 514 #define WGS_STATE_ESTABLISHED 3 515 #define WGS_STATE_DESTROYING 4 516 517 volatile uint32_t 518 wgs_time_established; 519 volatile uint32_t 520 wgs_time_last_data_sent; 521 bool wgs_is_initiator; 522 523 uint32_t wgs_local_index; 524 uint32_t wgs_remote_index; 525 #ifdef __HAVE_ATOMIC64_LOADSTORE 526 volatile uint64_t 527 wgs_send_counter; 528 #else 529 kmutex_t wgs_send_counter_lock; 530 uint64_t wgs_send_counter; 531 #endif 532 533 struct { 534 kmutex_t lock; 535 struct sliwin window; 536 } *wgs_recvwin; 537 538 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 539 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 540 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 541 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 542 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 543 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 544 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 545 }; 546 547 struct wg_sockaddr { 548 union { 549 struct sockaddr_storage _ss; 550 struct sockaddr _sa; 551 struct sockaddr_in _sin; 552 struct sockaddr_in6 _sin6; 553 }; 554 struct psref_target wgsa_psref; 555 }; 556 557 #define wgsatoss(wgsa) (&(wgsa)->_ss) 558 #define wgsatosa(wgsa) (&(wgsa)->_sa) 559 #define wgsatosin(wgsa) (&(wgsa)->_sin) 560 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 561 562 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 563 564 struct wg_peer; 565 struct wg_allowedip { 566 struct radix_node wga_nodes[2]; 567 struct wg_sockaddr _wga_sa_addr; 568 struct wg_sockaddr _wga_sa_mask; 569 #define wga_sa_addr _wga_sa_addr._sa 570 #define wga_sa_mask _wga_sa_mask._sa 571 572 int wga_family; 573 uint8_t wga_cidr; 574 union { 575 struct in_addr _ip4; 576 struct in6_addr _ip6; 577 } wga_addr; 578 #define wga_addr4 wga_addr._ip4 579 #define wga_addr6 wga_addr._ip6 580 581 struct wg_peer *wga_peer; 582 }; 583 584 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 585 586 struct wg_ppsratecheck { 587 struct timeval wgprc_lasttime; 588 int wgprc_curpps; 589 }; 590 591 struct wg_softc; 592 struct wg_peer { 593 struct wg_softc *wgp_sc; 594 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 595 struct pslist_entry wgp_peerlist_entry; 596 pserialize_t wgp_psz; 597 struct psref_target wgp_psref; 598 kmutex_t *wgp_lock; 599 kmutex_t *wgp_intr_lock; 600 601 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 602 struct wg_sockaddr *wgp_endpoint; 603 struct wg_sockaddr *wgp_endpoint0; 604 volatile unsigned wgp_endpoint_changing; 605 bool wgp_endpoint_available; 606 607 /* The preshared key (optional) */ 608 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 609 610 struct wg_session *wgp_session_stable; 611 struct wg_session *wgp_session_unstable; 612 613 /* first outgoing packet awaiting session initiation */ 614 struct mbuf *volatile wgp_pending; 615 616 /* timestamp in big-endian */ 617 wg_timestamp_t wgp_timestamp_latest_init; 618 619 struct timespec wgp_last_handshake_time; 620 621 callout_t wgp_handshake_timeout_timer; 622 callout_t wgp_session_dtor_timer; 623 624 time_t wgp_handshake_start_time; 625 626 volatile unsigned wgp_force_rekey; 627 628 int wgp_n_allowedips; 629 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 630 631 time_t wgp_latest_cookie_time; 632 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 633 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 634 bool wgp_last_sent_mac1_valid; 635 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 636 bool wgp_last_sent_cookie_valid; 637 638 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 639 640 time_t wgp_last_cookiesecret_time; 641 uint8_t wgp_cookiesecret[WG_COOKIESECRET_LEN]; 642 643 struct wg_ppsratecheck wgp_ppsratecheck; 644 645 struct work wgp_work; 646 unsigned int wgp_tasks; 647 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 648 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 649 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 650 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 651 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 652 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 653 }; 654 655 struct wg_ops; 656 657 struct wg_softc { 658 struct ifnet wg_if; 659 LIST_ENTRY(wg_softc) wg_list; 660 kmutex_t *wg_lock; 661 kmutex_t *wg_intr_lock; 662 krwlock_t *wg_rwlock; 663 664 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 665 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 666 667 int wg_npeers; 668 struct pslist_head wg_peers; 669 struct thmap *wg_peers_bypubkey; 670 struct thmap *wg_peers_byname; 671 struct thmap *wg_sessions_byindex; 672 uint16_t wg_listen_port; 673 674 struct threadpool *wg_threadpool; 675 676 struct threadpool_job wg_job; 677 int wg_upcalls; 678 #define WG_UPCALL_INET __BIT(0) 679 #define WG_UPCALL_INET6 __BIT(1) 680 681 #ifdef INET 682 struct socket *wg_so4; 683 struct radix_node_head *wg_rtable_ipv4; 684 #endif 685 #ifdef INET6 686 struct socket *wg_so6; 687 struct radix_node_head *wg_rtable_ipv6; 688 #endif 689 690 struct wg_ppsratecheck wg_ppsratecheck; 691 692 struct wg_ops *wg_ops; 693 694 #ifdef WG_RUMPKERNEL 695 struct wg_user *wg_user; 696 #endif 697 }; 698 699 /* [W] 6.1 Preliminaries */ 700 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 701 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 702 #define WG_REKEY_AFTER_TIME 120 703 #define WG_REJECT_AFTER_TIME 180 704 #define WG_REKEY_ATTEMPT_TIME 90 705 #define WG_REKEY_TIMEOUT 5 706 #define WG_KEEPALIVE_TIMEOUT 10 707 708 #define WG_COOKIE_TIME 120 709 #define WG_COOKIESECRET_TIME (2 * 60) 710 711 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 712 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 713 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 714 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 715 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 716 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 717 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 718 719 static struct mbuf * 720 wg_get_mbuf(size_t, size_t); 721 722 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 723 struct mbuf *); 724 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 725 const uint32_t, const uint8_t [WG_MAC_LEN], 726 const struct sockaddr *); 727 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 728 struct wg_session *, const struct wg_msg_init *); 729 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 730 731 static struct wg_peer * 732 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 733 struct psref *); 734 static struct wg_peer * 735 wg_lookup_peer_by_pubkey(struct wg_softc *, 736 const uint8_t [WG_STATIC_KEY_LEN], struct psref *); 737 738 static struct wg_session * 739 wg_lookup_session_by_index(struct wg_softc *, 740 const uint32_t, struct psref *); 741 742 static void wg_update_endpoint_if_necessary(struct wg_peer *, 743 const struct sockaddr *); 744 745 static void wg_schedule_session_dtor_timer(struct wg_peer *); 746 747 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 748 static void wg_calculate_keys(struct wg_session *, const bool); 749 750 static void wg_clear_states(struct wg_session *); 751 752 static void wg_get_peer(struct wg_peer *, struct psref *); 753 static void wg_put_peer(struct wg_peer *, struct psref *); 754 755 static int wg_send_so(struct wg_peer *, struct mbuf *); 756 static int wg_send_udp(struct wg_peer *, struct mbuf *); 757 static int wg_output(struct ifnet *, struct mbuf *, 758 const struct sockaddr *, const struct rtentry *); 759 static void wg_input(struct ifnet *, struct mbuf *, const int); 760 static int wg_ioctl(struct ifnet *, u_long, void *); 761 static int wg_bind_port(struct wg_softc *, const uint16_t); 762 static int wg_init(struct ifnet *); 763 #ifdef ALTQ 764 static void wg_start(struct ifnet *); 765 #endif 766 static void wg_stop(struct ifnet *, int); 767 768 static void wg_peer_work(struct work *, void *); 769 static void wg_job(struct threadpool_job *); 770 static void wgintr(void *); 771 static void wg_purge_pending_packets(struct wg_peer *); 772 773 static int wg_clone_create(struct if_clone *, int); 774 static int wg_clone_destroy(struct ifnet *); 775 776 struct wg_ops { 777 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 778 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 779 void (*input)(struct ifnet *, struct mbuf *, const int); 780 int (*bind_port)(struct wg_softc *, const uint16_t); 781 }; 782 783 struct wg_ops wg_ops_rumpkernel = { 784 .send_hs_msg = wg_send_so, 785 .send_data_msg = wg_send_udp, 786 .input = wg_input, 787 .bind_port = wg_bind_port, 788 }; 789 790 #ifdef WG_RUMPKERNEL 791 static bool wg_user_mode(struct wg_softc *); 792 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 793 794 static int wg_send_user(struct wg_peer *, struct mbuf *); 795 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 796 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 797 798 struct wg_ops wg_ops_rumpuser = { 799 .send_hs_msg = wg_send_user, 800 .send_data_msg = wg_send_user, 801 .input = wg_input_user, 802 .bind_port = wg_bind_port_user, 803 }; 804 #endif 805 806 #define WG_PEER_READER_FOREACH(wgp, wg) \ 807 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 808 wgp_peerlist_entry) 809 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 810 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 811 wgp_peerlist_entry) 812 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 813 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 814 #define WG_PEER_WRITER_REMOVE(wgp) \ 815 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 816 817 struct wg_route { 818 struct radix_node wgr_nodes[2]; 819 struct wg_peer *wgr_peer; 820 }; 821 822 static struct radix_node_head * 823 wg_rnh(struct wg_softc *wg, const int family) 824 { 825 826 switch (family) { 827 case AF_INET: 828 return wg->wg_rtable_ipv4; 829 #ifdef INET6 830 case AF_INET6: 831 return wg->wg_rtable_ipv6; 832 #endif 833 default: 834 return NULL; 835 } 836 } 837 838 839 /* 840 * Global variables 841 */ 842 static volatile unsigned wg_count __cacheline_aligned; 843 844 struct psref_class *wg_psref_class __read_mostly; 845 846 static struct if_clone wg_cloner = 847 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 848 849 static struct pktqueue *wg_pktq __read_mostly; 850 static struct workqueue *wg_wq __read_mostly; 851 852 void wgattach(int); 853 /* ARGSUSED */ 854 void 855 wgattach(int count) 856 { 857 /* 858 * Nothing to do here, initialization is handled by the 859 * module initialization code in wginit() below). 860 */ 861 } 862 863 static void 864 wginit(void) 865 { 866 867 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 868 869 if_clone_attach(&wg_cloner); 870 } 871 872 /* 873 * XXX Kludge: This should just happen in wginit, but workqueue_create 874 * cannot be run until after CPUs have been detected, and wginit runs 875 * before configure. 876 */ 877 static int 878 wginitqueues(void) 879 { 880 int error __diagused; 881 882 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 883 KASSERT(wg_pktq != NULL); 884 885 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 886 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 887 KASSERT(error == 0); 888 889 return 0; 890 } 891 892 static void 893 wg_guarantee_initialized(void) 894 { 895 static ONCE_DECL(init); 896 int error __diagused; 897 898 error = RUN_ONCE(&init, wginitqueues); 899 KASSERT(error == 0); 900 } 901 902 static int 903 wg_count_inc(void) 904 { 905 unsigned o, n; 906 907 do { 908 o = atomic_load_relaxed(&wg_count); 909 if (o == UINT_MAX) 910 return ENFILE; 911 n = o + 1; 912 } while (atomic_cas_uint(&wg_count, o, n) != o); 913 914 return 0; 915 } 916 917 static void 918 wg_count_dec(void) 919 { 920 unsigned c __diagused; 921 922 c = atomic_dec_uint_nv(&wg_count); 923 KASSERT(c != UINT_MAX); 924 } 925 926 static int 927 wgdetach(void) 928 { 929 930 /* Prevent new interface creation. */ 931 if_clone_detach(&wg_cloner); 932 933 /* Check whether there are any existing interfaces. */ 934 if (atomic_load_relaxed(&wg_count)) { 935 /* Back out -- reattach the cloner. */ 936 if_clone_attach(&wg_cloner); 937 return EBUSY; 938 } 939 940 /* No interfaces left. Nuke it. */ 941 if (wg_wq) 942 workqueue_destroy(wg_wq); 943 if (wg_pktq) 944 pktq_destroy(wg_pktq); 945 psref_class_destroy(wg_psref_class); 946 947 return 0; 948 } 949 950 static void 951 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 952 uint8_t hash[WG_HASH_LEN]) 953 { 954 /* [W] 5.4: CONSTRUCTION */ 955 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 956 /* [W] 5.4: IDENTIFIER */ 957 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 958 struct blake2s state; 959 960 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 961 signature, strlen(signature)); 962 963 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 964 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 965 966 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 967 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 968 blake2s_update(&state, id, strlen(id)); 969 blake2s_final(&state, hash); 970 971 WG_DUMP_HASH("ckey", ckey); 972 WG_DUMP_HASH("hash", hash); 973 } 974 975 static void 976 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 977 const size_t inputsize) 978 { 979 struct blake2s state; 980 981 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 982 blake2s_update(&state, hash, WG_HASH_LEN); 983 blake2s_update(&state, input, inputsize); 984 blake2s_final(&state, hash); 985 } 986 987 static void 988 wg_algo_mac(uint8_t out[], const size_t outsize, 989 const uint8_t key[], const size_t keylen, 990 const uint8_t input1[], const size_t input1len, 991 const uint8_t input2[], const size_t input2len) 992 { 993 struct blake2s state; 994 995 blake2s_init(&state, outsize, key, keylen); 996 997 blake2s_update(&state, input1, input1len); 998 if (input2 != NULL) 999 blake2s_update(&state, input2, input2len); 1000 blake2s_final(&state, out); 1001 } 1002 1003 static void 1004 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 1005 const uint8_t input1[], const size_t input1len, 1006 const uint8_t input2[], const size_t input2len) 1007 { 1008 struct blake2s state; 1009 /* [W] 5.4: LABEL-MAC1 */ 1010 const char *label = "mac1----"; 1011 uint8_t key[WG_HASH_LEN]; 1012 1013 blake2s_init(&state, sizeof(key), NULL, 0); 1014 blake2s_update(&state, label, strlen(label)); 1015 blake2s_update(&state, input1, input1len); 1016 blake2s_final(&state, key); 1017 1018 blake2s_init(&state, outsize, key, sizeof(key)); 1019 if (input2 != NULL) 1020 blake2s_update(&state, input2, input2len); 1021 blake2s_final(&state, out); 1022 } 1023 1024 static void 1025 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 1026 const uint8_t input1[], const size_t input1len) 1027 { 1028 struct blake2s state; 1029 /* [W] 5.4: LABEL-COOKIE */ 1030 const char *label = "cookie--"; 1031 1032 blake2s_init(&state, outsize, NULL, 0); 1033 blake2s_update(&state, label, strlen(label)); 1034 blake2s_update(&state, input1, input1len); 1035 blake2s_final(&state, out); 1036 } 1037 1038 static void 1039 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 1040 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 1041 { 1042 1043 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1044 1045 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 1046 crypto_scalarmult_base(pubkey, privkey); 1047 } 1048 1049 static void 1050 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 1051 const uint8_t privkey[WG_STATIC_KEY_LEN], 1052 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 1053 { 1054 1055 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1056 1057 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1058 KASSERT(ret == 0); 1059 } 1060 1061 static void 1062 wg_algo_hmac(uint8_t out[], const size_t outlen, 1063 const uint8_t key[], const size_t keylen, 1064 const uint8_t in[], const size_t inlen) 1065 { 1066 #define IPAD 0x36 1067 #define OPAD 0x5c 1068 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1069 uint8_t ipad[HMAC_BLOCK_LEN]; 1070 uint8_t opad[HMAC_BLOCK_LEN]; 1071 size_t i; 1072 struct blake2s state; 1073 1074 KASSERT(outlen == WG_HASH_LEN); 1075 KASSERT(keylen <= HMAC_BLOCK_LEN); 1076 1077 memcpy(hmackey, key, keylen); 1078 1079 for (i = 0; i < sizeof(hmackey); i++) { 1080 ipad[i] = hmackey[i] ^ IPAD; 1081 opad[i] = hmackey[i] ^ OPAD; 1082 } 1083 1084 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1085 blake2s_update(&state, ipad, sizeof(ipad)); 1086 blake2s_update(&state, in, inlen); 1087 blake2s_final(&state, out); 1088 1089 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1090 blake2s_update(&state, opad, sizeof(opad)); 1091 blake2s_update(&state, out, WG_HASH_LEN); 1092 blake2s_final(&state, out); 1093 #undef IPAD 1094 #undef OPAD 1095 } 1096 1097 static void 1098 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1099 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1100 const uint8_t input[], const size_t inputlen) 1101 { 1102 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1103 uint8_t one[1]; 1104 1105 /* 1106 * [N] 4.3: "an input_key_material byte sequence with length 1107 * either zero bytes, 32 bytes, or DHLEN bytes." 1108 */ 1109 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1110 1111 WG_DUMP_HASH("ckey", ckey); 1112 if (input != NULL) 1113 WG_DUMP_HASH("input", input); 1114 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1115 input, inputlen); 1116 WG_DUMP_HASH("tmp1", tmp1); 1117 one[0] = 1; 1118 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1119 one, sizeof(one)); 1120 WG_DUMP_HASH("out1", out1); 1121 if (out2 == NULL) 1122 return; 1123 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1124 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1125 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1126 tmp2, sizeof(tmp2)); 1127 WG_DUMP_HASH("out2", out2); 1128 if (out3 == NULL) 1129 return; 1130 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1131 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1132 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1133 tmp2, sizeof(tmp2)); 1134 WG_DUMP_HASH("out3", out3); 1135 } 1136 1137 static void __noinline 1138 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1139 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1140 const uint8_t local_key[WG_STATIC_KEY_LEN], 1141 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1142 { 1143 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1144 1145 wg_algo_dh(dhout, local_key, remote_key); 1146 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1147 1148 WG_DUMP_HASH("dhout", dhout); 1149 WG_DUMP_HASH("ckey", ckey); 1150 if (cipher_key != NULL) 1151 WG_DUMP_HASH("cipher_key", cipher_key); 1152 } 1153 1154 static void 1155 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1156 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1157 const uint8_t auth[], size_t authlen) 1158 { 1159 uint8_t nonce[(32 + 64) / 8] = {0}; 1160 long long unsigned int outsize; 1161 int error __diagused; 1162 1163 le64enc(&nonce[4], counter); 1164 1165 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1166 plainsize, auth, authlen, NULL, nonce, key); 1167 KASSERT(error == 0); 1168 KASSERT(outsize == expected_outsize); 1169 } 1170 1171 static int 1172 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1173 const uint64_t counter, const uint8_t encrypted[], 1174 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1175 { 1176 uint8_t nonce[(32 + 64) / 8] = {0}; 1177 long long unsigned int outsize; 1178 int error; 1179 1180 le64enc(&nonce[4], counter); 1181 1182 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1183 encrypted, encryptedsize, auth, authlen, nonce, key); 1184 if (error == 0) 1185 KASSERT(outsize == expected_outsize); 1186 return error; 1187 } 1188 1189 static void 1190 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1191 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1192 const uint8_t auth[], size_t authlen, 1193 const uint8_t nonce[WG_SALT_LEN]) 1194 { 1195 long long unsigned int outsize; 1196 int error __diagused; 1197 1198 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1199 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1200 plain, plainsize, auth, authlen, NULL, nonce, key); 1201 KASSERT(error == 0); 1202 KASSERT(outsize == expected_outsize); 1203 } 1204 1205 static int 1206 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1207 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1208 const uint8_t auth[], size_t authlen, 1209 const uint8_t nonce[WG_SALT_LEN]) 1210 { 1211 long long unsigned int outsize; 1212 int error; 1213 1214 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1215 encrypted, encryptedsize, auth, authlen, nonce, key); 1216 if (error == 0) 1217 KASSERT(outsize == expected_outsize); 1218 return error; 1219 } 1220 1221 static void 1222 wg_algo_tai64n(wg_timestamp_t timestamp) 1223 { 1224 struct timespec ts; 1225 1226 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1227 getnanotime(&ts); 1228 /* TAI64 label in external TAI64 format */ 1229 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1230 /* second beginning from 1970 TAI */ 1231 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1232 /* nanosecond in big-endian format */ 1233 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1234 } 1235 1236 /* 1237 * wg_get_stable_session(wgp, psref) 1238 * 1239 * Get a passive reference to the current stable session, or 1240 * return NULL if there is no current stable session. 1241 * 1242 * The pointer is always there but the session is not necessarily 1243 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1244 * the session may transition from ESTABLISHED to DESTROYING while 1245 * holding the passive reference. 1246 */ 1247 static struct wg_session * 1248 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1249 { 1250 int s; 1251 struct wg_session *wgs; 1252 1253 s = pserialize_read_enter(); 1254 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1255 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1256 wgs = NULL; 1257 else 1258 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1259 pserialize_read_exit(s); 1260 1261 return wgs; 1262 } 1263 1264 static void 1265 wg_put_session(struct wg_session *wgs, struct psref *psref) 1266 { 1267 1268 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1269 } 1270 1271 static void 1272 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1273 { 1274 struct wg_peer *wgp = wgs->wgs_peer; 1275 struct wg_session *wgs0 __diagused; 1276 void *garbage; 1277 1278 KASSERT(mutex_owned(wgp->wgp_lock)); 1279 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1280 1281 /* Remove the session from the table. */ 1282 wgs0 = thmap_del(wg->wg_sessions_byindex, 1283 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1284 KASSERT(wgs0 == wgs); 1285 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1286 1287 /* Wait for passive references to drain. */ 1288 pserialize_perform(wgp->wgp_psz); 1289 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1290 1291 /* 1292 * Free memory, zero state, and transition to UNKNOWN. We have 1293 * exclusive access to the session now, so there is no need for 1294 * an atomic store. 1295 */ 1296 thmap_gc(wg->wg_sessions_byindex, garbage); 1297 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_UNKNOWN\n", 1298 wgs->wgs_local_index, wgs->wgs_remote_index); 1299 wgs->wgs_local_index = 0; 1300 wgs->wgs_remote_index = 0; 1301 wg_clear_states(wgs); 1302 wgs->wgs_state = WGS_STATE_UNKNOWN; 1303 } 1304 1305 /* 1306 * wg_get_session_index(wg, wgs) 1307 * 1308 * Choose a session index for wgs->wgs_local_index, and store it 1309 * in wg's table of sessions by index. 1310 * 1311 * wgs must be the unstable session of its peer, and must be 1312 * transitioning out of the UNKNOWN state. 1313 */ 1314 static void 1315 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1316 { 1317 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1318 struct wg_session *wgs0; 1319 uint32_t index; 1320 1321 KASSERT(mutex_owned(wgp->wgp_lock)); 1322 KASSERT(wgs == wgp->wgp_session_unstable); 1323 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1324 wgs->wgs_state); 1325 1326 do { 1327 /* Pick a uniform random index. */ 1328 index = cprng_strong32(); 1329 1330 /* Try to take it. */ 1331 wgs->wgs_local_index = index; 1332 wgs0 = thmap_put(wg->wg_sessions_byindex, 1333 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1334 1335 /* If someone else beat us, start over. */ 1336 } while (__predict_false(wgs0 != wgs)); 1337 } 1338 1339 /* 1340 * wg_put_session_index(wg, wgs) 1341 * 1342 * Remove wgs from the table of sessions by index, wait for any 1343 * passive references to drain, and transition the session to the 1344 * UNKNOWN state. 1345 * 1346 * wgs must be the unstable session of its peer, and must not be 1347 * UNKNOWN or ESTABLISHED. 1348 */ 1349 static void 1350 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1351 { 1352 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1353 1354 KASSERT(mutex_owned(wgp->wgp_lock)); 1355 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1356 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1357 1358 wg_destroy_session(wg, wgs); 1359 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1360 } 1361 1362 /* 1363 * Handshake patterns 1364 * 1365 * [W] 5: "These messages use the "IK" pattern from Noise" 1366 * [N] 7.5. Interactive handshake patterns (fundamental) 1367 * "The first character refers to the initiators static key:" 1368 * "I = Static key for initiator Immediately transmitted to responder, 1369 * despite reduced or absent identity hiding" 1370 * "The second character refers to the responders static key:" 1371 * "K = Static key for responder Known to initiator" 1372 * "IK: 1373 * <- s 1374 * ... 1375 * -> e, es, s, ss 1376 * <- e, ee, se" 1377 * [N] 9.4. Pattern modifiers 1378 * "IKpsk2: 1379 * <- s 1380 * ... 1381 * -> e, es, s, ss 1382 * <- e, ee, se, psk" 1383 */ 1384 static void 1385 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1386 struct wg_session *wgs, struct wg_msg_init *wgmi) 1387 { 1388 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1389 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1390 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1391 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1392 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1393 1394 KASSERT(mutex_owned(wgp->wgp_lock)); 1395 KASSERT(wgs == wgp->wgp_session_unstable); 1396 KASSERTMSG(wgs->wgs_state == WGS_STATE_INIT_ACTIVE, "state=%d", 1397 wgs->wgs_state); 1398 1399 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1400 wgmi->wgmi_sender = wgs->wgs_local_index; 1401 1402 /* [W] 5.4.2: First Message: Initiator to Responder */ 1403 1404 /* Ci := HASH(CONSTRUCTION) */ 1405 /* Hi := HASH(Ci || IDENTIFIER) */ 1406 wg_init_key_and_hash(ckey, hash); 1407 /* Hi := HASH(Hi || Sr^pub) */ 1408 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1409 1410 WG_DUMP_HASH("hash", hash); 1411 1412 /* [N] 2.2: "e" */ 1413 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1414 wg_algo_generate_keypair(pubkey, privkey); 1415 /* Ci := KDF1(Ci, Ei^pub) */ 1416 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1417 /* msg.ephemeral := Ei^pub */ 1418 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1419 /* Hi := HASH(Hi || msg.ephemeral) */ 1420 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1421 1422 WG_DUMP_HASH("ckey", ckey); 1423 WG_DUMP_HASH("hash", hash); 1424 1425 /* [N] 2.2: "es" */ 1426 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1427 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1428 1429 /* [N] 2.2: "s" */ 1430 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1431 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1432 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1433 hash, sizeof(hash)); 1434 /* Hi := HASH(Hi || msg.static) */ 1435 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1436 1437 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1438 1439 /* [N] 2.2: "ss" */ 1440 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1441 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1442 1443 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1444 wg_timestamp_t timestamp; 1445 wg_algo_tai64n(timestamp); 1446 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1447 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1448 /* Hi := HASH(Hi || msg.timestamp) */ 1449 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1450 1451 /* [W] 5.4.4 Cookie MACs */ 1452 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1453 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1454 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1455 /* Need mac1 to decrypt a cookie from a cookie message */ 1456 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1457 sizeof(wgp->wgp_last_sent_mac1)); 1458 wgp->wgp_last_sent_mac1_valid = true; 1459 1460 if (wgp->wgp_latest_cookie_time == 0 || 1461 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1462 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1463 else { 1464 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1465 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1466 (const uint8_t *)wgmi, 1467 offsetof(struct wg_msg_init, wgmi_mac2), 1468 NULL, 0); 1469 } 1470 1471 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1472 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1473 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1474 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1475 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1476 } 1477 1478 static void __noinline 1479 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1480 const struct sockaddr *src) 1481 { 1482 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1483 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1484 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1485 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1486 struct wg_peer *wgp; 1487 struct wg_session *wgs; 1488 int error, ret; 1489 struct psref psref_peer; 1490 uint8_t mac1[WG_MAC_LEN]; 1491 1492 WG_TRACE("init msg received"); 1493 1494 wg_algo_mac_mac1(mac1, sizeof(mac1), 1495 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1496 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1497 1498 /* 1499 * [W] 5.3: Denial of Service Mitigation & Cookies 1500 * "the responder, ..., must always reject messages with an invalid 1501 * msg.mac1" 1502 */ 1503 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1504 WG_DLOG("mac1 is invalid\n"); 1505 return; 1506 } 1507 1508 /* 1509 * [W] 5.4.2: First Message: Initiator to Responder 1510 * "When the responder receives this message, it does the same 1511 * operations so that its final state variables are identical, 1512 * replacing the operands of the DH function to produce equivalent 1513 * values." 1514 * Note that the following comments of operations are just copies of 1515 * the initiator's ones. 1516 */ 1517 1518 /* Ci := HASH(CONSTRUCTION) */ 1519 /* Hi := HASH(Ci || IDENTIFIER) */ 1520 wg_init_key_and_hash(ckey, hash); 1521 /* Hi := HASH(Hi || Sr^pub) */ 1522 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1523 1524 /* [N] 2.2: "e" */ 1525 /* Ci := KDF1(Ci, Ei^pub) */ 1526 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1527 sizeof(wgmi->wgmi_ephemeral)); 1528 /* Hi := HASH(Hi || msg.ephemeral) */ 1529 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1530 1531 WG_DUMP_HASH("ckey", ckey); 1532 1533 /* [N] 2.2: "es" */ 1534 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1535 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1536 1537 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1538 1539 /* [N] 2.2: "s" */ 1540 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1541 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1542 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1543 if (error != 0) { 1544 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1545 "%s: wg_algo_aead_dec for secret key failed\n", 1546 if_name(&wg->wg_if)); 1547 return; 1548 } 1549 /* Hi := HASH(Hi || msg.static) */ 1550 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1551 1552 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1553 if (wgp == NULL) { 1554 WG_DLOG("peer not found\n"); 1555 return; 1556 } 1557 1558 /* 1559 * Lock the peer to serialize access to cookie state. 1560 * 1561 * XXX Can we safely avoid holding the lock across DH? Take it 1562 * just to verify mac2 and then unlock/DH/lock? 1563 */ 1564 mutex_enter(wgp->wgp_lock); 1565 1566 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1567 WG_TRACE("under load"); 1568 /* 1569 * [W] 5.3: Denial of Service Mitigation & Cookies 1570 * "the responder, ..., and when under load may reject messages 1571 * with an invalid msg.mac2. If the responder receives a 1572 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1573 * and is under load, it may respond with a cookie reply 1574 * message" 1575 */ 1576 uint8_t zero[WG_MAC_LEN] = {0}; 1577 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1578 WG_TRACE("sending a cookie message: no cookie included"); 1579 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1580 wgmi->wgmi_mac1, src); 1581 goto out; 1582 } 1583 if (!wgp->wgp_last_sent_cookie_valid) { 1584 WG_TRACE("sending a cookie message: no cookie sent ever"); 1585 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1586 wgmi->wgmi_mac1, src); 1587 goto out; 1588 } 1589 uint8_t mac2[WG_MAC_LEN]; 1590 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1591 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1592 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1593 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1594 WG_DLOG("mac2 is invalid\n"); 1595 goto out; 1596 } 1597 WG_TRACE("under load, but continue to sending"); 1598 } 1599 1600 /* [N] 2.2: "ss" */ 1601 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1602 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1603 1604 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1605 wg_timestamp_t timestamp; 1606 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1607 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1608 hash, sizeof(hash)); 1609 if (error != 0) { 1610 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1611 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1612 if_name(&wg->wg_if), wgp->wgp_name); 1613 goto out; 1614 } 1615 /* Hi := HASH(Hi || msg.timestamp) */ 1616 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1617 1618 /* 1619 * [W] 5.1 "The responder keeps track of the greatest timestamp 1620 * received per peer and discards packets containing 1621 * timestamps less than or equal to it." 1622 */ 1623 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1624 sizeof(timestamp)); 1625 if (ret <= 0) { 1626 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1627 "%s: peer %s: invalid init msg: timestamp is old\n", 1628 if_name(&wg->wg_if), wgp->wgp_name); 1629 goto out; 1630 } 1631 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1632 1633 /* 1634 * Message is good -- we're committing to handle it now, unless 1635 * we were already initiating a session. 1636 */ 1637 wgs = wgp->wgp_session_unstable; 1638 switch (wgs->wgs_state) { 1639 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1640 break; 1641 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1642 /* XXX Who wins if both sides send INIT? */ 1643 WG_TRACE("Session already initializing, ignoring the message"); 1644 goto out; 1645 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1646 WG_TRACE("Session already initializing, destroying old states"); 1647 /* 1648 * XXX Avoid this -- just resend our response -- if the 1649 * INIT message is identical to the previous one. 1650 */ 1651 wg_put_session_index(wg, wgs); 1652 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1653 wgs->wgs_state); 1654 break; 1655 case WGS_STATE_ESTABLISHED: /* can't happen */ 1656 panic("unstable session can't be established"); 1657 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1658 WG_TRACE("Session destroying, but force to clear"); 1659 wg_put_session_index(wg, wgs); 1660 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1661 wgs->wgs_state); 1662 break; 1663 default: 1664 panic("invalid session state: %d", wgs->wgs_state); 1665 } 1666 1667 /* 1668 * Assign a fresh session index. 1669 */ 1670 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1671 wgs->wgs_state); 1672 wg_get_session_index(wg, wgs); 1673 1674 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1675 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1676 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1677 sizeof(wgmi->wgmi_ephemeral)); 1678 1679 wg_update_endpoint_if_necessary(wgp, src); 1680 1681 /* 1682 * Count the time of the INIT message as the time of 1683 * establishment -- this is used to decide when to erase keys, 1684 * and we want to start counting as soon as we have generated 1685 * keys. 1686 * 1687 * No need for atomic store because the session can't be used 1688 * in the rx or tx paths yet -- not until we transition to 1689 * INTI_PASSIVE. 1690 */ 1691 wgs->wgs_time_established = time_uptime32; 1692 wg_schedule_session_dtor_timer(wgp); 1693 1694 /* 1695 * Respond to the initiator with our ephemeral public key. 1696 */ 1697 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1698 1699 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]:" 1700 " calculate keys as responder\n", 1701 wgs->wgs_local_index, wgs->wgs_remote_index); 1702 wg_calculate_keys(wgs, false); 1703 wg_clear_states(wgs); 1704 1705 /* 1706 * Session is ready to receive data now that we have received 1707 * the peer initiator's ephemeral key pair, generated our 1708 * responder's ephemeral key pair, and derived a session key. 1709 * 1710 * Transition from UNKNOWN to INIT_PASSIVE to publish it to the 1711 * data rx path, wg_handle_msg_data, where the 1712 * atomic_load_acquire matching this atomic_store_release 1713 * happens. 1714 * 1715 * (Session is not, however, ready to send data until the peer 1716 * has acknowledged our response by sending its first data 1717 * packet. So don't swap the sessions yet.) 1718 */ 1719 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_INIT_PASSIVE\n", 1720 wgs->wgs_local_index, wgs->wgs_remote_index); 1721 atomic_store_release(&wgs->wgs_state, WGS_STATE_INIT_PASSIVE); 1722 WG_TRACE("WGS_STATE_INIT_PASSIVE"); 1723 1724 out: 1725 mutex_exit(wgp->wgp_lock); 1726 wg_put_peer(wgp, &psref_peer); 1727 } 1728 1729 static struct socket * 1730 wg_get_so_by_af(struct wg_softc *wg, const int af) 1731 { 1732 1733 switch (af) { 1734 #ifdef INET 1735 case AF_INET: 1736 return wg->wg_so4; 1737 #endif 1738 #ifdef INET6 1739 case AF_INET6: 1740 return wg->wg_so6; 1741 #endif 1742 default: 1743 panic("wg: no such af: %d", af); 1744 } 1745 } 1746 1747 static struct socket * 1748 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1749 { 1750 1751 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1752 } 1753 1754 static struct wg_sockaddr * 1755 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1756 { 1757 struct wg_sockaddr *wgsa; 1758 int s; 1759 1760 s = pserialize_read_enter(); 1761 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1762 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1763 pserialize_read_exit(s); 1764 1765 return wgsa; 1766 } 1767 1768 static void 1769 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1770 { 1771 1772 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1773 } 1774 1775 static int 1776 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1777 { 1778 int error; 1779 struct socket *so; 1780 struct psref psref; 1781 struct wg_sockaddr *wgsa; 1782 1783 wgsa = wg_get_endpoint_sa(wgp, &psref); 1784 so = wg_get_so_by_peer(wgp, wgsa); 1785 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1786 wg_put_sa(wgp, wgsa, &psref); 1787 1788 return error; 1789 } 1790 1791 static int 1792 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1793 { 1794 int error; 1795 struct mbuf *m; 1796 struct wg_msg_init *wgmi; 1797 struct wg_session *wgs; 1798 1799 KASSERT(mutex_owned(wgp->wgp_lock)); 1800 1801 wgs = wgp->wgp_session_unstable; 1802 /* XXX pull dispatch out into wg_task_send_init_message */ 1803 switch (wgs->wgs_state) { 1804 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1805 break; 1806 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1807 WG_TRACE("Session already initializing, skip starting new one"); 1808 return EBUSY; 1809 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1810 WG_TRACE("Session already initializing, waiting for peer"); 1811 return EBUSY; 1812 case WGS_STATE_ESTABLISHED: /* can't happen */ 1813 panic("unstable session can't be established"); 1814 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1815 WG_TRACE("Session destroying"); 1816 wg_put_session_index(wg, wgs); 1817 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1818 wgs->wgs_state); 1819 break; 1820 } 1821 1822 /* 1823 * Assign a fresh session index. 1824 */ 1825 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1826 wgs->wgs_state); 1827 wg_get_session_index(wg, wgs); 1828 1829 /* 1830 * We have initiated a session. Transition to INIT_ACTIVE. 1831 * This doesn't publish it for use in the data rx path, 1832 * wg_handle_msg_data, or in the data tx path, wg_output -- we 1833 * have to wait for the peer to respond with their ephemeral 1834 * public key before we can derive a session key for tx/rx. 1835 * Hence only atomic_store_relaxed. 1836 */ 1837 WG_DLOG("session[L=%"PRIx32" R=(unknown)] -> WGS_STATE_INIT_ACTIVE\n", 1838 wgs->wgs_local_index); 1839 atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_INIT_ACTIVE); 1840 1841 m = m_gethdr(M_WAIT, MT_DATA); 1842 if (sizeof(*wgmi) > MHLEN) { 1843 m_clget(m, M_WAIT); 1844 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1845 } 1846 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1847 wgmi = mtod(m, struct wg_msg_init *); 1848 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1849 1850 error = wg->wg_ops->send_hs_msg(wgp, m); 1851 if (error == 0) { 1852 WG_TRACE("init msg sent"); 1853 1854 if (wgp->wgp_handshake_start_time == 0) 1855 wgp->wgp_handshake_start_time = time_uptime; 1856 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1857 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1858 } else { 1859 wg_put_session_index(wg, wgs); 1860 /* Initiation failed; toss packet waiting for it if any. */ 1861 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 1862 m_freem(m); 1863 } 1864 1865 return error; 1866 } 1867 1868 static void 1869 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1870 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1871 const struct wg_msg_init *wgmi) 1872 { 1873 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1874 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1875 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1876 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1877 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1878 1879 KASSERT(mutex_owned(wgp->wgp_lock)); 1880 KASSERT(wgs == wgp->wgp_session_unstable); 1881 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1882 wgs->wgs_state); 1883 1884 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1885 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1886 1887 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1888 wgmr->wgmr_sender = wgs->wgs_local_index; 1889 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1890 1891 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1892 1893 /* [N] 2.2: "e" */ 1894 /* Er^priv, Er^pub := DH-GENERATE() */ 1895 wg_algo_generate_keypair(pubkey, privkey); 1896 /* Cr := KDF1(Cr, Er^pub) */ 1897 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1898 /* msg.ephemeral := Er^pub */ 1899 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1900 /* Hr := HASH(Hr || msg.ephemeral) */ 1901 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1902 1903 WG_DUMP_HASH("ckey", ckey); 1904 WG_DUMP_HASH("hash", hash); 1905 1906 /* [N] 2.2: "ee" */ 1907 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1908 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1909 1910 /* [N] 2.2: "se" */ 1911 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1912 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1913 1914 /* [N] 9.2: "psk" */ 1915 { 1916 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1917 /* Cr, r, k := KDF3(Cr, Q) */ 1918 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1919 sizeof(wgp->wgp_psk)); 1920 /* Hr := HASH(Hr || r) */ 1921 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1922 } 1923 1924 /* msg.empty := AEAD(k, 0, e, Hr) */ 1925 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1926 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1927 /* Hr := HASH(Hr || msg.empty) */ 1928 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1929 1930 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1931 1932 /* [W] 5.4.4: Cookie MACs */ 1933 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1934 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1935 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1936 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1937 /* Need mac1 to decrypt a cookie from a cookie message */ 1938 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1939 sizeof(wgp->wgp_last_sent_mac1)); 1940 wgp->wgp_last_sent_mac1_valid = true; 1941 1942 if (wgp->wgp_latest_cookie_time == 0 || 1943 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1944 /* msg.mac2 := 0^16 */ 1945 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1946 else { 1947 /* msg.mac2 := MAC(Lm, msg_b) */ 1948 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1949 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1950 (const uint8_t *)wgmr, 1951 offsetof(struct wg_msg_resp, wgmr_mac2), 1952 NULL, 0); 1953 } 1954 1955 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1956 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1957 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1958 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1959 wgs->wgs_remote_index = wgmi->wgmi_sender; 1960 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1961 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1962 } 1963 1964 static void 1965 wg_swap_sessions(struct wg_peer *wgp) 1966 { 1967 struct wg_session *wgs, *wgs_prev; 1968 1969 KASSERT(mutex_owned(wgp->wgp_lock)); 1970 1971 wgs = wgp->wgp_session_unstable; 1972 KASSERTMSG(wgs->wgs_state == WGS_STATE_ESTABLISHED, "state=%d", 1973 wgs->wgs_state); 1974 1975 wgs_prev = wgp->wgp_session_stable; 1976 KASSERTMSG((wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1977 wgs_prev->wgs_state == WGS_STATE_UNKNOWN), 1978 "state=%d", wgs_prev->wgs_state); 1979 atomic_store_release(&wgp->wgp_session_stable, wgs); 1980 wgp->wgp_session_unstable = wgs_prev; 1981 } 1982 1983 static void __noinline 1984 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1985 const struct sockaddr *src) 1986 { 1987 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1988 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1989 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1990 struct wg_peer *wgp; 1991 struct wg_session *wgs; 1992 struct psref psref; 1993 int error; 1994 uint8_t mac1[WG_MAC_LEN]; 1995 struct wg_session *wgs_prev; 1996 struct mbuf *m; 1997 1998 wg_algo_mac_mac1(mac1, sizeof(mac1), 1999 wg->wg_pubkey, sizeof(wg->wg_pubkey), 2000 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 2001 2002 /* 2003 * [W] 5.3: Denial of Service Mitigation & Cookies 2004 * "the responder, ..., must always reject messages with an invalid 2005 * msg.mac1" 2006 */ 2007 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 2008 WG_DLOG("mac1 is invalid\n"); 2009 return; 2010 } 2011 2012 WG_TRACE("resp msg received"); 2013 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 2014 if (wgs == NULL) { 2015 WG_TRACE("No session found"); 2016 return; 2017 } 2018 2019 wgp = wgs->wgs_peer; 2020 2021 mutex_enter(wgp->wgp_lock); 2022 2023 /* If we weren't waiting for a handshake response, drop it. */ 2024 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 2025 WG_TRACE("peer sent spurious handshake response, ignoring"); 2026 goto out; 2027 } 2028 2029 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 2030 WG_TRACE("under load"); 2031 /* 2032 * [W] 5.3: Denial of Service Mitigation & Cookies 2033 * "the responder, ..., and when under load may reject messages 2034 * with an invalid msg.mac2. If the responder receives a 2035 * message with a valid msg.mac1 yet with an invalid msg.mac2, 2036 * and is under load, it may respond with a cookie reply 2037 * message" 2038 */ 2039 uint8_t zero[WG_MAC_LEN] = {0}; 2040 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 2041 WG_TRACE("sending a cookie message: no cookie included"); 2042 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 2043 wgmr->wgmr_mac1, src); 2044 goto out; 2045 } 2046 if (!wgp->wgp_last_sent_cookie_valid) { 2047 WG_TRACE("sending a cookie message: no cookie sent ever"); 2048 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 2049 wgmr->wgmr_mac1, src); 2050 goto out; 2051 } 2052 uint8_t mac2[WG_MAC_LEN]; 2053 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 2054 WG_COOKIE_LEN, (const uint8_t *)wgmr, 2055 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 2056 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 2057 WG_DLOG("mac2 is invalid\n"); 2058 goto out; 2059 } 2060 WG_TRACE("under load, but continue to sending"); 2061 } 2062 2063 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 2064 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 2065 2066 /* 2067 * [W] 5.4.3 Second Message: Responder to Initiator 2068 * "When the initiator receives this message, it does the same 2069 * operations so that its final state variables are identical, 2070 * replacing the operands of the DH function to produce equivalent 2071 * values." 2072 * Note that the following comments of operations are just copies of 2073 * the initiator's ones. 2074 */ 2075 2076 /* [N] 2.2: "e" */ 2077 /* Cr := KDF1(Cr, Er^pub) */ 2078 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 2079 sizeof(wgmr->wgmr_ephemeral)); 2080 /* Hr := HASH(Hr || msg.ephemeral) */ 2081 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 2082 2083 WG_DUMP_HASH("ckey", ckey); 2084 WG_DUMP_HASH("hash", hash); 2085 2086 /* [N] 2.2: "ee" */ 2087 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 2088 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 2089 wgmr->wgmr_ephemeral); 2090 2091 /* [N] 2.2: "se" */ 2092 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 2093 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 2094 2095 /* [N] 9.2: "psk" */ 2096 { 2097 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 2098 /* Cr, r, k := KDF3(Cr, Q) */ 2099 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 2100 sizeof(wgp->wgp_psk)); 2101 /* Hr := HASH(Hr || r) */ 2102 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 2103 } 2104 2105 { 2106 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 2107 /* msg.empty := AEAD(k, 0, e, Hr) */ 2108 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 2109 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 2110 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 2111 if (error != 0) { 2112 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2113 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 2114 if_name(&wg->wg_if), wgp->wgp_name); 2115 goto out; 2116 } 2117 /* Hr := HASH(Hr || msg.empty) */ 2118 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 2119 } 2120 2121 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 2122 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 2123 wgs->wgs_remote_index = wgmr->wgmr_sender; 2124 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 2125 2126 KASSERTMSG(wgs->wgs_state == WGS_STATE_INIT_ACTIVE, "state=%d", 2127 wgs->wgs_state); 2128 wgs->wgs_time_established = time_uptime32; 2129 wg_schedule_session_dtor_timer(wgp); 2130 wgs->wgs_time_last_data_sent = 0; 2131 wgs->wgs_is_initiator = true; 2132 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]:" 2133 " calculate keys as initiator\n", 2134 wgs->wgs_local_index, wgs->wgs_remote_index); 2135 wg_calculate_keys(wgs, true); 2136 wg_clear_states(wgs); 2137 2138 /* 2139 * Session is ready to receive data now that we have received 2140 * the responder's response. 2141 * 2142 * Transition from INIT_ACTIVE to ESTABLISHED to publish it to 2143 * the data rx path, wg_handle_msg_data. 2144 */ 2145 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32" -> WGS_STATE_ESTABLISHED\n", 2146 wgs->wgs_local_index, wgs->wgs_remote_index); 2147 atomic_store_release(&wgs->wgs_state, WGS_STATE_ESTABLISHED); 2148 WG_TRACE("WGS_STATE_ESTABLISHED"); 2149 2150 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 2151 2152 /* 2153 * Session is ready to send data now that we have received the 2154 * responder's response. 2155 * 2156 * Swap the sessions to publish the new one as the stable 2157 * session for the data tx path, wg_output. 2158 */ 2159 wg_swap_sessions(wgp); 2160 KASSERT(wgs == wgp->wgp_session_stable); 2161 wgs_prev = wgp->wgp_session_unstable; 2162 getnanotime(&wgp->wgp_last_handshake_time); 2163 wgp->wgp_handshake_start_time = 0; 2164 wgp->wgp_last_sent_mac1_valid = false; 2165 wgp->wgp_last_sent_cookie_valid = false; 2166 2167 wg_update_endpoint_if_necessary(wgp, src); 2168 2169 /* 2170 * If we had a data packet queued up, send it; otherwise send a 2171 * keepalive message -- either way we have to send something 2172 * immediately or else the responder will never answer. 2173 */ 2174 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2175 kpreempt_disable(); 2176 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2177 M_SETCTX(m, wgp); 2178 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2179 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2180 if_name(&wg->wg_if)); 2181 m_freem(m); 2182 } 2183 kpreempt_enable(); 2184 } else { 2185 wg_send_keepalive_msg(wgp, wgs); 2186 } 2187 2188 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2189 /* 2190 * Transition ESTABLISHED->DESTROYING. The session 2191 * will remain usable for the data rx path to process 2192 * packets still in flight to us, but we won't use it 2193 * for data tx. 2194 */ 2195 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]" 2196 " -> WGS_STATE_DESTROYING\n", 2197 wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index); 2198 atomic_store_relaxed(&wgs_prev->wgs_state, 2199 WGS_STATE_DESTROYING); 2200 } else { 2201 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2202 "state=%d", wgs_prev->wgs_state); 2203 } 2204 2205 out: 2206 mutex_exit(wgp->wgp_lock); 2207 wg_put_session(wgs, &psref); 2208 } 2209 2210 static int 2211 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2212 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2213 { 2214 int error; 2215 struct mbuf *m; 2216 struct wg_msg_resp *wgmr; 2217 2218 KASSERT(mutex_owned(wgp->wgp_lock)); 2219 KASSERT(wgs == wgp->wgp_session_unstable); 2220 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 2221 wgs->wgs_state); 2222 2223 m = m_gethdr(M_WAIT, MT_DATA); 2224 if (sizeof(*wgmr) > MHLEN) { 2225 m_clget(m, M_WAIT); 2226 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2227 } 2228 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2229 wgmr = mtod(m, struct wg_msg_resp *); 2230 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2231 2232 error = wg->wg_ops->send_hs_msg(wgp, m); 2233 if (error == 0) 2234 WG_TRACE("resp msg sent"); 2235 return error; 2236 } 2237 2238 static struct wg_peer * 2239 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2240 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2241 { 2242 struct wg_peer *wgp; 2243 2244 int s = pserialize_read_enter(); 2245 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2246 if (wgp != NULL) 2247 wg_get_peer(wgp, psref); 2248 pserialize_read_exit(s); 2249 2250 return wgp; 2251 } 2252 2253 static void 2254 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2255 struct wg_msg_cookie *wgmc, const uint32_t sender, 2256 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2257 { 2258 uint8_t cookie[WG_COOKIE_LEN]; 2259 uint8_t key[WG_HASH_LEN]; 2260 uint8_t addr[sizeof(struct in6_addr)]; 2261 size_t addrlen; 2262 uint16_t uh_sport; /* be */ 2263 2264 KASSERT(mutex_owned(wgp->wgp_lock)); 2265 2266 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2267 wgmc->wgmc_receiver = sender; 2268 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2269 2270 /* 2271 * [W] 5.4.7: Under Load: Cookie Reply Message 2272 * "The secret variable, Rm, changes every two minutes to a 2273 * random value" 2274 */ 2275 if ((time_uptime - wgp->wgp_last_cookiesecret_time) > 2276 WG_COOKIESECRET_TIME) { 2277 cprng_strong(kern_cprng, wgp->wgp_cookiesecret, 2278 sizeof(wgp->wgp_cookiesecret), 0); 2279 wgp->wgp_last_cookiesecret_time = time_uptime; 2280 } 2281 2282 switch (src->sa_family) { 2283 case AF_INET: { 2284 const struct sockaddr_in *sin = satocsin(src); 2285 addrlen = sizeof(sin->sin_addr); 2286 memcpy(addr, &sin->sin_addr, addrlen); 2287 uh_sport = sin->sin_port; 2288 break; 2289 } 2290 #ifdef INET6 2291 case AF_INET6: { 2292 const struct sockaddr_in6 *sin6 = satocsin6(src); 2293 addrlen = sizeof(sin6->sin6_addr); 2294 memcpy(addr, &sin6->sin6_addr, addrlen); 2295 uh_sport = sin6->sin6_port; 2296 break; 2297 } 2298 #endif 2299 default: 2300 panic("invalid af=%d", src->sa_family); 2301 } 2302 2303 wg_algo_mac(cookie, sizeof(cookie), 2304 wgp->wgp_cookiesecret, sizeof(wgp->wgp_cookiesecret), 2305 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2306 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2307 sizeof(wg->wg_pubkey)); 2308 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2309 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2310 2311 /* Need to store to calculate mac2 */ 2312 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2313 wgp->wgp_last_sent_cookie_valid = true; 2314 } 2315 2316 static int 2317 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2318 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2319 const struct sockaddr *src) 2320 { 2321 int error; 2322 struct mbuf *m; 2323 struct wg_msg_cookie *wgmc; 2324 2325 KASSERT(mutex_owned(wgp->wgp_lock)); 2326 2327 m = m_gethdr(M_WAIT, MT_DATA); 2328 if (sizeof(*wgmc) > MHLEN) { 2329 m_clget(m, M_WAIT); 2330 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2331 } 2332 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2333 wgmc = mtod(m, struct wg_msg_cookie *); 2334 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2335 2336 error = wg->wg_ops->send_hs_msg(wgp, m); 2337 if (error == 0) 2338 WG_TRACE("cookie msg sent"); 2339 return error; 2340 } 2341 2342 static bool 2343 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2344 { 2345 #ifdef WG_DEBUG_PARAMS 2346 if (wg_force_underload) 2347 return true; 2348 #endif 2349 2350 /* 2351 * XXX we don't have a means of a load estimation. The purpose of 2352 * the mechanism is a DoS mitigation, so we consider frequent handshake 2353 * messages as (a kind of) load; if a message of the same type comes 2354 * to a peer within 1 second, we consider we are under load. 2355 */ 2356 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2357 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2358 return (time_uptime - last) == 0; 2359 } 2360 2361 static void 2362 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2363 { 2364 2365 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2366 2367 /* 2368 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2369 */ 2370 if (initiator) { 2371 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2372 wgs->wgs_chaining_key, NULL, 0); 2373 } else { 2374 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2375 wgs->wgs_chaining_key, NULL, 0); 2376 } 2377 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2378 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2379 } 2380 2381 static uint64_t 2382 wg_session_get_send_counter(struct wg_session *wgs) 2383 { 2384 #ifdef __HAVE_ATOMIC64_LOADSTORE 2385 return atomic_load_relaxed(&wgs->wgs_send_counter); 2386 #else 2387 uint64_t send_counter; 2388 2389 mutex_enter(&wgs->wgs_send_counter_lock); 2390 send_counter = wgs->wgs_send_counter; 2391 mutex_exit(&wgs->wgs_send_counter_lock); 2392 2393 return send_counter; 2394 #endif 2395 } 2396 2397 static uint64_t 2398 wg_session_inc_send_counter(struct wg_session *wgs) 2399 { 2400 #ifdef __HAVE_ATOMIC64_LOADSTORE 2401 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2402 #else 2403 uint64_t send_counter; 2404 2405 mutex_enter(&wgs->wgs_send_counter_lock); 2406 send_counter = wgs->wgs_send_counter++; 2407 mutex_exit(&wgs->wgs_send_counter_lock); 2408 2409 return send_counter; 2410 #endif 2411 } 2412 2413 static void 2414 wg_clear_states(struct wg_session *wgs) 2415 { 2416 2417 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2418 2419 wgs->wgs_send_counter = 0; 2420 sliwin_reset(&wgs->wgs_recvwin->window); 2421 2422 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2423 wgs_clear(handshake_hash); 2424 wgs_clear(chaining_key); 2425 wgs_clear(ephemeral_key_pub); 2426 wgs_clear(ephemeral_key_priv); 2427 wgs_clear(ephemeral_key_peer); 2428 #undef wgs_clear 2429 } 2430 2431 static struct wg_session * 2432 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2433 struct psref *psref) 2434 { 2435 struct wg_session *wgs; 2436 2437 int s = pserialize_read_enter(); 2438 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2439 if (wgs != NULL) { 2440 uint32_t oindex __diagused = 2441 atomic_load_relaxed(&wgs->wgs_local_index); 2442 KASSERTMSG(index == oindex, 2443 "index=%"PRIx32" wgs->wgs_local_index=%"PRIx32, 2444 index, oindex); 2445 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2446 } 2447 pserialize_read_exit(s); 2448 2449 return wgs; 2450 } 2451 2452 static void 2453 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2454 { 2455 struct mbuf *m; 2456 2457 /* 2458 * [W] 6.5 Passive Keepalive 2459 * "A keepalive message is simply a transport data message with 2460 * a zero-length encapsulated encrypted inner-packet." 2461 */ 2462 WG_TRACE(""); 2463 m = m_gethdr(M_WAIT, MT_DATA); 2464 wg_send_data_msg(wgp, wgs, m); 2465 } 2466 2467 static bool 2468 wg_need_to_send_init_message(struct wg_session *wgs) 2469 { 2470 /* 2471 * [W] 6.2 Transport Message Limits 2472 * "if a peer is the initiator of a current secure session, 2473 * WireGuard will send a handshake initiation message to begin 2474 * a new secure session ... if after receiving a transport data 2475 * message, the current secure session is (REJECT-AFTER-TIME 2476 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2477 * not yet acted upon this event." 2478 */ 2479 return wgs->wgs_is_initiator && 2480 atomic_load_relaxed(&wgs->wgs_time_last_data_sent) == 0 && 2481 ((time_uptime32 - 2482 atomic_load_relaxed(&wgs->wgs_time_established)) >= 2483 (wg_reject_after_time - wg_keepalive_timeout - 2484 wg_rekey_timeout)); 2485 } 2486 2487 static void 2488 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2489 { 2490 2491 mutex_enter(wgp->wgp_intr_lock); 2492 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2493 if (wgp->wgp_tasks == 0) 2494 /* 2495 * XXX If the current CPU is already loaded -- e.g., if 2496 * there's already a bunch of handshakes queued up -- 2497 * consider tossing this over to another CPU to 2498 * distribute the load. 2499 */ 2500 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2501 wgp->wgp_tasks |= task; 2502 mutex_exit(wgp->wgp_intr_lock); 2503 } 2504 2505 static void 2506 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2507 { 2508 struct wg_sockaddr *wgsa_prev; 2509 2510 WG_TRACE("Changing endpoint"); 2511 2512 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2513 wgsa_prev = wgp->wgp_endpoint; 2514 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2515 wgp->wgp_endpoint0 = wgsa_prev; 2516 atomic_store_release(&wgp->wgp_endpoint_available, true); 2517 2518 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2519 } 2520 2521 static bool 2522 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2523 { 2524 uint16_t packet_len; 2525 const struct ip *ip; 2526 2527 if (__predict_false(decrypted_len < sizeof(*ip))) { 2528 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2529 sizeof(*ip)); 2530 return false; 2531 } 2532 2533 ip = (const struct ip *)packet; 2534 if (ip->ip_v == 4) 2535 *af = AF_INET; 2536 else if (ip->ip_v == 6) 2537 *af = AF_INET6; 2538 else { 2539 WG_DLOG("ip_v=%d\n", ip->ip_v); 2540 return false; 2541 } 2542 2543 WG_DLOG("af=%d\n", *af); 2544 2545 switch (*af) { 2546 #ifdef INET 2547 case AF_INET: 2548 packet_len = ntohs(ip->ip_len); 2549 break; 2550 #endif 2551 #ifdef INET6 2552 case AF_INET6: { 2553 const struct ip6_hdr *ip6; 2554 2555 if (__predict_false(decrypted_len < sizeof(*ip6))) { 2556 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2557 sizeof(*ip6)); 2558 return false; 2559 } 2560 2561 ip6 = (const struct ip6_hdr *)packet; 2562 packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); 2563 break; 2564 } 2565 #endif 2566 default: 2567 return false; 2568 } 2569 2570 if (packet_len > decrypted_len) { 2571 WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len, 2572 decrypted_len); 2573 return false; 2574 } 2575 2576 return true; 2577 } 2578 2579 static bool 2580 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2581 int af, char *packet) 2582 { 2583 struct sockaddr_storage ss; 2584 struct sockaddr *sa; 2585 struct psref psref; 2586 struct wg_peer *wgp; 2587 bool ok; 2588 2589 /* 2590 * II CRYPTOKEY ROUTING 2591 * "it will only accept it if its source IP resolves in the 2592 * table to the public key used in the secure session for 2593 * decrypting it." 2594 */ 2595 2596 if (af == AF_INET) { 2597 const struct ip *ip = (const struct ip *)packet; 2598 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2599 sockaddr_in_init(sin, &ip->ip_src, 0); 2600 sa = sintosa(sin); 2601 #ifdef INET6 2602 } else { 2603 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2604 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2605 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2606 sa = sin6tosa(sin6); 2607 #endif 2608 } 2609 2610 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2611 ok = (wgp == wgp_expected); 2612 if (wgp != NULL) 2613 wg_put_peer(wgp, &psref); 2614 2615 return ok; 2616 } 2617 2618 static void 2619 wg_session_dtor_timer(void *arg) 2620 { 2621 struct wg_peer *wgp = arg; 2622 2623 WG_TRACE("enter"); 2624 2625 wg_schedule_session_dtor_timer(wgp); 2626 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2627 } 2628 2629 static void 2630 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2631 { 2632 2633 /* 2634 * If the periodic session destructor is already pending to 2635 * handle the previous session, that's fine -- leave it in 2636 * place; it will be scheduled again. 2637 */ 2638 if (callout_pending(&wgp->wgp_session_dtor_timer)) { 2639 WG_DLOG("session dtor already pending\n"); 2640 return; 2641 } 2642 2643 WG_DLOG("scheduling session dtor in %u secs\n", wg_reject_after_time); 2644 callout_schedule(&wgp->wgp_session_dtor_timer, 2645 wg_reject_after_time*hz); 2646 } 2647 2648 static bool 2649 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2650 { 2651 if (sa1->sa_family != sa2->sa_family) 2652 return false; 2653 2654 switch (sa1->sa_family) { 2655 #ifdef INET 2656 case AF_INET: 2657 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2658 #endif 2659 #ifdef INET6 2660 case AF_INET6: 2661 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2662 #endif 2663 default: 2664 return false; 2665 } 2666 } 2667 2668 static void 2669 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2670 const struct sockaddr *src) 2671 { 2672 struct wg_sockaddr *wgsa; 2673 struct psref psref; 2674 2675 wgsa = wg_get_endpoint_sa(wgp, &psref); 2676 2677 #ifdef WG_DEBUG_LOG 2678 char oldaddr[128], newaddr[128]; 2679 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2680 sockaddr_format(src, newaddr, sizeof(newaddr)); 2681 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2682 #endif 2683 2684 /* 2685 * III: "Since the packet has authenticated correctly, the source IP of 2686 * the outer UDP/IP packet is used to update the endpoint for peer..." 2687 */ 2688 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2689 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2690 /* XXX We can't change the endpoint twice in a short period */ 2691 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2692 wg_change_endpoint(wgp, src); 2693 } 2694 } 2695 2696 wg_put_sa(wgp, wgsa, &psref); 2697 } 2698 2699 static void __noinline 2700 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2701 const struct sockaddr *src) 2702 { 2703 struct wg_msg_data *wgmd; 2704 char *encrypted_buf = NULL, *decrypted_buf; 2705 size_t encrypted_len, decrypted_len; 2706 struct wg_session *wgs; 2707 struct wg_peer *wgp; 2708 int state; 2709 uint32_t age; 2710 size_t mlen; 2711 struct psref psref; 2712 int error, af; 2713 bool success, free_encrypted_buf = false, ok; 2714 struct mbuf *n; 2715 2716 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2717 wgmd = mtod(m, struct wg_msg_data *); 2718 2719 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2720 WG_TRACE("data"); 2721 2722 /* Find the putative session, or drop. */ 2723 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2724 if (wgs == NULL) { 2725 WG_TRACE("No session found"); 2726 m_freem(m); 2727 return; 2728 } 2729 2730 /* 2731 * We are only ready to handle data when in INIT_PASSIVE, 2732 * ESTABLISHED, or DESTROYING. All transitions out of that 2733 * state dissociate the session index and drain psrefs. 2734 * 2735 * atomic_load_acquire matches atomic_store_release in either 2736 * wg_handle_msg_init or wg_handle_msg_resp. (The transition 2737 * INIT_PASSIVE to ESTABLISHED in wg_task_establish_session 2738 * doesn't make a difference for this rx path.) 2739 */ 2740 state = atomic_load_acquire(&wgs->wgs_state); 2741 switch (state) { 2742 case WGS_STATE_UNKNOWN: 2743 case WGS_STATE_INIT_ACTIVE: 2744 WG_TRACE("not yet ready for data"); 2745 goto out; 2746 case WGS_STATE_INIT_PASSIVE: 2747 case WGS_STATE_ESTABLISHED: 2748 case WGS_STATE_DESTROYING: 2749 break; 2750 } 2751 2752 /* 2753 * Reject if the session is too old. 2754 */ 2755 age = time_uptime32 - atomic_load_relaxed(&wgs->wgs_time_established); 2756 if (__predict_false(age >= wg_reject_after_time)) { 2757 WG_DLOG("session %"PRIx32" too old, %"PRIu32" sec\n", 2758 wgmd->wgmd_receiver, age); 2759 goto out; 2760 } 2761 2762 /* 2763 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2764 * to update the endpoint if authentication succeeds. 2765 */ 2766 wgp = wgs->wgs_peer; 2767 2768 /* 2769 * Reject outrageously wrong sequence numbers before doing any 2770 * crypto work or taking any locks. 2771 */ 2772 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2773 le64toh(wgmd->wgmd_counter)); 2774 if (error) { 2775 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2776 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2777 if_name(&wg->wg_if), wgp->wgp_name, 2778 le64toh(wgmd->wgmd_counter)); 2779 goto out; 2780 } 2781 2782 /* Ensure the payload and authenticator are contiguous. */ 2783 mlen = m_length(m); 2784 encrypted_len = mlen - sizeof(*wgmd); 2785 if (encrypted_len < WG_AUTHTAG_LEN) { 2786 WG_DLOG("Short encrypted_len: %zu\n", encrypted_len); 2787 goto out; 2788 } 2789 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2790 if (success) { 2791 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2792 } else { 2793 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2794 if (encrypted_buf == NULL) { 2795 WG_DLOG("failed to allocate encrypted_buf\n"); 2796 goto out; 2797 } 2798 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2799 free_encrypted_buf = true; 2800 } 2801 /* m_ensure_contig may change m regardless of its result */ 2802 KASSERT(m->m_len >= sizeof(*wgmd)); 2803 wgmd = mtod(m, struct wg_msg_data *); 2804 2805 #ifdef WG_DEBUG_PACKET 2806 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2807 hexdump(printf, "incoming packet", encrypted_buf, 2808 encrypted_len); 2809 } 2810 #endif 2811 /* 2812 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2813 * a zero-length buffer (XXX). Drop if plaintext is longer 2814 * than MCLBYTES (XXX). 2815 */ 2816 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2817 if (decrypted_len > MCLBYTES) { 2818 /* FIXME handle larger data than MCLBYTES */ 2819 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2820 goto out; 2821 } 2822 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2823 if (n == NULL) { 2824 WG_DLOG("wg_get_mbuf failed\n"); 2825 goto out; 2826 } 2827 decrypted_buf = mtod(n, char *); 2828 2829 /* Decrypt and verify the packet. */ 2830 WG_DLOG("mlen=%zu, encrypted_len=%zu\n", mlen, encrypted_len); 2831 error = wg_algo_aead_dec(decrypted_buf, 2832 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2833 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2834 encrypted_len, NULL, 0); 2835 if (error != 0) { 2836 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2837 "%s: peer %s: failed to wg_algo_aead_dec\n", 2838 if_name(&wg->wg_if), wgp->wgp_name); 2839 m_freem(n); 2840 goto out; 2841 } 2842 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2843 2844 /* Packet is genuine. Reject it if a replay or just too old. */ 2845 mutex_enter(&wgs->wgs_recvwin->lock); 2846 error = sliwin_update(&wgs->wgs_recvwin->window, 2847 le64toh(wgmd->wgmd_counter)); 2848 mutex_exit(&wgs->wgs_recvwin->lock); 2849 if (error) { 2850 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2851 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2852 if_name(&wg->wg_if), wgp->wgp_name, 2853 le64toh(wgmd->wgmd_counter)); 2854 m_freem(n); 2855 goto out; 2856 } 2857 2858 #ifdef WG_DEBUG_PACKET 2859 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2860 hexdump(printf, "tkey_recv", wgs->wgs_tkey_recv, 2861 sizeof(wgs->wgs_tkey_recv)); 2862 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 2863 hexdump(printf, "decrypted_buf", decrypted_buf, 2864 decrypted_len); 2865 } 2866 #endif 2867 /* We're done with m now; free it and chuck the pointers. */ 2868 m_freem(m); 2869 m = NULL; 2870 wgmd = NULL; 2871 2872 /* 2873 * The packet is genuine. Update the peer's endpoint if the 2874 * source address changed. 2875 * 2876 * XXX How to prevent DoS by replaying genuine packets from the 2877 * wrong source address? 2878 */ 2879 wg_update_endpoint_if_necessary(wgp, src); 2880 2881 /* 2882 * Validate the encapsulated packet header and get the address 2883 * family, or drop. 2884 */ 2885 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2886 if (!ok) { 2887 m_freem(n); 2888 goto update_state; 2889 } 2890 2891 /* Submit it into our network stack if routable. */ 2892 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2893 if (ok) { 2894 wg->wg_ops->input(&wg->wg_if, n, af); 2895 } else { 2896 char addrstr[INET6_ADDRSTRLEN]; 2897 memset(addrstr, 0, sizeof(addrstr)); 2898 if (af == AF_INET) { 2899 const struct ip *ip = (const struct ip *)decrypted_buf; 2900 IN_PRINT(addrstr, &ip->ip_src); 2901 #ifdef INET6 2902 } else if (af == AF_INET6) { 2903 const struct ip6_hdr *ip6 = 2904 (const struct ip6_hdr *)decrypted_buf; 2905 IN6_PRINT(addrstr, &ip6->ip6_src); 2906 #endif 2907 } 2908 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2909 "%s: peer %s: invalid source address (%s)\n", 2910 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2911 m_freem(n); 2912 /* 2913 * The inner address is invalid however the session is valid 2914 * so continue the session processing below. 2915 */ 2916 } 2917 n = NULL; 2918 2919 update_state: 2920 /* Update the state machine if necessary. */ 2921 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2922 /* 2923 * We were waiting for the initiator to send their 2924 * first data transport message, and that has happened. 2925 * Schedule a task to establish this session. 2926 */ 2927 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2928 } else { 2929 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2930 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2931 } 2932 /* 2933 * [W] 6.5 Passive Keepalive 2934 * "If a peer has received a validly-authenticated transport 2935 * data message (section 5.4.6), but does not have any packets 2936 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2937 * a keepalive message." 2938 */ 2939 const uint32_t now = time_uptime32; 2940 const uint32_t time_last_data_sent = 2941 atomic_load_relaxed(&wgs->wgs_time_last_data_sent); 2942 WG_DLOG("time_uptime32=%"PRIu32 2943 " wgs_time_last_data_sent=%"PRIu32"\n", 2944 now, time_last_data_sent); 2945 if ((now - time_last_data_sent) >= wg_keepalive_timeout) { 2946 WG_TRACE("Schedule sending keepalive message"); 2947 /* 2948 * We can't send a keepalive message here to avoid 2949 * a deadlock; we already hold the solock of a socket 2950 * that is used to send the message. 2951 */ 2952 wg_schedule_peer_task(wgp, 2953 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2954 } 2955 } 2956 out: 2957 wg_put_session(wgs, &psref); 2958 m_freem(m); 2959 if (free_encrypted_buf) 2960 kmem_intr_free(encrypted_buf, encrypted_len); 2961 } 2962 2963 static void __noinline 2964 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2965 { 2966 struct wg_session *wgs; 2967 struct wg_peer *wgp; 2968 struct psref psref; 2969 int error; 2970 uint8_t key[WG_HASH_LEN]; 2971 uint8_t cookie[WG_COOKIE_LEN]; 2972 2973 WG_TRACE("cookie msg received"); 2974 2975 /* Find the putative session. */ 2976 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2977 if (wgs == NULL) { 2978 WG_TRACE("No session found"); 2979 return; 2980 } 2981 2982 /* Lock the peer so we can update the cookie state. */ 2983 wgp = wgs->wgs_peer; 2984 mutex_enter(wgp->wgp_lock); 2985 2986 if (!wgp->wgp_last_sent_mac1_valid) { 2987 WG_TRACE("No valid mac1 sent (or expired)"); 2988 goto out; 2989 } 2990 2991 /* 2992 * wgp_last_sent_mac1_valid is only set to true when we are 2993 * transitioning to INIT_ACTIVE or INIT_PASSIVE, and always 2994 * cleared on transition out of them. 2995 */ 2996 KASSERTMSG((wgs->wgs_state == WGS_STATE_INIT_ACTIVE || 2997 wgs->wgs_state == WGS_STATE_INIT_PASSIVE), 2998 "state=%d", wgs->wgs_state); 2999 3000 /* Decrypt the cookie and store it for later handshake retry. */ 3001 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 3002 sizeof(wgp->wgp_pubkey)); 3003 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 3004 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 3005 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 3006 wgmc->wgmc_salt); 3007 if (error != 0) { 3008 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 3009 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 3010 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 3011 goto out; 3012 } 3013 /* 3014 * [W] 6.6: Interaction with Cookie Reply System 3015 * "it should simply store the decrypted cookie value from the cookie 3016 * reply message, and wait for the expiration of the REKEY-TIMEOUT 3017 * timer for retrying a handshake initiation message." 3018 */ 3019 wgp->wgp_latest_cookie_time = time_uptime; 3020 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 3021 out: 3022 mutex_exit(wgp->wgp_lock); 3023 wg_put_session(wgs, &psref); 3024 } 3025 3026 static struct mbuf * 3027 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 3028 { 3029 struct wg_msg wgm; 3030 size_t mbuflen; 3031 size_t msglen; 3032 3033 /* 3034 * Get the mbuf chain length. It is already guaranteed, by 3035 * wg_overudp_cb, to be large enough for a struct wg_msg. 3036 */ 3037 mbuflen = m_length(m); 3038 KASSERT(mbuflen >= sizeof(struct wg_msg)); 3039 3040 /* 3041 * Copy the message header (32-bit message type) out -- we'll 3042 * worry about contiguity and alignment later. 3043 */ 3044 m_copydata(m, 0, sizeof(wgm), &wgm); 3045 switch (le32toh(wgm.wgm_type)) { 3046 case WG_MSG_TYPE_INIT: 3047 msglen = sizeof(struct wg_msg_init); 3048 break; 3049 case WG_MSG_TYPE_RESP: 3050 msglen = sizeof(struct wg_msg_resp); 3051 break; 3052 case WG_MSG_TYPE_COOKIE: 3053 msglen = sizeof(struct wg_msg_cookie); 3054 break; 3055 case WG_MSG_TYPE_DATA: 3056 msglen = sizeof(struct wg_msg_data); 3057 break; 3058 default: 3059 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 3060 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 3061 le32toh(wgm.wgm_type)); 3062 goto error; 3063 } 3064 3065 /* Verify the mbuf chain is long enough for this type of message. */ 3066 if (__predict_false(mbuflen < msglen)) { 3067 WG_DLOG("Invalid msg size: mbuflen=%zu type=%u\n", mbuflen, 3068 le32toh(wgm.wgm_type)); 3069 goto error; 3070 } 3071 3072 /* Make the message header contiguous if necessary. */ 3073 if (__predict_false(m->m_len < msglen)) { 3074 m = m_pullup(m, msglen); 3075 if (m == NULL) 3076 return NULL; 3077 } 3078 3079 return m; 3080 3081 error: 3082 m_freem(m); 3083 return NULL; 3084 } 3085 3086 static void 3087 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 3088 const struct sockaddr *src) 3089 { 3090 struct wg_msg *wgm; 3091 3092 KASSERT(curlwp->l_pflag & LP_BOUND); 3093 3094 m = wg_validate_msg_header(wg, m); 3095 if (__predict_false(m == NULL)) 3096 return; 3097 3098 KASSERT(m->m_len >= sizeof(struct wg_msg)); 3099 wgm = mtod(m, struct wg_msg *); 3100 switch (le32toh(wgm->wgm_type)) { 3101 case WG_MSG_TYPE_INIT: 3102 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 3103 break; 3104 case WG_MSG_TYPE_RESP: 3105 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 3106 break; 3107 case WG_MSG_TYPE_COOKIE: 3108 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 3109 break; 3110 case WG_MSG_TYPE_DATA: 3111 wg_handle_msg_data(wg, m, src); 3112 /* wg_handle_msg_data frees m for us */ 3113 return; 3114 default: 3115 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 3116 } 3117 3118 m_freem(m); 3119 } 3120 3121 static void 3122 wg_receive_packets(struct wg_softc *wg, const int af) 3123 { 3124 3125 for (;;) { 3126 int error, flags; 3127 struct socket *so; 3128 struct mbuf *m = NULL; 3129 struct uio dummy_uio; 3130 struct mbuf *paddr = NULL; 3131 struct sockaddr *src; 3132 3133 so = wg_get_so_by_af(wg, af); 3134 flags = MSG_DONTWAIT; 3135 dummy_uio.uio_resid = 1000000000; 3136 3137 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 3138 &flags); 3139 if (error || m == NULL) { 3140 //if (error == EWOULDBLOCK) 3141 return; 3142 } 3143 3144 KASSERT(paddr != NULL); 3145 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 3146 src = mtod(paddr, struct sockaddr *); 3147 3148 wg_handle_packet(wg, m, src); 3149 } 3150 } 3151 3152 static void 3153 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 3154 { 3155 3156 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 3157 } 3158 3159 static void 3160 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 3161 { 3162 3163 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 3164 } 3165 3166 static void 3167 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 3168 { 3169 struct wg_session *wgs; 3170 3171 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 3172 3173 KASSERT(mutex_owned(wgp->wgp_lock)); 3174 3175 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 3176 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 3177 if_name(&wg->wg_if)); 3178 /* XXX should do something? */ 3179 return; 3180 } 3181 3182 /* 3183 * If we already have an established session, there's no need 3184 * to initiate a new one -- unless the rekey-after-time or 3185 * rekey-after-messages limits have passed. 3186 */ 3187 wgs = wgp->wgp_session_stable; 3188 if (wgs->wgs_state == WGS_STATE_ESTABLISHED && 3189 !atomic_swap_uint(&wgp->wgp_force_rekey, 0)) 3190 return; 3191 3192 /* 3193 * Ensure we're initiating a new session. If the unstable 3194 * session is already INIT_ACTIVE or INIT_PASSIVE, this does 3195 * nothing. 3196 */ 3197 wg_send_handshake_msg_init(wg, wgp); 3198 } 3199 3200 static void 3201 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 3202 { 3203 struct wg_session *wgs; 3204 3205 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 3206 3207 KASSERT(mutex_owned(wgp->wgp_lock)); 3208 KASSERT(wgp->wgp_handshake_start_time != 0); 3209 3210 wgs = wgp->wgp_session_unstable; 3211 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3212 return; 3213 3214 /* 3215 * XXX no real need to assign a new index here, but we do need 3216 * to transition to UNKNOWN temporarily 3217 */ 3218 wg_put_session_index(wg, wgs); 3219 3220 /* [W] 6.4 Handshake Initiation Retransmission */ 3221 if ((time_uptime - wgp->wgp_handshake_start_time) > 3222 wg_rekey_attempt_time) { 3223 /* Give up handshaking */ 3224 wgp->wgp_handshake_start_time = 0; 3225 WG_TRACE("give up"); 3226 3227 /* 3228 * If a new data packet comes, handshaking will be retried 3229 * and a new session would be established at that time, 3230 * however we don't want to send pending packets then. 3231 */ 3232 wg_purge_pending_packets(wgp); 3233 return; 3234 } 3235 3236 wg_task_send_init_message(wg, wgp); 3237 } 3238 3239 static void 3240 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3241 { 3242 struct wg_session *wgs, *wgs_prev; 3243 struct mbuf *m; 3244 3245 KASSERT(mutex_owned(wgp->wgp_lock)); 3246 3247 wgs = wgp->wgp_session_unstable; 3248 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3249 /* XXX Can this happen? */ 3250 return; 3251 3252 wgs->wgs_time_last_data_sent = 0; 3253 wgs->wgs_is_initiator = false; 3254 3255 /* 3256 * Session was already ready to receive data. Transition from 3257 * INIT_PASSIVE to ESTABLISHED just so we can swap the 3258 * sessions. 3259 * 3260 * atomic_store_relaxed because this doesn't affect the data rx 3261 * path, wg_handle_msg_data -- changing from INIT_PASSIVE to 3262 * ESTABLISHED makes no difference to the data rx path, and the 3263 * transition to INIT_PASSIVE with store-release already 3264 * published the state needed by the data rx path. 3265 */ 3266 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_ESTABLISHED\n", 3267 wgs->wgs_local_index, wgs->wgs_remote_index); 3268 atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_ESTABLISHED); 3269 WG_TRACE("WGS_STATE_ESTABLISHED"); 3270 3271 /* 3272 * Session is ready to send data too now that we have received 3273 * the peer initiator's first data packet. 3274 * 3275 * Swap the sessions to publish the new one as the stable 3276 * session for the data tx path, wg_output. 3277 */ 3278 wg_swap_sessions(wgp); 3279 KASSERT(wgs == wgp->wgp_session_stable); 3280 wgs_prev = wgp->wgp_session_unstable; 3281 getnanotime(&wgp->wgp_last_handshake_time); 3282 wgp->wgp_handshake_start_time = 0; 3283 wgp->wgp_last_sent_mac1_valid = false; 3284 wgp->wgp_last_sent_cookie_valid = false; 3285 3286 /* If we had a data packet queued up, send it. */ 3287 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3288 kpreempt_disable(); 3289 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3290 M_SETCTX(m, wgp); 3291 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3292 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3293 if_name(&wg->wg_if)); 3294 m_freem(m); 3295 } 3296 kpreempt_enable(); 3297 } 3298 3299 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3300 /* 3301 * Transition ESTABLISHED->DESTROYING. The session 3302 * will remain usable for the data rx path to process 3303 * packets still in flight to us, but we won't use it 3304 * for data tx. 3305 */ 3306 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]" 3307 " -> WGS_STATE_DESTROYING\n", 3308 wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index); 3309 atomic_store_relaxed(&wgs_prev->wgs_state, 3310 WGS_STATE_DESTROYING); 3311 } else { 3312 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3313 "state=%d", wgs_prev->wgs_state); 3314 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]" 3315 " -> WGS_STATE_UNKNOWN\n", 3316 wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index); 3317 wgs_prev->wgs_local_index = 0; /* paranoia */ 3318 wgs_prev->wgs_remote_index = 0; /* paranoia */ 3319 wg_clear_states(wgs_prev); /* paranoia */ 3320 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3321 } 3322 } 3323 3324 static void 3325 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3326 { 3327 3328 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3329 3330 KASSERT(mutex_owned(wgp->wgp_lock)); 3331 3332 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3333 pserialize_perform(wgp->wgp_psz); 3334 mutex_exit(wgp->wgp_lock); 3335 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3336 wg_psref_class); 3337 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3338 wg_psref_class); 3339 mutex_enter(wgp->wgp_lock); 3340 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3341 } 3342 } 3343 3344 static void 3345 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3346 { 3347 struct wg_session *wgs; 3348 3349 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3350 3351 KASSERT(mutex_owned(wgp->wgp_lock)); 3352 3353 wgs = wgp->wgp_session_stable; 3354 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3355 return; 3356 3357 wg_send_keepalive_msg(wgp, wgs); 3358 } 3359 3360 static void 3361 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3362 { 3363 struct wg_session *wgs; 3364 uint32_t age; 3365 3366 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3367 3368 KASSERT(mutex_owned(wgp->wgp_lock)); 3369 3370 /* 3371 * If theres's any previous unstable session, i.e., one that 3372 * was ESTABLISHED and is now DESTROYING, older than 3373 * reject-after-time, destroy it. Upcoming sessions are still 3374 * in INIT_ACTIVE or INIT_PASSIVE -- we don't touch those here. 3375 * 3376 * No atomic for access to wgs_time_established because it is 3377 * only updated under wgp_lock. 3378 */ 3379 wgs = wgp->wgp_session_unstable; 3380 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 3381 if (wgs->wgs_state == WGS_STATE_DESTROYING && 3382 ((age = (time_uptime32 - wgs->wgs_time_established)) >= 3383 wg_reject_after_time)) { 3384 WG_DLOG("destroying past session %"PRIu32" sec old\n", age); 3385 wg_put_session_index(wg, wgs); 3386 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 3387 wgs->wgs_state); 3388 } 3389 3390 /* 3391 * If theres's any ESTABLISHED stable session older than 3392 * reject-after-time, destroy it. (The stable session can also 3393 * be in UNKNOWN state -- nothing to do in that case) 3394 */ 3395 wgs = wgp->wgp_session_stable; 3396 KASSERT(wgs->wgs_state != WGS_STATE_INIT_ACTIVE); 3397 KASSERT(wgs->wgs_state != WGS_STATE_INIT_PASSIVE); 3398 KASSERT(wgs->wgs_state != WGS_STATE_DESTROYING); 3399 if (wgs->wgs_state == WGS_STATE_ESTABLISHED && 3400 ((age = (time_uptime32 - wgs->wgs_time_established)) >= 3401 wg_reject_after_time)) { 3402 WG_DLOG("destroying current session %"PRIu32" sec old\n", age); 3403 atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_DESTROYING); 3404 wg_put_session_index(wg, wgs); 3405 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 3406 wgs->wgs_state); 3407 } 3408 3409 /* 3410 * If there's no sessions left, no need to have the timer run 3411 * until the next time around -- halt it. 3412 * 3413 * It is only ever scheduled with wgp_lock held or in the 3414 * callout itself, and callout_halt prevents rescheudling 3415 * itself, so this never races with rescheduling. 3416 */ 3417 if (wgp->wgp_session_unstable->wgs_state == WGS_STATE_UNKNOWN && 3418 wgp->wgp_session_stable->wgs_state == WGS_STATE_UNKNOWN) 3419 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3420 } 3421 3422 static void 3423 wg_peer_work(struct work *wk, void *cookie) 3424 { 3425 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3426 struct wg_softc *wg = wgp->wgp_sc; 3427 unsigned int tasks; 3428 3429 mutex_enter(wgp->wgp_intr_lock); 3430 while ((tasks = wgp->wgp_tasks) != 0) { 3431 wgp->wgp_tasks = 0; 3432 mutex_exit(wgp->wgp_intr_lock); 3433 3434 mutex_enter(wgp->wgp_lock); 3435 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3436 wg_task_send_init_message(wg, wgp); 3437 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3438 wg_task_retry_handshake(wg, wgp); 3439 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3440 wg_task_establish_session(wg, wgp); 3441 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3442 wg_task_endpoint_changed(wg, wgp); 3443 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3444 wg_task_send_keepalive_message(wg, wgp); 3445 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3446 wg_task_destroy_prev_session(wg, wgp); 3447 mutex_exit(wgp->wgp_lock); 3448 3449 mutex_enter(wgp->wgp_intr_lock); 3450 } 3451 mutex_exit(wgp->wgp_intr_lock); 3452 } 3453 3454 static void 3455 wg_job(struct threadpool_job *job) 3456 { 3457 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3458 int bound, upcalls; 3459 3460 mutex_enter(wg->wg_intr_lock); 3461 while ((upcalls = wg->wg_upcalls) != 0) { 3462 wg->wg_upcalls = 0; 3463 mutex_exit(wg->wg_intr_lock); 3464 bound = curlwp_bind(); 3465 if (ISSET(upcalls, WG_UPCALL_INET)) 3466 wg_receive_packets(wg, AF_INET); 3467 if (ISSET(upcalls, WG_UPCALL_INET6)) 3468 wg_receive_packets(wg, AF_INET6); 3469 curlwp_bindx(bound); 3470 mutex_enter(wg->wg_intr_lock); 3471 } 3472 threadpool_job_done(job); 3473 mutex_exit(wg->wg_intr_lock); 3474 } 3475 3476 static int 3477 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3478 { 3479 int error; 3480 uint16_t old_port = wg->wg_listen_port; 3481 3482 if (port != 0 && old_port == port) 3483 return 0; 3484 3485 struct sockaddr_in _sin, *sin = &_sin; 3486 sin->sin_len = sizeof(*sin); 3487 sin->sin_family = AF_INET; 3488 sin->sin_addr.s_addr = INADDR_ANY; 3489 sin->sin_port = htons(port); 3490 3491 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3492 if (error != 0) 3493 return error; 3494 3495 #ifdef INET6 3496 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3497 sin6->sin6_len = sizeof(*sin6); 3498 sin6->sin6_family = AF_INET6; 3499 sin6->sin6_addr = in6addr_any; 3500 sin6->sin6_port = htons(port); 3501 3502 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3503 if (error != 0) 3504 return error; 3505 #endif 3506 3507 wg->wg_listen_port = port; 3508 3509 return 0; 3510 } 3511 3512 static void 3513 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3514 { 3515 struct wg_softc *wg = cookie; 3516 int reason; 3517 3518 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3519 WG_UPCALL_INET : 3520 WG_UPCALL_INET6; 3521 3522 mutex_enter(wg->wg_intr_lock); 3523 wg->wg_upcalls |= reason; 3524 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3525 mutex_exit(wg->wg_intr_lock); 3526 } 3527 3528 static int 3529 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3530 struct sockaddr *src, void *arg) 3531 { 3532 struct wg_softc *wg = arg; 3533 struct wg_msg wgm; 3534 struct mbuf *m = *mp; 3535 3536 WG_TRACE("enter"); 3537 3538 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3539 KASSERT(offset <= m_length(m)); 3540 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3541 /* drop on the floor */ 3542 m_freem(m); 3543 return -1; 3544 } 3545 3546 /* 3547 * Copy the message header (32-bit message type) out -- we'll 3548 * worry about contiguity and alignment later. 3549 */ 3550 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3551 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3552 3553 /* 3554 * Handle DATA packets promptly as they arrive, if they are in 3555 * an active session. Other packets may require expensive 3556 * public-key crypto and are not as sensitive to latency, so 3557 * defer them to the worker thread. 3558 */ 3559 switch (le32toh(wgm.wgm_type)) { 3560 case WG_MSG_TYPE_DATA: 3561 /* handle immediately */ 3562 m_adj(m, offset); 3563 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3564 m = m_pullup(m, sizeof(struct wg_msg_data)); 3565 if (m == NULL) 3566 return -1; 3567 } 3568 wg_handle_msg_data(wg, m, src); 3569 *mp = NULL; 3570 return 1; 3571 case WG_MSG_TYPE_INIT: 3572 case WG_MSG_TYPE_RESP: 3573 case WG_MSG_TYPE_COOKIE: 3574 /* pass through to so_receive in wg_receive_packets */ 3575 return 0; 3576 default: 3577 /* drop on the floor */ 3578 m_freem(m); 3579 return -1; 3580 } 3581 } 3582 3583 static int 3584 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3585 { 3586 int error; 3587 struct socket *so; 3588 3589 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3590 if (error != 0) 3591 return error; 3592 3593 solock(so); 3594 so->so_upcallarg = wg; 3595 so->so_upcall = wg_so_upcall; 3596 so->so_rcv.sb_flags |= SB_UPCALL; 3597 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3598 sounlock(so); 3599 3600 *sop = so; 3601 3602 return 0; 3603 } 3604 3605 static bool 3606 wg_session_hit_limits(struct wg_session *wgs) 3607 { 3608 uint32_t time_established = 3609 atomic_load_relaxed(&wgs->wgs_time_established); 3610 3611 /* 3612 * [W] 6.2: Transport Message Limits 3613 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3614 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3615 * comes first, WireGuard will refuse to send or receive any more 3616 * transport data messages using the current secure session, ..." 3617 */ 3618 KASSERT(time_established != 0 || time_uptime > UINT32_MAX); 3619 if ((time_uptime32 - time_established) > wg_reject_after_time) { 3620 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3621 return true; 3622 } else if (wg_session_get_send_counter(wgs) > 3623 wg_reject_after_messages) { 3624 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3625 return true; 3626 } 3627 3628 return false; 3629 } 3630 3631 static void 3632 wgintr(void *cookie) 3633 { 3634 struct wg_peer *wgp; 3635 struct wg_session *wgs; 3636 struct mbuf *m; 3637 struct psref psref; 3638 3639 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3640 wgp = M_GETCTX(m, struct wg_peer *); 3641 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3642 WG_TRACE("no stable session"); 3643 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3644 goto next0; 3645 } 3646 if (__predict_false(wg_session_hit_limits(wgs))) { 3647 WG_TRACE("stable session hit limits"); 3648 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3649 goto next1; 3650 } 3651 wg_send_data_msg(wgp, wgs, m); 3652 m = NULL; /* consumed */ 3653 next1: wg_put_session(wgs, &psref); 3654 next0: m_freem(m); 3655 /* XXX Yield to avoid userland starvation? */ 3656 } 3657 } 3658 3659 static void 3660 wg_purge_pending_packets(struct wg_peer *wgp) 3661 { 3662 struct mbuf *m; 3663 3664 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 3665 m_freem(m); 3666 pktq_barrier(wg_pktq); 3667 } 3668 3669 static void 3670 wg_handshake_timeout_timer(void *arg) 3671 { 3672 struct wg_peer *wgp = arg; 3673 3674 WG_TRACE("enter"); 3675 3676 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3677 } 3678 3679 static struct wg_peer * 3680 wg_alloc_peer(struct wg_softc *wg) 3681 { 3682 struct wg_peer *wgp; 3683 3684 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3685 3686 wgp->wgp_sc = wg; 3687 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3688 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3689 wg_handshake_timeout_timer, wgp); 3690 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3691 callout_setfunc(&wgp->wgp_session_dtor_timer, 3692 wg_session_dtor_timer, wgp); 3693 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3694 wgp->wgp_endpoint_changing = false; 3695 wgp->wgp_endpoint_available = false; 3696 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3697 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3698 wgp->wgp_psz = pserialize_create(); 3699 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3700 3701 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3702 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3703 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3704 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3705 3706 struct wg_session *wgs; 3707 wgp->wgp_session_stable = 3708 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3709 wgp->wgp_session_unstable = 3710 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3711 wgs = wgp->wgp_session_stable; 3712 wgs->wgs_peer = wgp; 3713 wgs->wgs_state = WGS_STATE_UNKNOWN; 3714 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3715 #ifndef __HAVE_ATOMIC64_LOADSTORE 3716 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3717 #endif 3718 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3719 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3720 3721 wgs = wgp->wgp_session_unstable; 3722 wgs->wgs_peer = wgp; 3723 wgs->wgs_state = WGS_STATE_UNKNOWN; 3724 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3725 #ifndef __HAVE_ATOMIC64_LOADSTORE 3726 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3727 #endif 3728 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3729 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3730 3731 return wgp; 3732 } 3733 3734 static void 3735 wg_destroy_peer(struct wg_peer *wgp) 3736 { 3737 struct wg_session *wgs; 3738 struct wg_softc *wg = wgp->wgp_sc; 3739 3740 /* Prevent new packets from this peer on any source address. */ 3741 rw_enter(wg->wg_rwlock, RW_WRITER); 3742 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3743 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3744 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3745 struct radix_node *rn; 3746 3747 KASSERT(rnh != NULL); 3748 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3749 &wga->wga_sa_mask, rnh); 3750 if (rn == NULL) { 3751 char addrstr[128]; 3752 sockaddr_format(&wga->wga_sa_addr, addrstr, 3753 sizeof(addrstr)); 3754 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3755 if_name(&wg->wg_if), addrstr); 3756 } 3757 } 3758 rw_exit(wg->wg_rwlock); 3759 3760 /* Purge pending packets. */ 3761 wg_purge_pending_packets(wgp); 3762 3763 /* Halt all packet processing and timeouts. */ 3764 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3765 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3766 3767 /* Wait for any queued work to complete. */ 3768 workqueue_wait(wg_wq, &wgp->wgp_work); 3769 3770 wgs = wgp->wgp_session_unstable; 3771 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3772 mutex_enter(wgp->wgp_lock); 3773 wg_destroy_session(wg, wgs); 3774 mutex_exit(wgp->wgp_lock); 3775 } 3776 mutex_destroy(&wgs->wgs_recvwin->lock); 3777 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3778 #ifndef __HAVE_ATOMIC64_LOADSTORE 3779 mutex_destroy(&wgs->wgs_send_counter_lock); 3780 #endif 3781 kmem_free(wgs, sizeof(*wgs)); 3782 3783 wgs = wgp->wgp_session_stable; 3784 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3785 mutex_enter(wgp->wgp_lock); 3786 wg_destroy_session(wg, wgs); 3787 mutex_exit(wgp->wgp_lock); 3788 } 3789 mutex_destroy(&wgs->wgs_recvwin->lock); 3790 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3791 #ifndef __HAVE_ATOMIC64_LOADSTORE 3792 mutex_destroy(&wgs->wgs_send_counter_lock); 3793 #endif 3794 kmem_free(wgs, sizeof(*wgs)); 3795 3796 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3797 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3798 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3799 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3800 3801 pserialize_destroy(wgp->wgp_psz); 3802 mutex_obj_free(wgp->wgp_intr_lock); 3803 mutex_obj_free(wgp->wgp_lock); 3804 3805 kmem_free(wgp, sizeof(*wgp)); 3806 } 3807 3808 static void 3809 wg_destroy_all_peers(struct wg_softc *wg) 3810 { 3811 struct wg_peer *wgp, *wgp0 __diagused; 3812 void *garbage_byname, *garbage_bypubkey; 3813 3814 restart: 3815 garbage_byname = garbage_bypubkey = NULL; 3816 mutex_enter(wg->wg_lock); 3817 WG_PEER_WRITER_FOREACH(wgp, wg) { 3818 if (wgp->wgp_name[0]) { 3819 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3820 strlen(wgp->wgp_name)); 3821 KASSERT(wgp0 == wgp); 3822 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3823 } 3824 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3825 sizeof(wgp->wgp_pubkey)); 3826 KASSERT(wgp0 == wgp); 3827 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3828 WG_PEER_WRITER_REMOVE(wgp); 3829 wg->wg_npeers--; 3830 mutex_enter(wgp->wgp_lock); 3831 pserialize_perform(wgp->wgp_psz); 3832 mutex_exit(wgp->wgp_lock); 3833 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3834 break; 3835 } 3836 mutex_exit(wg->wg_lock); 3837 3838 if (wgp == NULL) 3839 return; 3840 3841 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3842 3843 wg_destroy_peer(wgp); 3844 thmap_gc(wg->wg_peers_byname, garbage_byname); 3845 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3846 3847 goto restart; 3848 } 3849 3850 static int 3851 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3852 { 3853 struct wg_peer *wgp, *wgp0 __diagused; 3854 void *garbage_byname, *garbage_bypubkey; 3855 3856 mutex_enter(wg->wg_lock); 3857 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3858 if (wgp != NULL) { 3859 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3860 sizeof(wgp->wgp_pubkey)); 3861 KASSERT(wgp0 == wgp); 3862 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3863 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3864 WG_PEER_WRITER_REMOVE(wgp); 3865 wg->wg_npeers--; 3866 if (wg->wg_npeers == 0) 3867 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3868 mutex_enter(wgp->wgp_lock); 3869 pserialize_perform(wgp->wgp_psz); 3870 mutex_exit(wgp->wgp_lock); 3871 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3872 } 3873 mutex_exit(wg->wg_lock); 3874 3875 if (wgp == NULL) 3876 return ENOENT; 3877 3878 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3879 3880 wg_destroy_peer(wgp); 3881 thmap_gc(wg->wg_peers_byname, garbage_byname); 3882 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3883 3884 return 0; 3885 } 3886 3887 static int 3888 wg_if_attach(struct wg_softc *wg) 3889 { 3890 3891 wg->wg_if.if_addrlen = 0; 3892 wg->wg_if.if_mtu = WG_MTU; 3893 wg->wg_if.if_flags = IFF_MULTICAST; 3894 wg->wg_if.if_extflags = IFEF_MPSAFE; 3895 wg->wg_if.if_ioctl = wg_ioctl; 3896 wg->wg_if.if_output = wg_output; 3897 wg->wg_if.if_init = wg_init; 3898 #ifdef ALTQ 3899 wg->wg_if.if_start = wg_start; 3900 #endif 3901 wg->wg_if.if_stop = wg_stop; 3902 wg->wg_if.if_type = IFT_OTHER; 3903 wg->wg_if.if_dlt = DLT_NULL; 3904 wg->wg_if.if_softc = wg; 3905 #ifdef ALTQ 3906 IFQ_SET_READY(&wg->wg_if.if_snd); 3907 #endif 3908 if_initialize(&wg->wg_if); 3909 3910 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3911 if_alloc_sadl(&wg->wg_if); 3912 if_register(&wg->wg_if); 3913 3914 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3915 3916 return 0; 3917 } 3918 3919 static void 3920 wg_if_detach(struct wg_softc *wg) 3921 { 3922 struct ifnet *ifp = &wg->wg_if; 3923 3924 bpf_detach(ifp); 3925 if_detach(ifp); 3926 } 3927 3928 static int 3929 wg_clone_create(struct if_clone *ifc, int unit) 3930 { 3931 struct wg_softc *wg; 3932 int error; 3933 3934 wg_guarantee_initialized(); 3935 3936 error = wg_count_inc(); 3937 if (error) 3938 return error; 3939 3940 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3941 3942 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3943 3944 PSLIST_INIT(&wg->wg_peers); 3945 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3946 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3947 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3948 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3949 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3950 wg->wg_rwlock = rw_obj_alloc(); 3951 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3952 "%s", if_name(&wg->wg_if)); 3953 wg->wg_ops = &wg_ops_rumpkernel; 3954 3955 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3956 if (error) 3957 goto fail0; 3958 3959 #ifdef INET 3960 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3961 if (error) 3962 goto fail1; 3963 rn_inithead((void **)&wg->wg_rtable_ipv4, 3964 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3965 #endif 3966 #ifdef INET6 3967 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3968 if (error) 3969 goto fail2; 3970 rn_inithead((void **)&wg->wg_rtable_ipv6, 3971 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3972 #endif 3973 3974 error = wg_if_attach(wg); 3975 if (error) 3976 goto fail3; 3977 3978 return 0; 3979 3980 fail4: __unused 3981 wg_if_detach(wg); 3982 fail3: wg_destroy_all_peers(wg); 3983 #ifdef INET6 3984 solock(wg->wg_so6); 3985 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3986 sounlock(wg->wg_so6); 3987 #endif 3988 #ifdef INET 3989 solock(wg->wg_so4); 3990 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3991 sounlock(wg->wg_so4); 3992 #endif 3993 mutex_enter(wg->wg_intr_lock); 3994 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3995 mutex_exit(wg->wg_intr_lock); 3996 #ifdef INET6 3997 if (wg->wg_rtable_ipv6 != NULL) 3998 free(wg->wg_rtable_ipv6, M_RTABLE); 3999 soclose(wg->wg_so6); 4000 fail2: 4001 #endif 4002 #ifdef INET 4003 if (wg->wg_rtable_ipv4 != NULL) 4004 free(wg->wg_rtable_ipv4, M_RTABLE); 4005 soclose(wg->wg_so4); 4006 fail1: 4007 #endif 4008 threadpool_put(wg->wg_threadpool, PRI_NONE); 4009 fail0: threadpool_job_destroy(&wg->wg_job); 4010 rw_obj_free(wg->wg_rwlock); 4011 mutex_obj_free(wg->wg_intr_lock); 4012 mutex_obj_free(wg->wg_lock); 4013 thmap_destroy(wg->wg_sessions_byindex); 4014 thmap_destroy(wg->wg_peers_byname); 4015 thmap_destroy(wg->wg_peers_bypubkey); 4016 PSLIST_DESTROY(&wg->wg_peers); 4017 kmem_free(wg, sizeof(*wg)); 4018 wg_count_dec(); 4019 return error; 4020 } 4021 4022 static int 4023 wg_clone_destroy(struct ifnet *ifp) 4024 { 4025 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 4026 4027 #ifdef WG_RUMPKERNEL 4028 if (wg_user_mode(wg)) { 4029 rumpuser_wg_destroy(wg->wg_user); 4030 wg->wg_user = NULL; 4031 } 4032 #endif 4033 4034 wg_if_detach(wg); 4035 wg_destroy_all_peers(wg); 4036 #ifdef INET6 4037 solock(wg->wg_so6); 4038 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 4039 sounlock(wg->wg_so6); 4040 #endif 4041 #ifdef INET 4042 solock(wg->wg_so4); 4043 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 4044 sounlock(wg->wg_so4); 4045 #endif 4046 mutex_enter(wg->wg_intr_lock); 4047 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 4048 mutex_exit(wg->wg_intr_lock); 4049 #ifdef INET6 4050 if (wg->wg_rtable_ipv6 != NULL) 4051 free(wg->wg_rtable_ipv6, M_RTABLE); 4052 soclose(wg->wg_so6); 4053 #endif 4054 #ifdef INET 4055 if (wg->wg_rtable_ipv4 != NULL) 4056 free(wg->wg_rtable_ipv4, M_RTABLE); 4057 soclose(wg->wg_so4); 4058 #endif 4059 threadpool_put(wg->wg_threadpool, PRI_NONE); 4060 threadpool_job_destroy(&wg->wg_job); 4061 rw_obj_free(wg->wg_rwlock); 4062 mutex_obj_free(wg->wg_intr_lock); 4063 mutex_obj_free(wg->wg_lock); 4064 thmap_destroy(wg->wg_sessions_byindex); 4065 thmap_destroy(wg->wg_peers_byname); 4066 thmap_destroy(wg->wg_peers_bypubkey); 4067 PSLIST_DESTROY(&wg->wg_peers); 4068 kmem_free(wg, sizeof(*wg)); 4069 wg_count_dec(); 4070 4071 return 0; 4072 } 4073 4074 static struct wg_peer * 4075 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 4076 struct psref *psref) 4077 { 4078 struct radix_node_head *rnh; 4079 struct radix_node *rn; 4080 struct wg_peer *wgp = NULL; 4081 struct wg_allowedip *wga; 4082 4083 #ifdef WG_DEBUG_LOG 4084 char addrstr[128]; 4085 sockaddr_format(sa, addrstr, sizeof(addrstr)); 4086 WG_DLOG("sa=%s\n", addrstr); 4087 #endif 4088 4089 rw_enter(wg->wg_rwlock, RW_READER); 4090 4091 rnh = wg_rnh(wg, sa->sa_family); 4092 if (rnh == NULL) 4093 goto out; 4094 4095 rn = rnh->rnh_matchaddr(sa, rnh); 4096 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 4097 goto out; 4098 4099 WG_TRACE("success"); 4100 4101 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 4102 wgp = wga->wga_peer; 4103 wg_get_peer(wgp, psref); 4104 4105 out: 4106 rw_exit(wg->wg_rwlock); 4107 return wgp; 4108 } 4109 4110 static void 4111 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 4112 struct wg_session *wgs, struct wg_msg_data *wgmd) 4113 { 4114 4115 memset(wgmd, 0, sizeof(*wgmd)); 4116 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 4117 wgmd->wgmd_receiver = wgs->wgs_remote_index; 4118 /* [W] 5.4.6: msg.counter := Nm^send */ 4119 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 4120 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 4121 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 4122 } 4123 4124 static int 4125 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 4126 const struct rtentry *rt) 4127 { 4128 struct wg_softc *wg = ifp->if_softc; 4129 struct wg_peer *wgp = NULL; 4130 struct wg_session *wgs = NULL; 4131 struct psref wgp_psref, wgs_psref; 4132 int bound; 4133 int error; 4134 4135 bound = curlwp_bind(); 4136 4137 /* TODO make the nest limit configurable via sysctl */ 4138 error = if_tunnel_check_nesting(ifp, m, 1); 4139 if (error) { 4140 WGLOG(LOG_ERR, 4141 "%s: tunneling loop detected and packet dropped\n", 4142 if_name(&wg->wg_if)); 4143 goto out0; 4144 } 4145 4146 #ifdef ALTQ 4147 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 4148 & ALTQF_ENABLED; 4149 if (altq) 4150 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 4151 #endif 4152 4153 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 4154 4155 m->m_flags &= ~(M_BCAST|M_MCAST); 4156 4157 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 4158 if (wgp == NULL) { 4159 WG_TRACE("peer not found"); 4160 error = EHOSTUNREACH; 4161 goto out0; 4162 } 4163 4164 /* Clear checksum-offload flags. */ 4165 m->m_pkthdr.csum_flags = 0; 4166 m->m_pkthdr.csum_data = 0; 4167 4168 /* Check whether there's an established session. */ 4169 wgs = wg_get_stable_session(wgp, &wgs_psref); 4170 if (wgs == NULL) { 4171 /* 4172 * No established session. If we're the first to try 4173 * sending data, schedule a handshake and queue the 4174 * packet for when the handshake is done; otherwise 4175 * just drop the packet and let the ongoing handshake 4176 * attempt continue. We could queue more data packets 4177 * but it's not clear that's worthwhile. 4178 */ 4179 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 4180 m = NULL; /* consume */ 4181 WG_TRACE("queued first packet; init handshake"); 4182 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4183 } else { 4184 WG_TRACE("first packet already queued, dropping"); 4185 } 4186 goto out1; 4187 } 4188 4189 /* There's an established session. Toss it in the queue. */ 4190 #ifdef ALTQ 4191 if (altq) { 4192 mutex_enter(ifp->if_snd.ifq_lock); 4193 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 4194 M_SETCTX(m, wgp); 4195 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 4196 m = NULL; /* consume */ 4197 } 4198 mutex_exit(ifp->if_snd.ifq_lock); 4199 if (m == NULL) { 4200 wg_start(ifp); 4201 goto out2; 4202 } 4203 } 4204 #endif 4205 kpreempt_disable(); 4206 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4207 M_SETCTX(m, wgp); 4208 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4209 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4210 if_name(&wg->wg_if)); 4211 error = ENOBUFS; 4212 goto out3; 4213 } 4214 m = NULL; /* consumed */ 4215 error = 0; 4216 out3: kpreempt_enable(); 4217 4218 #ifdef ALTQ 4219 out2: 4220 #endif 4221 wg_put_session(wgs, &wgs_psref); 4222 out1: wg_put_peer(wgp, &wgp_psref); 4223 out0: m_freem(m); 4224 curlwp_bindx(bound); 4225 return error; 4226 } 4227 4228 static int 4229 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 4230 { 4231 struct psref psref; 4232 struct wg_sockaddr *wgsa; 4233 int error; 4234 struct socket *so; 4235 4236 wgsa = wg_get_endpoint_sa(wgp, &psref); 4237 so = wg_get_so_by_peer(wgp, wgsa); 4238 solock(so); 4239 if (wgsatosa(wgsa)->sa_family == AF_INET) { 4240 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 4241 } else { 4242 #ifdef INET6 4243 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 4244 NULL, curlwp); 4245 #else 4246 m_freem(m); 4247 error = EPFNOSUPPORT; 4248 #endif 4249 } 4250 sounlock(so); 4251 wg_put_sa(wgp, wgsa, &psref); 4252 4253 return error; 4254 } 4255 4256 /* Inspired by pppoe_get_mbuf */ 4257 static struct mbuf * 4258 wg_get_mbuf(size_t leading_len, size_t len) 4259 { 4260 struct mbuf *m; 4261 4262 KASSERT(leading_len <= MCLBYTES); 4263 KASSERT(len <= MCLBYTES - leading_len); 4264 4265 m = m_gethdr(M_DONTWAIT, MT_DATA); 4266 if (m == NULL) 4267 return NULL; 4268 if (len + leading_len > MHLEN) { 4269 m_clget(m, M_DONTWAIT); 4270 if ((m->m_flags & M_EXT) == 0) { 4271 m_free(m); 4272 return NULL; 4273 } 4274 } 4275 m->m_data += leading_len; 4276 m->m_pkthdr.len = m->m_len = len; 4277 4278 return m; 4279 } 4280 4281 static int 4282 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 4283 struct mbuf *m) 4284 { 4285 struct wg_softc *wg = wgp->wgp_sc; 4286 int error; 4287 size_t inner_len, padded_len, encrypted_len; 4288 char *padded_buf = NULL; 4289 size_t mlen; 4290 struct wg_msg_data *wgmd; 4291 bool free_padded_buf = false; 4292 struct mbuf *n; 4293 size_t leading_len = max_hdr + sizeof(struct udphdr); 4294 4295 mlen = m_length(m); 4296 inner_len = mlen; 4297 padded_len = roundup(mlen, 16); 4298 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4299 WG_DLOG("inner=%zu, padded=%zu, encrypted_len=%zu\n", 4300 inner_len, padded_len, encrypted_len); 4301 if (mlen != 0) { 4302 bool success; 4303 success = m_ensure_contig(&m, padded_len); 4304 if (success) { 4305 padded_buf = mtod(m, char *); 4306 } else { 4307 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4308 if (padded_buf == NULL) { 4309 error = ENOBUFS; 4310 goto end; 4311 } 4312 free_padded_buf = true; 4313 m_copydata(m, 0, mlen, padded_buf); 4314 } 4315 memset(padded_buf + mlen, 0, padded_len - inner_len); 4316 } 4317 4318 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4319 if (n == NULL) { 4320 error = ENOBUFS; 4321 goto end; 4322 } 4323 KASSERT(n->m_len >= sizeof(*wgmd)); 4324 wgmd = mtod(n, struct wg_msg_data *); 4325 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4326 #ifdef WG_DEBUG_PACKET 4327 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4328 hexdump(printf, "padded_buf", padded_buf, 4329 padded_len); 4330 } 4331 #endif 4332 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4333 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4334 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4335 padded_buf, padded_len, 4336 NULL, 0); 4337 #ifdef WG_DEBUG_PACKET 4338 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4339 hexdump(printf, "tkey_send", wgs->wgs_tkey_send, 4340 sizeof(wgs->wgs_tkey_send)); 4341 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 4342 hexdump(printf, "outgoing packet", 4343 (char *)wgmd + sizeof(*wgmd), encrypted_len); 4344 size_t decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 4345 char *decrypted_buf = kmem_intr_alloc((decrypted_len + 4346 WG_AUTHTAG_LEN/*XXX*/), KM_NOSLEEP); 4347 if (decrypted_buf != NULL) { 4348 error = wg_algo_aead_dec( 4349 1 + decrypted_buf /* force misalignment */, 4350 encrypted_len - WG_AUTHTAG_LEN /* XXX */, 4351 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4352 (char *)wgmd + sizeof(*wgmd), encrypted_len, 4353 NULL, 0); 4354 if (error) { 4355 WG_DLOG("wg_algo_aead_dec failed: %d\n", 4356 error); 4357 } 4358 if (!consttime_memequal(1 + decrypted_buf, 4359 (char *)wgmd + sizeof(*wgmd), 4360 decrypted_len)) { 4361 WG_DLOG("wg_algo_aead_dec returned garbage\n"); 4362 } 4363 kmem_intr_free(decrypted_buf, (decrypted_len + 4364 WG_AUTHTAG_LEN/*XXX*/)); 4365 } 4366 } 4367 #endif 4368 4369 error = wg->wg_ops->send_data_msg(wgp, n); 4370 if (error == 0) { 4371 struct ifnet *ifp = &wg->wg_if; 4372 if_statadd(ifp, if_obytes, mlen); 4373 if_statinc(ifp, if_opackets); 4374 if (wgs->wgs_is_initiator && 4375 ((time_uptime32 - 4376 atomic_load_relaxed(&wgs->wgs_time_established)) >= 4377 wg_rekey_after_time)) { 4378 /* 4379 * [W] 6.2 Transport Message Limits 4380 * "if a peer is the initiator of a current secure 4381 * session, WireGuard will send a handshake initiation 4382 * message to begin a new secure session if, after 4383 * transmitting a transport data message, the current 4384 * secure session is REKEY-AFTER-TIME seconds old," 4385 */ 4386 WG_TRACE("rekey after time"); 4387 atomic_store_relaxed(&wgp->wgp_force_rekey, 1); 4388 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4389 } 4390 const uint32_t now = time_uptime32; 4391 atomic_store_relaxed(&wgs->wgs_time_last_data_sent, 4392 MAX(now, 1)); 4393 if (wg_session_get_send_counter(wgs) >= 4394 wg_rekey_after_messages) { 4395 /* 4396 * [W] 6.2 Transport Message Limits 4397 * "WireGuard will try to create a new session, by 4398 * sending a handshake initiation message (section 4399 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4400 * transport data messages..." 4401 */ 4402 WG_TRACE("rekey after messages"); 4403 atomic_store_relaxed(&wgp->wgp_force_rekey, 1); 4404 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4405 } 4406 } 4407 end: 4408 m_freem(m); 4409 if (free_padded_buf) 4410 kmem_intr_free(padded_buf, padded_len); 4411 return error; 4412 } 4413 4414 static void 4415 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4416 { 4417 pktqueue_t *pktq; 4418 size_t pktlen; 4419 4420 KASSERT(af == AF_INET || af == AF_INET6); 4421 4422 WG_TRACE(""); 4423 4424 m_set_rcvif(m, ifp); 4425 pktlen = m->m_pkthdr.len; 4426 4427 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4428 4429 switch (af) { 4430 case AF_INET: 4431 pktq = ip_pktq; 4432 break; 4433 #ifdef INET6 4434 case AF_INET6: 4435 pktq = ip6_pktq; 4436 break; 4437 #endif 4438 default: 4439 panic("invalid af=%d", af); 4440 } 4441 4442 kpreempt_disable(); 4443 const u_int h = curcpu()->ci_index; 4444 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4445 if_statadd(ifp, if_ibytes, pktlen); 4446 if_statinc(ifp, if_ipackets); 4447 } else { 4448 m_freem(m); 4449 } 4450 kpreempt_enable(); 4451 } 4452 4453 static void 4454 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4455 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4456 { 4457 4458 crypto_scalarmult_base(pubkey, privkey); 4459 } 4460 4461 static int 4462 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4463 { 4464 struct radix_node_head *rnh; 4465 struct radix_node *rn; 4466 int error = 0; 4467 4468 rw_enter(wg->wg_rwlock, RW_WRITER); 4469 rnh = wg_rnh(wg, wga->wga_family); 4470 KASSERT(rnh != NULL); 4471 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4472 wga->wga_nodes); 4473 rw_exit(wg->wg_rwlock); 4474 4475 if (rn == NULL) 4476 error = EEXIST; 4477 4478 return error; 4479 } 4480 4481 static int 4482 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4483 struct wg_peer **wgpp) 4484 { 4485 int error = 0; 4486 const void *pubkey; 4487 size_t pubkey_len; 4488 const void *psk; 4489 size_t psk_len; 4490 const char *name = NULL; 4491 4492 if (prop_dictionary_get_string(peer, "name", &name)) { 4493 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4494 error = EINVAL; 4495 goto out; 4496 } 4497 } 4498 4499 if (!prop_dictionary_get_data(peer, "public_key", 4500 &pubkey, &pubkey_len)) { 4501 error = EINVAL; 4502 goto out; 4503 } 4504 #ifdef WG_DEBUG_DUMP 4505 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4506 char *hex = gethexdump(pubkey, pubkey_len); 4507 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%zu\n%s\n", 4508 pubkey, pubkey_len, hex); 4509 puthexdump(hex, pubkey, pubkey_len); 4510 } 4511 #endif 4512 4513 struct wg_peer *wgp = wg_alloc_peer(wg); 4514 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4515 if (name != NULL) 4516 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4517 4518 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4519 if (psk_len != sizeof(wgp->wgp_psk)) { 4520 error = EINVAL; 4521 goto out; 4522 } 4523 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4524 } 4525 4526 const void *addr; 4527 size_t addr_len; 4528 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4529 4530 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4531 goto skip_endpoint; 4532 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4533 addr_len > sizeof(*wgsatoss(wgsa))) { 4534 error = EINVAL; 4535 goto out; 4536 } 4537 memcpy(wgsatoss(wgsa), addr, addr_len); 4538 switch (wgsa_family(wgsa)) { 4539 case AF_INET: 4540 #ifdef INET6 4541 case AF_INET6: 4542 #endif 4543 break; 4544 default: 4545 error = EPFNOSUPPORT; 4546 goto out; 4547 } 4548 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4549 error = EINVAL; 4550 goto out; 4551 } 4552 { 4553 char addrstr[128]; 4554 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4555 WG_DLOG("addr=%s\n", addrstr); 4556 } 4557 wgp->wgp_endpoint_available = true; 4558 4559 prop_array_t allowedips; 4560 skip_endpoint: 4561 allowedips = prop_dictionary_get(peer, "allowedips"); 4562 if (allowedips == NULL) 4563 goto skip; 4564 4565 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4566 prop_dictionary_t prop_allowedip; 4567 int j = 0; 4568 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4569 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4570 4571 if (!prop_dictionary_get_int(prop_allowedip, "family", 4572 &wga->wga_family)) 4573 continue; 4574 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4575 &addr, &addr_len)) 4576 continue; 4577 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4578 &wga->wga_cidr)) 4579 continue; 4580 4581 switch (wga->wga_family) { 4582 case AF_INET: { 4583 struct sockaddr_in sin; 4584 char addrstr[128]; 4585 struct in_addr mask; 4586 struct sockaddr_in sin_mask; 4587 4588 if (addr_len != sizeof(struct in_addr)) 4589 return EINVAL; 4590 memcpy(&wga->wga_addr4, addr, addr_len); 4591 4592 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4593 0); 4594 sockaddr_copy(&wga->wga_sa_addr, 4595 sizeof(sin), sintosa(&sin)); 4596 4597 sockaddr_format(sintosa(&sin), 4598 addrstr, sizeof(addrstr)); 4599 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4600 4601 in_len2mask(&mask, wga->wga_cidr); 4602 sockaddr_in_init(&sin_mask, &mask, 0); 4603 sockaddr_copy(&wga->wga_sa_mask, 4604 sizeof(sin_mask), sintosa(&sin_mask)); 4605 4606 break; 4607 } 4608 #ifdef INET6 4609 case AF_INET6: { 4610 struct sockaddr_in6 sin6; 4611 char addrstr[128]; 4612 struct in6_addr mask; 4613 struct sockaddr_in6 sin6_mask; 4614 4615 if (addr_len != sizeof(struct in6_addr)) 4616 return EINVAL; 4617 memcpy(&wga->wga_addr6, addr, addr_len); 4618 4619 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4620 0, 0, 0); 4621 sockaddr_copy(&wga->wga_sa_addr, 4622 sizeof(sin6), sin6tosa(&sin6)); 4623 4624 sockaddr_format(sin6tosa(&sin6), 4625 addrstr, sizeof(addrstr)); 4626 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4627 4628 in6_prefixlen2mask(&mask, wga->wga_cidr); 4629 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4630 sockaddr_copy(&wga->wga_sa_mask, 4631 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4632 4633 break; 4634 } 4635 #endif 4636 default: 4637 error = EINVAL; 4638 goto out; 4639 } 4640 wga->wga_peer = wgp; 4641 4642 error = wg_rtable_add_route(wg, wga); 4643 if (error != 0) 4644 goto out; 4645 4646 j++; 4647 } 4648 wgp->wgp_n_allowedips = j; 4649 skip: 4650 *wgpp = wgp; 4651 out: 4652 return error; 4653 } 4654 4655 static int 4656 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4657 { 4658 int error; 4659 char *buf; 4660 4661 WG_DLOG("buf=%p, len=%zu\n", ifd->ifd_data, ifd->ifd_len); 4662 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4663 return E2BIG; 4664 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4665 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4666 if (error != 0) 4667 return error; 4668 buf[ifd->ifd_len] = '\0'; 4669 #ifdef WG_DEBUG_DUMP 4670 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4671 log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len), 4672 (const char *)buf); 4673 } 4674 #endif 4675 *_buf = buf; 4676 return 0; 4677 } 4678 4679 static int 4680 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4681 { 4682 int error; 4683 prop_dictionary_t prop_dict; 4684 char *buf = NULL; 4685 const void *privkey; 4686 size_t privkey_len; 4687 4688 error = wg_alloc_prop_buf(&buf, ifd); 4689 if (error != 0) 4690 return error; 4691 error = EINVAL; 4692 prop_dict = prop_dictionary_internalize(buf); 4693 if (prop_dict == NULL) 4694 goto out; 4695 if (!prop_dictionary_get_data(prop_dict, "private_key", 4696 &privkey, &privkey_len)) 4697 goto out; 4698 #ifdef WG_DEBUG_DUMP 4699 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4700 char *hex = gethexdump(privkey, privkey_len); 4701 log(LOG_DEBUG, "privkey=%p, privkey_len=%zu\n%s\n", 4702 privkey, privkey_len, hex); 4703 puthexdump(hex, privkey, privkey_len); 4704 } 4705 #endif 4706 if (privkey_len != WG_STATIC_KEY_LEN) 4707 goto out; 4708 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4709 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4710 error = 0; 4711 4712 out: 4713 kmem_free(buf, ifd->ifd_len + 1); 4714 return error; 4715 } 4716 4717 static int 4718 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4719 { 4720 int error; 4721 prop_dictionary_t prop_dict; 4722 char *buf = NULL; 4723 uint16_t port; 4724 4725 error = wg_alloc_prop_buf(&buf, ifd); 4726 if (error != 0) 4727 return error; 4728 error = EINVAL; 4729 prop_dict = prop_dictionary_internalize(buf); 4730 if (prop_dict == NULL) 4731 goto out; 4732 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4733 goto out; 4734 4735 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4736 4737 out: 4738 kmem_free(buf, ifd->ifd_len + 1); 4739 return error; 4740 } 4741 4742 static int 4743 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4744 { 4745 int error; 4746 prop_dictionary_t prop_dict; 4747 char *buf = NULL; 4748 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4749 4750 error = wg_alloc_prop_buf(&buf, ifd); 4751 if (error != 0) 4752 return error; 4753 error = EINVAL; 4754 prop_dict = prop_dictionary_internalize(buf); 4755 if (prop_dict == NULL) 4756 goto out; 4757 4758 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4759 if (error != 0) 4760 goto out; 4761 4762 mutex_enter(wg->wg_lock); 4763 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4764 sizeof(wgp->wgp_pubkey)) != NULL || 4765 (wgp->wgp_name[0] && 4766 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4767 strlen(wgp->wgp_name)) != NULL)) { 4768 mutex_exit(wg->wg_lock); 4769 wg_destroy_peer(wgp); 4770 error = EEXIST; 4771 goto out; 4772 } 4773 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4774 sizeof(wgp->wgp_pubkey), wgp); 4775 KASSERT(wgp0 == wgp); 4776 if (wgp->wgp_name[0]) { 4777 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4778 strlen(wgp->wgp_name), wgp); 4779 KASSERT(wgp0 == wgp); 4780 } 4781 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4782 wg->wg_npeers++; 4783 mutex_exit(wg->wg_lock); 4784 4785 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4786 4787 out: 4788 kmem_free(buf, ifd->ifd_len + 1); 4789 return error; 4790 } 4791 4792 static int 4793 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4794 { 4795 int error; 4796 prop_dictionary_t prop_dict; 4797 char *buf = NULL; 4798 const char *name; 4799 4800 error = wg_alloc_prop_buf(&buf, ifd); 4801 if (error != 0) 4802 return error; 4803 error = EINVAL; 4804 prop_dict = prop_dictionary_internalize(buf); 4805 if (prop_dict == NULL) 4806 goto out; 4807 4808 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4809 goto out; 4810 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4811 goto out; 4812 4813 error = wg_destroy_peer_name(wg, name); 4814 out: 4815 kmem_free(buf, ifd->ifd_len + 1); 4816 return error; 4817 } 4818 4819 static bool 4820 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4821 { 4822 int au = cmd == SIOCGDRVSPEC ? 4823 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4824 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4825 return kauth_authorize_network(kauth_cred_get(), 4826 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4827 (void *)cmd, NULL) == 0; 4828 } 4829 4830 static int 4831 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4832 { 4833 int error = ENOMEM; 4834 prop_dictionary_t prop_dict; 4835 prop_array_t peers = NULL; 4836 char *buf; 4837 struct wg_peer *wgp; 4838 int s, i; 4839 4840 prop_dict = prop_dictionary_create(); 4841 if (prop_dict == NULL) 4842 goto error; 4843 4844 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4845 if (!prop_dictionary_set_data(prop_dict, "private_key", 4846 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4847 goto error; 4848 } 4849 4850 if (wg->wg_listen_port != 0) { 4851 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4852 wg->wg_listen_port)) 4853 goto error; 4854 } 4855 4856 if (wg->wg_npeers == 0) 4857 goto skip_peers; 4858 4859 peers = prop_array_create(); 4860 if (peers == NULL) 4861 goto error; 4862 4863 s = pserialize_read_enter(); 4864 i = 0; 4865 WG_PEER_READER_FOREACH(wgp, wg) { 4866 struct wg_sockaddr *wgsa; 4867 struct psref wgp_psref, wgsa_psref; 4868 prop_dictionary_t prop_peer; 4869 4870 wg_get_peer(wgp, &wgp_psref); 4871 pserialize_read_exit(s); 4872 4873 prop_peer = prop_dictionary_create(); 4874 if (prop_peer == NULL) 4875 goto next; 4876 4877 if (strlen(wgp->wgp_name) > 0) { 4878 if (!prop_dictionary_set_string(prop_peer, "name", 4879 wgp->wgp_name)) 4880 goto next; 4881 } 4882 4883 if (!prop_dictionary_set_data(prop_peer, "public_key", 4884 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4885 goto next; 4886 4887 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4888 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4889 sizeof(wgp->wgp_psk))) { 4890 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4891 if (!prop_dictionary_set_data(prop_peer, 4892 "preshared_key", 4893 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4894 goto next; 4895 } 4896 } 4897 4898 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4899 CTASSERT(AF_UNSPEC == 0); 4900 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4901 !prop_dictionary_set_data(prop_peer, "endpoint", 4902 wgsatoss(wgsa), 4903 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4904 wg_put_sa(wgp, wgsa, &wgsa_psref); 4905 goto next; 4906 } 4907 wg_put_sa(wgp, wgsa, &wgsa_psref); 4908 4909 const struct timespec *t = &wgp->wgp_last_handshake_time; 4910 4911 if (!prop_dictionary_set_uint64(prop_peer, 4912 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4913 goto next; 4914 if (!prop_dictionary_set_uint32(prop_peer, 4915 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4916 goto next; 4917 4918 if (wgp->wgp_n_allowedips == 0) 4919 goto skip_allowedips; 4920 4921 prop_array_t allowedips = prop_array_create(); 4922 if (allowedips == NULL) 4923 goto next; 4924 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4925 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4926 prop_dictionary_t prop_allowedip; 4927 4928 prop_allowedip = prop_dictionary_create(); 4929 if (prop_allowedip == NULL) 4930 break; 4931 4932 if (!prop_dictionary_set_int(prop_allowedip, "family", 4933 wga->wga_family)) 4934 goto _next; 4935 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4936 wga->wga_cidr)) 4937 goto _next; 4938 4939 switch (wga->wga_family) { 4940 case AF_INET: 4941 if (!prop_dictionary_set_data(prop_allowedip, 4942 "ip", &wga->wga_addr4, 4943 sizeof(wga->wga_addr4))) 4944 goto _next; 4945 break; 4946 #ifdef INET6 4947 case AF_INET6: 4948 if (!prop_dictionary_set_data(prop_allowedip, 4949 "ip", &wga->wga_addr6, 4950 sizeof(wga->wga_addr6))) 4951 goto _next; 4952 break; 4953 #endif 4954 default: 4955 break; 4956 } 4957 prop_array_set(allowedips, j, prop_allowedip); 4958 _next: 4959 prop_object_release(prop_allowedip); 4960 } 4961 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4962 prop_object_release(allowedips); 4963 4964 skip_allowedips: 4965 4966 prop_array_set(peers, i, prop_peer); 4967 next: 4968 if (prop_peer) 4969 prop_object_release(prop_peer); 4970 i++; 4971 4972 s = pserialize_read_enter(); 4973 wg_put_peer(wgp, &wgp_psref); 4974 } 4975 pserialize_read_exit(s); 4976 4977 prop_dictionary_set(prop_dict, "peers", peers); 4978 prop_object_release(peers); 4979 peers = NULL; 4980 4981 skip_peers: 4982 buf = prop_dictionary_externalize(prop_dict); 4983 if (buf == NULL) 4984 goto error; 4985 if (ifd->ifd_len < (strlen(buf) + 1)) { 4986 error = EINVAL; 4987 goto error; 4988 } 4989 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4990 4991 free(buf, 0); 4992 error: 4993 if (peers != NULL) 4994 prop_object_release(peers); 4995 if (prop_dict != NULL) 4996 prop_object_release(prop_dict); 4997 4998 return error; 4999 } 5000 5001 static int 5002 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 5003 { 5004 struct wg_softc *wg = ifp->if_softc; 5005 struct ifreq *ifr = data; 5006 struct ifaddr *ifa = data; 5007 struct ifdrv *ifd = data; 5008 int error = 0; 5009 5010 switch (cmd) { 5011 case SIOCINITIFADDR: 5012 if (ifa->ifa_addr->sa_family != AF_LINK && 5013 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 5014 (IFF_UP | IFF_RUNNING)) { 5015 ifp->if_flags |= IFF_UP; 5016 error = if_init(ifp); 5017 } 5018 return error; 5019 case SIOCADDMULTI: 5020 case SIOCDELMULTI: 5021 switch (ifr->ifr_addr.sa_family) { 5022 case AF_INET: /* IP supports Multicast */ 5023 break; 5024 #ifdef INET6 5025 case AF_INET6: /* IP6 supports Multicast */ 5026 break; 5027 #endif 5028 default: /* Other protocols doesn't support Multicast */ 5029 error = EAFNOSUPPORT; 5030 break; 5031 } 5032 return error; 5033 case SIOCSDRVSPEC: 5034 if (!wg_is_authorized(wg, cmd)) { 5035 return EPERM; 5036 } 5037 switch (ifd->ifd_cmd) { 5038 case WG_IOCTL_SET_PRIVATE_KEY: 5039 error = wg_ioctl_set_private_key(wg, ifd); 5040 break; 5041 case WG_IOCTL_SET_LISTEN_PORT: 5042 error = wg_ioctl_set_listen_port(wg, ifd); 5043 break; 5044 case WG_IOCTL_ADD_PEER: 5045 error = wg_ioctl_add_peer(wg, ifd); 5046 break; 5047 case WG_IOCTL_DELETE_PEER: 5048 error = wg_ioctl_delete_peer(wg, ifd); 5049 break; 5050 default: 5051 error = EINVAL; 5052 break; 5053 } 5054 return error; 5055 case SIOCGDRVSPEC: 5056 return wg_ioctl_get(wg, ifd); 5057 case SIOCSIFFLAGS: 5058 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 5059 break; 5060 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 5061 case IFF_RUNNING: 5062 /* 5063 * If interface is marked down and it is running, 5064 * then stop and disable it. 5065 */ 5066 if_stop(ifp, 1); 5067 break; 5068 case IFF_UP: 5069 /* 5070 * If interface is marked up and it is stopped, then 5071 * start it. 5072 */ 5073 error = if_init(ifp); 5074 break; 5075 default: 5076 break; 5077 } 5078 return error; 5079 #ifdef WG_RUMPKERNEL 5080 case SIOCSLINKSTR: 5081 error = wg_ioctl_linkstr(wg, ifd); 5082 if (error == 0) 5083 wg->wg_ops = &wg_ops_rumpuser; 5084 return error; 5085 #endif 5086 default: 5087 break; 5088 } 5089 5090 error = ifioctl_common(ifp, cmd, data); 5091 5092 #ifdef WG_RUMPKERNEL 5093 if (!wg_user_mode(wg)) 5094 return error; 5095 5096 /* Do the same to the corresponding tun device on the host */ 5097 /* 5098 * XXX Actually the command has not been handled yet. It 5099 * will be handled via pr_ioctl form doifioctl later. 5100 */ 5101 switch (cmd) { 5102 case SIOCAIFADDR: 5103 case SIOCDIFADDR: { 5104 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 5105 struct in_aliasreq *ifra = &_ifra; 5106 KASSERT(error == ENOTTY); 5107 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 5108 IFNAMSIZ); 5109 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 5110 if (error == 0) 5111 error = ENOTTY; 5112 break; 5113 } 5114 #ifdef INET6 5115 case SIOCAIFADDR_IN6: 5116 case SIOCDIFADDR_IN6: { 5117 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 5118 struct in6_aliasreq *ifra = &_ifra; 5119 KASSERT(error == ENOTTY); 5120 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 5121 IFNAMSIZ); 5122 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 5123 if (error == 0) 5124 error = ENOTTY; 5125 break; 5126 } 5127 #endif 5128 } 5129 #endif /* WG_RUMPKERNEL */ 5130 5131 return error; 5132 } 5133 5134 static int 5135 wg_init(struct ifnet *ifp) 5136 { 5137 5138 ifp->if_flags |= IFF_RUNNING; 5139 5140 /* TODO flush pending packets. */ 5141 return 0; 5142 } 5143 5144 #ifdef ALTQ 5145 static void 5146 wg_start(struct ifnet *ifp) 5147 { 5148 struct mbuf *m; 5149 5150 for (;;) { 5151 IFQ_DEQUEUE(&ifp->if_snd, m); 5152 if (m == NULL) 5153 break; 5154 5155 kpreempt_disable(); 5156 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 5157 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 5158 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 5159 if_name(ifp)); 5160 m_freem(m); 5161 } 5162 kpreempt_enable(); 5163 } 5164 } 5165 #endif 5166 5167 static void 5168 wg_stop(struct ifnet *ifp, int disable) 5169 { 5170 5171 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 5172 ifp->if_flags &= ~IFF_RUNNING; 5173 5174 /* Need to do something? */ 5175 } 5176 5177 #ifdef WG_DEBUG_PARAMS 5178 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 5179 { 5180 const struct sysctlnode *node = NULL; 5181 5182 sysctl_createv(clog, 0, NULL, &node, 5183 CTLFLAG_PERMANENT, 5184 CTLTYPE_NODE, "wg", 5185 SYSCTL_DESCR("wg(4)"), 5186 NULL, 0, NULL, 0, 5187 CTL_NET, CTL_CREATE, CTL_EOL); 5188 sysctl_createv(clog, 0, &node, NULL, 5189 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5190 CTLTYPE_QUAD, "rekey_after_messages", 5191 SYSCTL_DESCR("session liftime by messages"), 5192 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 5193 sysctl_createv(clog, 0, &node, NULL, 5194 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5195 CTLTYPE_INT, "rekey_after_time", 5196 SYSCTL_DESCR("session liftime"), 5197 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 5198 sysctl_createv(clog, 0, &node, NULL, 5199 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5200 CTLTYPE_INT, "rekey_timeout", 5201 SYSCTL_DESCR("session handshake retry time"), 5202 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 5203 sysctl_createv(clog, 0, &node, NULL, 5204 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5205 CTLTYPE_INT, "rekey_attempt_time", 5206 SYSCTL_DESCR("session handshake timeout"), 5207 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 5208 sysctl_createv(clog, 0, &node, NULL, 5209 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5210 CTLTYPE_INT, "keepalive_timeout", 5211 SYSCTL_DESCR("keepalive timeout"), 5212 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 5213 sysctl_createv(clog, 0, &node, NULL, 5214 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5215 CTLTYPE_BOOL, "force_underload", 5216 SYSCTL_DESCR("force to detemine under load"), 5217 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 5218 sysctl_createv(clog, 0, &node, NULL, 5219 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5220 CTLTYPE_INT, "debug", 5221 SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"), 5222 NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL); 5223 } 5224 #endif 5225 5226 #ifdef WG_RUMPKERNEL 5227 static bool 5228 wg_user_mode(struct wg_softc *wg) 5229 { 5230 5231 return wg->wg_user != NULL; 5232 } 5233 5234 static int 5235 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 5236 { 5237 struct ifnet *ifp = &wg->wg_if; 5238 int error; 5239 5240 if (ifp->if_flags & IFF_UP) 5241 return EBUSY; 5242 5243 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 5244 /* XXX do nothing */ 5245 return 0; 5246 } else if (ifd->ifd_cmd != 0) { 5247 return EINVAL; 5248 } else if (wg->wg_user != NULL) { 5249 return EBUSY; 5250 } 5251 5252 /* Assume \0 included */ 5253 if (ifd->ifd_len > IFNAMSIZ) { 5254 return E2BIG; 5255 } else if (ifd->ifd_len < 1) { 5256 return EINVAL; 5257 } 5258 5259 char tun_name[IFNAMSIZ]; 5260 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 5261 if (error != 0) 5262 return error; 5263 5264 if (strncmp(tun_name, "tun", 3) != 0) 5265 return EINVAL; 5266 5267 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 5268 5269 return error; 5270 } 5271 5272 static int 5273 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 5274 { 5275 int error; 5276 struct psref psref; 5277 struct wg_sockaddr *wgsa; 5278 struct wg_softc *wg = wgp->wgp_sc; 5279 struct iovec iov[1]; 5280 5281 wgsa = wg_get_endpoint_sa(wgp, &psref); 5282 5283 iov[0].iov_base = mtod(m, void *); 5284 iov[0].iov_len = m->m_len; 5285 5286 /* Send messages to a peer via an ordinary socket. */ 5287 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 5288 5289 wg_put_sa(wgp, wgsa, &psref); 5290 5291 m_freem(m); 5292 5293 return error; 5294 } 5295 5296 static void 5297 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 5298 { 5299 struct wg_softc *wg = ifp->if_softc; 5300 struct iovec iov[2]; 5301 struct sockaddr_storage ss; 5302 5303 KASSERT(af == AF_INET || af == AF_INET6); 5304 5305 WG_TRACE(""); 5306 5307 if (af == AF_INET) { 5308 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 5309 struct ip *ip; 5310 5311 KASSERT(m->m_len >= sizeof(struct ip)); 5312 ip = mtod(m, struct ip *); 5313 sockaddr_in_init(sin, &ip->ip_dst, 0); 5314 } else { 5315 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 5316 struct ip6_hdr *ip6; 5317 5318 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 5319 ip6 = mtod(m, struct ip6_hdr *); 5320 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 5321 } 5322 5323 iov[0].iov_base = &ss; 5324 iov[0].iov_len = ss.ss_len; 5325 iov[1].iov_base = mtod(m, void *); 5326 iov[1].iov_len = m->m_len; 5327 5328 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5329 5330 /* Send decrypted packets to users via a tun. */ 5331 rumpuser_wg_send_user(wg->wg_user, iov, 2); 5332 5333 m_freem(m); 5334 } 5335 5336 static int 5337 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 5338 { 5339 int error; 5340 uint16_t old_port = wg->wg_listen_port; 5341 5342 if (port != 0 && old_port == port) 5343 return 0; 5344 5345 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5346 if (error == 0) 5347 wg->wg_listen_port = port; 5348 return error; 5349 } 5350 5351 /* 5352 * Receive user packets. 5353 */ 5354 void 5355 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5356 { 5357 struct ifnet *ifp = &wg->wg_if; 5358 struct mbuf *m; 5359 const struct sockaddr *dst; 5360 5361 WG_TRACE(""); 5362 5363 dst = iov[0].iov_base; 5364 5365 m = m_gethdr(M_DONTWAIT, MT_DATA); 5366 if (m == NULL) 5367 return; 5368 m->m_len = m->m_pkthdr.len = 0; 5369 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5370 5371 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5372 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5373 5374 (void)wg_output(ifp, m, dst, NULL); 5375 } 5376 5377 /* 5378 * Receive packets from a peer. 5379 */ 5380 void 5381 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5382 { 5383 struct mbuf *m; 5384 const struct sockaddr *src; 5385 int bound; 5386 5387 WG_TRACE(""); 5388 5389 src = iov[0].iov_base; 5390 5391 m = m_gethdr(M_DONTWAIT, MT_DATA); 5392 if (m == NULL) 5393 return; 5394 m->m_len = m->m_pkthdr.len = 0; 5395 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5396 5397 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5398 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5399 5400 bound = curlwp_bind(); 5401 wg_handle_packet(wg, m, src); 5402 curlwp_bindx(bound); 5403 } 5404 #endif /* WG_RUMPKERNEL */ 5405 5406 /* 5407 * Module infrastructure 5408 */ 5409 #include "if_module.h" 5410 5411 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5412
Indexes created Wed Sep 24 15:09:51 GMT 2025