if_wg.c revision 1.104
1 /* $NetBSD: if_wg.c,v 1.104 2024/07/28 14:47:37 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.104 2024/07/28 14:47:37 riastradh Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <lib/libkern/libkern.h> 87 88 #include <net/bpf.h> 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/if_wg.h> 92 #include <net/pktqueue.h> 93 #include <net/route.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip.h> 99 #include <netinet/ip_var.h> 100 #include <netinet/udp.h> 101 #include <netinet/udp_var.h> 102 103 #ifdef INET6 104 #include <netinet/ip6.h> 105 #include <netinet6/in6_pcb.h> 106 #include <netinet6/in6_var.h> 107 #include <netinet6/ip6_var.h> 108 #include <netinet6/udp6_var.h> 109 #endif /* INET6 */ 110 111 #include <prop/proplib.h> 112 113 #include <crypto/blake2/blake2s.h> 114 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 115 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 116 #include <crypto/sodium/crypto_scalarmult.h> 117 118 #include "ioconf.h" 119 120 #ifdef WG_RUMPKERNEL 121 #include "wg_user.h" 122 #endif 123 124 /* 125 * Data structures 126 * - struct wg_softc is an instance of wg interfaces 127 * - It has a list of peers (struct wg_peer) 128 * - It has a threadpool job that sends/receives handshake messages and 129 * runs event handlers 130 * - It has its own two routing tables: one is for IPv4 and the other IPv6 131 * - struct wg_peer is a representative of a peer 132 * - It has a struct work to handle handshakes and timer tasks 133 * - It has a pair of session instances (struct wg_session) 134 * - It has a pair of endpoint instances (struct wg_sockaddr) 135 * - Normally one endpoint is used and the second one is used only on 136 * a peer migration (a change of peer's IP address) 137 * - It has a list of IP addresses and sub networks called allowedips 138 * (struct wg_allowedip) 139 * - A packets sent over a session is allowed if its destination matches 140 * any IP addresses or sub networks of the list 141 * - struct wg_session represents a session of a secure tunnel with a peer 142 * - Two instances of sessions belong to a peer; a stable session and a 143 * unstable session 144 * - A handshake process of a session always starts with a unstable instance 145 * - Once a session is established, its instance becomes stable and the 146 * other becomes unstable instead 147 * - Data messages are always sent via a stable session 148 * 149 * Locking notes: 150 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 151 * - Changes to the peer list are serialized by wg_lock 152 * - The peer list may be read with pserialize(9) and psref(9) 153 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 154 * => XXX replace by pserialize when routing table is psz-safe 155 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 156 * only in thread context and serializes: 157 * - the stable and unstable session pointers 158 * - all unstable session state 159 * - Packet processing may be done in softint context: 160 * - The stable session can be read under pserialize(9) or psref(9) 161 * - The stable session is always ESTABLISHED 162 * - On a session swap, we must wait for all readers to release a 163 * reference to a stable session before changing wgs_state and 164 * session states 165 * - Lock order: wg_lock -> wgp_lock 166 */ 167 168 169 #define WGLOG(level, fmt, args...) \ 170 log(level, "%s: " fmt, __func__, ##args) 171 172 #define WG_DEBUG 173 174 /* Debug options */ 175 #ifdef WG_DEBUG 176 /* Output debug logs */ 177 #ifndef WG_DEBUG_LOG 178 #define WG_DEBUG_LOG 179 #endif 180 /* Output trace logs */ 181 #ifndef WG_DEBUG_TRACE 182 #define WG_DEBUG_TRACE 183 #endif 184 /* Output hash values, etc. */ 185 #ifndef WG_DEBUG_DUMP 186 #define WG_DEBUG_DUMP 187 #endif 188 /* debug packets */ 189 #ifndef WG_DEBUG_PACKET 190 #define WG_DEBUG_PACKET 191 #endif 192 /* Make some internal parameters configurable for testing and debugging */ 193 #ifndef WG_DEBUG_PARAMS 194 #define WG_DEBUG_PARAMS 195 #endif 196 #endif /* WG_DEBUG */ 197 198 #ifndef WG_DEBUG 199 # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \ 200 defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS) || \ 201 defined(WG_DEBUG_PACKET) 202 # define WG_DEBUG 203 # endif 204 #endif 205 206 #ifdef WG_DEBUG 207 int wg_debug; 208 #define WG_DEBUG_FLAGS_LOG 1 209 #define WG_DEBUG_FLAGS_TRACE 2 210 #define WG_DEBUG_FLAGS_DUMP 4 211 #define WG_DEBUG_FLAGS_PACKET 8 212 #endif 213 214 215 #ifdef WG_DEBUG_TRACE 216 #define WG_TRACE(msg) do { \ 217 if (wg_debug & WG_DEBUG_FLAGS_TRACE) \ 218 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \ 219 } while (0) 220 #else 221 #define WG_TRACE(msg) __nothing 222 #endif 223 224 #ifdef WG_DEBUG_LOG 225 #define WG_DLOG(fmt, args...) do { \ 226 if (wg_debug & WG_DEBUG_FLAGS_LOG) \ 227 log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \ 228 } while (0) 229 #else 230 #define WG_DLOG(fmt, args...) __nothing 231 #endif 232 233 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 234 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 235 &(wgprc)->wgprc_curpps, 1)) { \ 236 log(level, fmt, ##args); \ 237 } \ 238 } while (0) 239 240 #ifdef WG_DEBUG_PARAMS 241 static bool wg_force_underload = false; 242 #endif 243 244 #ifdef WG_DEBUG_DUMP 245 246 static char enomem[10] = "[enomem]"; 247 248 #define MAX_HDUMP_LEN 10000 /* large enough */ 249 250 251 static char * 252 gethexdump(const void *vp, size_t n) 253 { 254 char *buf; 255 const uint8_t *p = vp; 256 size_t i, alloc; 257 258 alloc = n; 259 if (n > MAX_HDUMP_LEN) 260 alloc = MAX_HDUMP_LEN; 261 buf = kmem_alloc(3 * alloc + 5, KM_NOSLEEP); 262 if (buf == NULL) 263 return enomem; 264 for (i = 0; i < alloc; i++) 265 snprintf(buf + 3 * i, 3 + 1, " %02hhx", p[i]); 266 if (alloc != n) 267 snprintf(buf + 3 * i, 4 + 1, " ..."); 268 return buf; 269 } 270 271 static void 272 puthexdump(char *buf, const void *p, size_t n) 273 { 274 275 if (buf == NULL || buf == enomem) 276 return; 277 if (n > MAX_HDUMP_LEN) 278 n = MAX_HDUMP_LEN; 279 kmem_free(buf, 3 * n + 5); 280 } 281 282 #ifdef WG_RUMPKERNEL 283 static void 284 wg_dump_buf(const char *func, const char *buf, const size_t size) 285 { 286 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 287 return; 288 289 char *hex = gethexdump(buf, size); 290 291 log(LOG_DEBUG, "%s: %s\n", func, hex); 292 puthexdump(hex, buf, size); 293 } 294 #endif 295 296 static void 297 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 298 const size_t size) 299 { 300 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 301 return; 302 303 char *hex = gethexdump(hash, size); 304 305 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex); 306 puthexdump(hex, hash, size); 307 } 308 309 #define WG_DUMP_HASH(name, hash) \ 310 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 311 #define WG_DUMP_HASH48(name, hash) \ 312 wg_dump_hash(__func__, name, hash, 48) 313 #define WG_DUMP_BUF(buf, size) \ 314 wg_dump_buf(__func__, buf, size) 315 #else 316 #define WG_DUMP_HASH(name, hash) __nothing 317 #define WG_DUMP_HASH48(name, hash) __nothing 318 #define WG_DUMP_BUF(buf, size) __nothing 319 #endif /* WG_DEBUG_DUMP */ 320 321 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 322 #define WG_MAX_PROPLEN 32766 323 324 #define WG_MTU 1420 325 #define WG_ALLOWEDIPS 16 326 327 #define CURVE25519_KEY_LEN 32 328 #define TAI64N_LEN sizeof(uint32_t) * 3 329 #define POLY1305_AUTHTAG_LEN 16 330 #define HMAC_BLOCK_LEN 64 331 332 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 333 /* [N] 4.3: Hash functions */ 334 #define NOISE_DHLEN 32 335 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 336 #define NOISE_HASHLEN 32 337 #define NOISE_BLOCKLEN 64 338 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 339 /* [N] 5.1: "k" */ 340 #define NOISE_CIPHER_KEY_LEN 32 341 /* 342 * [N] 9.2: "psk" 343 * "... psk is a 32-byte secret value provided by the application." 344 */ 345 #define NOISE_PRESHARED_KEY_LEN 32 346 347 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 348 #define WG_TIMESTAMP_LEN TAI64N_LEN 349 350 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 351 352 #define WG_COOKIE_LEN 16 353 #define WG_MAC_LEN 16 354 #define WG_COOKIESECRET_LEN 32 355 356 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 357 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 358 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 359 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 360 #define WG_HASH_LEN NOISE_HASHLEN 361 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 362 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 363 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 364 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 365 #define WG_DATA_KEY_LEN 32 366 #define WG_SALT_LEN 24 367 368 /* 369 * The protocol messages 370 */ 371 struct wg_msg { 372 uint32_t wgm_type; 373 } __packed; 374 375 /* [W] 5.4.2 First Message: Initiator to Responder */ 376 struct wg_msg_init { 377 uint32_t wgmi_type; 378 uint32_t wgmi_sender; 379 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 380 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 381 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 382 uint8_t wgmi_mac1[WG_MAC_LEN]; 383 uint8_t wgmi_mac2[WG_MAC_LEN]; 384 } __packed; 385 386 /* [W] 5.4.3 Second Message: Responder to Initiator */ 387 struct wg_msg_resp { 388 uint32_t wgmr_type; 389 uint32_t wgmr_sender; 390 uint32_t wgmr_receiver; 391 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 392 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 393 uint8_t wgmr_mac1[WG_MAC_LEN]; 394 uint8_t wgmr_mac2[WG_MAC_LEN]; 395 } __packed; 396 397 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 398 struct wg_msg_data { 399 uint32_t wgmd_type; 400 uint32_t wgmd_receiver; 401 uint64_t wgmd_counter; 402 uint32_t wgmd_packet[0]; 403 } __packed; 404 405 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 406 struct wg_msg_cookie { 407 uint32_t wgmc_type; 408 uint32_t wgmc_receiver; 409 uint8_t wgmc_salt[WG_SALT_LEN]; 410 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 411 } __packed; 412 413 #define WG_MSG_TYPE_INIT 1 414 #define WG_MSG_TYPE_RESP 2 415 #define WG_MSG_TYPE_COOKIE 3 416 #define WG_MSG_TYPE_DATA 4 417 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 418 419 /* Sliding windows */ 420 421 #define SLIWIN_BITS 2048u 422 #define SLIWIN_TYPE uint32_t 423 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 424 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 425 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 426 427 struct sliwin { 428 SLIWIN_TYPE B[SLIWIN_WORDS]; 429 uint64_t T; 430 }; 431 432 static void 433 sliwin_reset(struct sliwin *W) 434 { 435 436 memset(W, 0, sizeof(*W)); 437 } 438 439 static int 440 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 441 { 442 443 /* 444 * If it's more than one window older than the highest sequence 445 * number we've seen, reject. 446 */ 447 #ifdef __HAVE_ATOMIC64_LOADSTORE 448 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 449 return EAUTH; 450 #endif 451 452 /* 453 * Otherwise, we need to take the lock to decide, so don't 454 * reject just yet. Caller must serialize a call to 455 * sliwin_update in this case. 456 */ 457 return 0; 458 } 459 460 static int 461 sliwin_update(struct sliwin *W, uint64_t S) 462 { 463 unsigned word, bit; 464 465 /* 466 * If it's more than one window older than the highest sequence 467 * number we've seen, reject. 468 */ 469 if (S + SLIWIN_NPKT < W->T) 470 return EAUTH; 471 472 /* 473 * If it's higher than the highest sequence number we've seen, 474 * advance the window. 475 */ 476 if (S > W->T) { 477 uint64_t i = W->T / SLIWIN_BPW; 478 uint64_t j = S / SLIWIN_BPW; 479 unsigned k; 480 481 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 482 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 483 #ifdef __HAVE_ATOMIC64_LOADSTORE 484 atomic_store_relaxed(&W->T, S); 485 #else 486 W->T = S; 487 #endif 488 } 489 490 /* Test and set the bit -- if already set, reject. */ 491 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 492 bit = S % SLIWIN_BPW; 493 if (W->B[word] & (1UL << bit)) 494 return EAUTH; 495 W->B[word] |= 1U << bit; 496 497 /* Accept! */ 498 return 0; 499 } 500 501 struct wg_session { 502 struct wg_peer *wgs_peer; 503 struct psref_target 504 wgs_psref; 505 506 int wgs_state; 507 #define WGS_STATE_UNKNOWN 0 508 #define WGS_STATE_INIT_ACTIVE 1 509 #define WGS_STATE_INIT_PASSIVE 2 510 #define WGS_STATE_ESTABLISHED 3 511 #define WGS_STATE_DESTROYING 4 512 513 volatile uint32_t 514 wgs_time_established; 515 volatile uint32_t 516 wgs_time_last_data_sent; 517 bool wgs_is_initiator; 518 519 uint32_t wgs_local_index; 520 uint32_t wgs_remote_index; 521 #ifdef __HAVE_ATOMIC64_LOADSTORE 522 volatile uint64_t 523 wgs_send_counter; 524 #else 525 kmutex_t wgs_send_counter_lock; 526 uint64_t wgs_send_counter; 527 #endif 528 529 struct { 530 kmutex_t lock; 531 struct sliwin window; 532 } *wgs_recvwin; 533 534 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 535 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 536 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 537 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 538 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 539 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 540 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 541 }; 542 543 struct wg_sockaddr { 544 union { 545 struct sockaddr_storage _ss; 546 struct sockaddr _sa; 547 struct sockaddr_in _sin; 548 struct sockaddr_in6 _sin6; 549 }; 550 struct psref_target wgsa_psref; 551 }; 552 553 #define wgsatoss(wgsa) (&(wgsa)->_ss) 554 #define wgsatosa(wgsa) (&(wgsa)->_sa) 555 #define wgsatosin(wgsa) (&(wgsa)->_sin) 556 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 557 558 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 559 560 struct wg_peer; 561 struct wg_allowedip { 562 struct radix_node wga_nodes[2]; 563 struct wg_sockaddr _wga_sa_addr; 564 struct wg_sockaddr _wga_sa_mask; 565 #define wga_sa_addr _wga_sa_addr._sa 566 #define wga_sa_mask _wga_sa_mask._sa 567 568 int wga_family; 569 uint8_t wga_cidr; 570 union { 571 struct in_addr _ip4; 572 struct in6_addr _ip6; 573 } wga_addr; 574 #define wga_addr4 wga_addr._ip4 575 #define wga_addr6 wga_addr._ip6 576 577 struct wg_peer *wga_peer; 578 }; 579 580 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 581 582 struct wg_ppsratecheck { 583 struct timeval wgprc_lasttime; 584 int wgprc_curpps; 585 }; 586 587 struct wg_softc; 588 struct wg_peer { 589 struct wg_softc *wgp_sc; 590 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 591 struct pslist_entry wgp_peerlist_entry; 592 pserialize_t wgp_psz; 593 struct psref_target wgp_psref; 594 kmutex_t *wgp_lock; 595 kmutex_t *wgp_intr_lock; 596 597 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 598 struct wg_sockaddr *wgp_endpoint; 599 struct wg_sockaddr *wgp_endpoint0; 600 volatile unsigned wgp_endpoint_changing; 601 bool wgp_endpoint_available; 602 603 /* The preshared key (optional) */ 604 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 605 606 struct wg_session *wgp_session_stable; 607 struct wg_session *wgp_session_unstable; 608 609 /* first outgoing packet awaiting session initiation */ 610 struct mbuf *volatile wgp_pending; 611 612 /* timestamp in big-endian */ 613 wg_timestamp_t wgp_timestamp_latest_init; 614 615 struct timespec wgp_last_handshake_time; 616 617 callout_t wgp_handshake_timeout_timer; 618 callout_t wgp_session_dtor_timer; 619 620 time_t wgp_handshake_start_time; 621 622 volatile unsigned wgp_force_rekey; 623 624 int wgp_n_allowedips; 625 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 626 627 time_t wgp_latest_cookie_time; 628 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 629 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 630 bool wgp_last_sent_mac1_valid; 631 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 632 bool wgp_last_sent_cookie_valid; 633 634 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 635 636 time_t wgp_last_cookiesecret_time; 637 uint8_t wgp_cookiesecret[WG_COOKIESECRET_LEN]; 638 639 struct wg_ppsratecheck wgp_ppsratecheck; 640 641 struct work wgp_work; 642 unsigned int wgp_tasks; 643 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 644 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 645 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 646 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 647 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 648 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 649 }; 650 651 struct wg_ops; 652 653 struct wg_softc { 654 struct ifnet wg_if; 655 LIST_ENTRY(wg_softc) wg_list; 656 kmutex_t *wg_lock; 657 kmutex_t *wg_intr_lock; 658 krwlock_t *wg_rwlock; 659 660 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 661 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 662 663 int wg_npeers; 664 struct pslist_head wg_peers; 665 struct thmap *wg_peers_bypubkey; 666 struct thmap *wg_peers_byname; 667 struct thmap *wg_sessions_byindex; 668 uint16_t wg_listen_port; 669 670 struct threadpool *wg_threadpool; 671 672 struct threadpool_job wg_job; 673 int wg_upcalls; 674 #define WG_UPCALL_INET __BIT(0) 675 #define WG_UPCALL_INET6 __BIT(1) 676 677 #ifdef INET 678 struct socket *wg_so4; 679 struct radix_node_head *wg_rtable_ipv4; 680 #endif 681 #ifdef INET6 682 struct socket *wg_so6; 683 struct radix_node_head *wg_rtable_ipv6; 684 #endif 685 686 struct wg_ppsratecheck wg_ppsratecheck; 687 688 struct wg_ops *wg_ops; 689 690 #ifdef WG_RUMPKERNEL 691 struct wg_user *wg_user; 692 #endif 693 }; 694 695 /* [W] 6.1 Preliminaries */ 696 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 697 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 698 #define WG_REKEY_AFTER_TIME 120 699 #define WG_REJECT_AFTER_TIME 180 700 #define WG_REKEY_ATTEMPT_TIME 90 701 #define WG_REKEY_TIMEOUT 5 702 #define WG_KEEPALIVE_TIMEOUT 10 703 704 #define WG_COOKIE_TIME 120 705 #define WG_COOKIESECRET_TIME (2 * 60) 706 707 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 708 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 709 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 710 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 711 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 712 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 713 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 714 715 static struct mbuf * 716 wg_get_mbuf(size_t, size_t); 717 718 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 719 struct mbuf *); 720 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 721 const uint32_t, const uint8_t [WG_MAC_LEN], 722 const struct sockaddr *); 723 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 724 struct wg_session *, const struct wg_msg_init *); 725 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 726 727 static struct wg_peer * 728 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 729 struct psref *); 730 static struct wg_peer * 731 wg_lookup_peer_by_pubkey(struct wg_softc *, 732 const uint8_t [WG_STATIC_KEY_LEN], struct psref *); 733 734 static struct wg_session * 735 wg_lookup_session_by_index(struct wg_softc *, 736 const uint32_t, struct psref *); 737 738 static void wg_update_endpoint_if_necessary(struct wg_peer *, 739 const struct sockaddr *); 740 741 static void wg_schedule_session_dtor_timer(struct wg_peer *); 742 743 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 744 static void wg_calculate_keys(struct wg_session *, const bool); 745 746 static void wg_clear_states(struct wg_session *); 747 748 static void wg_get_peer(struct wg_peer *, struct psref *); 749 static void wg_put_peer(struct wg_peer *, struct psref *); 750 751 static int wg_send_so(struct wg_peer *, struct mbuf *); 752 static int wg_send_udp(struct wg_peer *, struct mbuf *); 753 static int wg_output(struct ifnet *, struct mbuf *, 754 const struct sockaddr *, const struct rtentry *); 755 static void wg_input(struct ifnet *, struct mbuf *, const int); 756 static int wg_ioctl(struct ifnet *, u_long, void *); 757 static int wg_bind_port(struct wg_softc *, const uint16_t); 758 static int wg_init(struct ifnet *); 759 #ifdef ALTQ 760 static void wg_start(struct ifnet *); 761 #endif 762 static void wg_stop(struct ifnet *, int); 763 764 static void wg_peer_work(struct work *, void *); 765 static void wg_job(struct threadpool_job *); 766 static void wgintr(void *); 767 static void wg_purge_pending_packets(struct wg_peer *); 768 769 static int wg_clone_create(struct if_clone *, int); 770 static int wg_clone_destroy(struct ifnet *); 771 772 struct wg_ops { 773 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 774 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 775 void (*input)(struct ifnet *, struct mbuf *, const int); 776 int (*bind_port)(struct wg_softc *, const uint16_t); 777 }; 778 779 struct wg_ops wg_ops_rumpkernel = { 780 .send_hs_msg = wg_send_so, 781 .send_data_msg = wg_send_udp, 782 .input = wg_input, 783 .bind_port = wg_bind_port, 784 }; 785 786 #ifdef WG_RUMPKERNEL 787 static bool wg_user_mode(struct wg_softc *); 788 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 789 790 static int wg_send_user(struct wg_peer *, struct mbuf *); 791 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 792 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 793 794 struct wg_ops wg_ops_rumpuser = { 795 .send_hs_msg = wg_send_user, 796 .send_data_msg = wg_send_user, 797 .input = wg_input_user, 798 .bind_port = wg_bind_port_user, 799 }; 800 #endif 801 802 #define WG_PEER_READER_FOREACH(wgp, wg) \ 803 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 804 wgp_peerlist_entry) 805 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 806 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 807 wgp_peerlist_entry) 808 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 809 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 810 #define WG_PEER_WRITER_REMOVE(wgp) \ 811 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 812 813 struct wg_route { 814 struct radix_node wgr_nodes[2]; 815 struct wg_peer *wgr_peer; 816 }; 817 818 static struct radix_node_head * 819 wg_rnh(struct wg_softc *wg, const int family) 820 { 821 822 switch (family) { 823 case AF_INET: 824 return wg->wg_rtable_ipv4; 825 #ifdef INET6 826 case AF_INET6: 827 return wg->wg_rtable_ipv6; 828 #endif 829 default: 830 return NULL; 831 } 832 } 833 834 835 /* 836 * Global variables 837 */ 838 static volatile unsigned wg_count __cacheline_aligned; 839 840 struct psref_class *wg_psref_class __read_mostly; 841 842 static struct if_clone wg_cloner = 843 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 844 845 static struct pktqueue *wg_pktq __read_mostly; 846 static struct workqueue *wg_wq __read_mostly; 847 848 void wgattach(int); 849 /* ARGSUSED */ 850 void 851 wgattach(int count) 852 { 853 /* 854 * Nothing to do here, initialization is handled by the 855 * module initialization code in wginit() below). 856 */ 857 } 858 859 static void 860 wginit(void) 861 { 862 863 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 864 865 if_clone_attach(&wg_cloner); 866 } 867 868 /* 869 * XXX Kludge: This should just happen in wginit, but workqueue_create 870 * cannot be run until after CPUs have been detected, and wginit runs 871 * before configure. 872 */ 873 static int 874 wginitqueues(void) 875 { 876 int error __diagused; 877 878 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 879 KASSERT(wg_pktq != NULL); 880 881 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 882 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 883 KASSERT(error == 0); 884 885 return 0; 886 } 887 888 static void 889 wg_guarantee_initialized(void) 890 { 891 static ONCE_DECL(init); 892 int error __diagused; 893 894 error = RUN_ONCE(&init, wginitqueues); 895 KASSERT(error == 0); 896 } 897 898 static int 899 wg_count_inc(void) 900 { 901 unsigned o, n; 902 903 do { 904 o = atomic_load_relaxed(&wg_count); 905 if (o == UINT_MAX) 906 return ENFILE; 907 n = o + 1; 908 } while (atomic_cas_uint(&wg_count, o, n) != o); 909 910 return 0; 911 } 912 913 static void 914 wg_count_dec(void) 915 { 916 unsigned c __diagused; 917 918 c = atomic_dec_uint_nv(&wg_count); 919 KASSERT(c != UINT_MAX); 920 } 921 922 static int 923 wgdetach(void) 924 { 925 926 /* Prevent new interface creation. */ 927 if_clone_detach(&wg_cloner); 928 929 /* Check whether there are any existing interfaces. */ 930 if (atomic_load_relaxed(&wg_count)) { 931 /* Back out -- reattach the cloner. */ 932 if_clone_attach(&wg_cloner); 933 return EBUSY; 934 } 935 936 /* No interfaces left. Nuke it. */ 937 if (wg_wq) 938 workqueue_destroy(wg_wq); 939 if (wg_pktq) 940 pktq_destroy(wg_pktq); 941 psref_class_destroy(wg_psref_class); 942 943 return 0; 944 } 945 946 static void 947 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 948 uint8_t hash[WG_HASH_LEN]) 949 { 950 /* [W] 5.4: CONSTRUCTION */ 951 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 952 /* [W] 5.4: IDENTIFIER */ 953 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 954 struct blake2s state; 955 956 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 957 signature, strlen(signature)); 958 959 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 960 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 961 962 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 963 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 964 blake2s_update(&state, id, strlen(id)); 965 blake2s_final(&state, hash); 966 967 WG_DUMP_HASH("ckey", ckey); 968 WG_DUMP_HASH("hash", hash); 969 } 970 971 static void 972 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 973 const size_t inputsize) 974 { 975 struct blake2s state; 976 977 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 978 blake2s_update(&state, hash, WG_HASH_LEN); 979 blake2s_update(&state, input, inputsize); 980 blake2s_final(&state, hash); 981 } 982 983 static void 984 wg_algo_mac(uint8_t out[], const size_t outsize, 985 const uint8_t key[], const size_t keylen, 986 const uint8_t input1[], const size_t input1len, 987 const uint8_t input2[], const size_t input2len) 988 { 989 struct blake2s state; 990 991 blake2s_init(&state, outsize, key, keylen); 992 993 blake2s_update(&state, input1, input1len); 994 if (input2 != NULL) 995 blake2s_update(&state, input2, input2len); 996 blake2s_final(&state, out); 997 } 998 999 static void 1000 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 1001 const uint8_t input1[], const size_t input1len, 1002 const uint8_t input2[], const size_t input2len) 1003 { 1004 struct blake2s state; 1005 /* [W] 5.4: LABEL-MAC1 */ 1006 const char *label = "mac1----"; 1007 uint8_t key[WG_HASH_LEN]; 1008 1009 blake2s_init(&state, sizeof(key), NULL, 0); 1010 blake2s_update(&state, label, strlen(label)); 1011 blake2s_update(&state, input1, input1len); 1012 blake2s_final(&state, key); 1013 1014 blake2s_init(&state, outsize, key, sizeof(key)); 1015 if (input2 != NULL) 1016 blake2s_update(&state, input2, input2len); 1017 blake2s_final(&state, out); 1018 } 1019 1020 static void 1021 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 1022 const uint8_t input1[], const size_t input1len) 1023 { 1024 struct blake2s state; 1025 /* [W] 5.4: LABEL-COOKIE */ 1026 const char *label = "cookie--"; 1027 1028 blake2s_init(&state, outsize, NULL, 0); 1029 blake2s_update(&state, label, strlen(label)); 1030 blake2s_update(&state, input1, input1len); 1031 blake2s_final(&state, out); 1032 } 1033 1034 static void 1035 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 1036 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 1037 { 1038 1039 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1040 1041 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 1042 crypto_scalarmult_base(pubkey, privkey); 1043 } 1044 1045 static void 1046 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 1047 const uint8_t privkey[WG_STATIC_KEY_LEN], 1048 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 1049 { 1050 1051 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1052 1053 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1054 KASSERT(ret == 0); 1055 } 1056 1057 static void 1058 wg_algo_hmac(uint8_t out[], const size_t outlen, 1059 const uint8_t key[], const size_t keylen, 1060 const uint8_t in[], const size_t inlen) 1061 { 1062 #define IPAD 0x36 1063 #define OPAD 0x5c 1064 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1065 uint8_t ipad[HMAC_BLOCK_LEN]; 1066 uint8_t opad[HMAC_BLOCK_LEN]; 1067 size_t i; 1068 struct blake2s state; 1069 1070 KASSERT(outlen == WG_HASH_LEN); 1071 KASSERT(keylen <= HMAC_BLOCK_LEN); 1072 1073 memcpy(hmackey, key, keylen); 1074 1075 for (i = 0; i < sizeof(hmackey); i++) { 1076 ipad[i] = hmackey[i] ^ IPAD; 1077 opad[i] = hmackey[i] ^ OPAD; 1078 } 1079 1080 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1081 blake2s_update(&state, ipad, sizeof(ipad)); 1082 blake2s_update(&state, in, inlen); 1083 blake2s_final(&state, out); 1084 1085 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1086 blake2s_update(&state, opad, sizeof(opad)); 1087 blake2s_update(&state, out, WG_HASH_LEN); 1088 blake2s_final(&state, out); 1089 #undef IPAD 1090 #undef OPAD 1091 } 1092 1093 static void 1094 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1095 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1096 const uint8_t input[], const size_t inputlen) 1097 { 1098 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1099 uint8_t one[1]; 1100 1101 /* 1102 * [N] 4.3: "an input_key_material byte sequence with length 1103 * either zero bytes, 32 bytes, or DHLEN bytes." 1104 */ 1105 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1106 1107 WG_DUMP_HASH("ckey", ckey); 1108 if (input != NULL) 1109 WG_DUMP_HASH("input", input); 1110 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1111 input, inputlen); 1112 WG_DUMP_HASH("tmp1", tmp1); 1113 one[0] = 1; 1114 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1115 one, sizeof(one)); 1116 WG_DUMP_HASH("out1", out1); 1117 if (out2 == NULL) 1118 return; 1119 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1120 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1121 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1122 tmp2, sizeof(tmp2)); 1123 WG_DUMP_HASH("out2", out2); 1124 if (out3 == NULL) 1125 return; 1126 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1127 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1128 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1129 tmp2, sizeof(tmp2)); 1130 WG_DUMP_HASH("out3", out3); 1131 } 1132 1133 static void __noinline 1134 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1135 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1136 const uint8_t local_key[WG_STATIC_KEY_LEN], 1137 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1138 { 1139 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1140 1141 wg_algo_dh(dhout, local_key, remote_key); 1142 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1143 1144 WG_DUMP_HASH("dhout", dhout); 1145 WG_DUMP_HASH("ckey", ckey); 1146 if (cipher_key != NULL) 1147 WG_DUMP_HASH("cipher_key", cipher_key); 1148 } 1149 1150 static void 1151 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1152 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1153 const uint8_t auth[], size_t authlen) 1154 { 1155 uint8_t nonce[(32 + 64) / 8] = {0}; 1156 long long unsigned int outsize; 1157 int error __diagused; 1158 1159 le64enc(&nonce[4], counter); 1160 1161 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1162 plainsize, auth, authlen, NULL, nonce, key); 1163 KASSERT(error == 0); 1164 KASSERT(outsize == expected_outsize); 1165 } 1166 1167 static int 1168 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1169 const uint64_t counter, const uint8_t encrypted[], 1170 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1171 { 1172 uint8_t nonce[(32 + 64) / 8] = {0}; 1173 long long unsigned int outsize; 1174 int error; 1175 1176 le64enc(&nonce[4], counter); 1177 1178 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1179 encrypted, encryptedsize, auth, authlen, nonce, key); 1180 if (error == 0) 1181 KASSERT(outsize == expected_outsize); 1182 return error; 1183 } 1184 1185 static void 1186 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1187 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1188 const uint8_t auth[], size_t authlen, 1189 const uint8_t nonce[WG_SALT_LEN]) 1190 { 1191 long long unsigned int outsize; 1192 int error __diagused; 1193 1194 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1195 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1196 plain, plainsize, auth, authlen, NULL, nonce, key); 1197 KASSERT(error == 0); 1198 KASSERT(outsize == expected_outsize); 1199 } 1200 1201 static int 1202 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1203 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1204 const uint8_t auth[], size_t authlen, 1205 const uint8_t nonce[WG_SALT_LEN]) 1206 { 1207 long long unsigned int outsize; 1208 int error; 1209 1210 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1211 encrypted, encryptedsize, auth, authlen, nonce, key); 1212 if (error == 0) 1213 KASSERT(outsize == expected_outsize); 1214 return error; 1215 } 1216 1217 static void 1218 wg_algo_tai64n(wg_timestamp_t timestamp) 1219 { 1220 struct timespec ts; 1221 1222 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1223 getnanotime(&ts); 1224 /* TAI64 label in external TAI64 format */ 1225 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1226 /* second beginning from 1970 TAI */ 1227 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1228 /* nanosecond in big-endian format */ 1229 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1230 } 1231 1232 /* 1233 * wg_get_stable_session(wgp, psref) 1234 * 1235 * Get a passive reference to the current stable session, or 1236 * return NULL if there is no current stable session. 1237 * 1238 * The pointer is always there but the session is not necessarily 1239 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1240 * the session may transition from ESTABLISHED to DESTROYING while 1241 * holding the passive reference. 1242 */ 1243 static struct wg_session * 1244 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1245 { 1246 int s; 1247 struct wg_session *wgs; 1248 1249 s = pserialize_read_enter(); 1250 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1251 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1252 wgs = NULL; 1253 else 1254 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1255 pserialize_read_exit(s); 1256 1257 return wgs; 1258 } 1259 1260 static void 1261 wg_put_session(struct wg_session *wgs, struct psref *psref) 1262 { 1263 1264 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1265 } 1266 1267 static void 1268 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1269 { 1270 struct wg_peer *wgp = wgs->wgs_peer; 1271 struct wg_session *wgs0 __diagused; 1272 void *garbage; 1273 1274 KASSERT(mutex_owned(wgp->wgp_lock)); 1275 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1276 1277 /* Remove the session from the table. */ 1278 wgs0 = thmap_del(wg->wg_sessions_byindex, 1279 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1280 KASSERT(wgs0 == wgs); 1281 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1282 1283 /* Wait for passive references to drain. */ 1284 pserialize_perform(wgp->wgp_psz); 1285 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1286 1287 /* 1288 * Free memory, zero state, and transition to UNKNOWN. We have 1289 * exclusive access to the session now, so there is no need for 1290 * an atomic store. 1291 */ 1292 thmap_gc(wg->wg_sessions_byindex, garbage); 1293 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_UNKNOWN\n", 1294 wgs->wgs_local_index, wgs->wgs_remote_index); 1295 wgs->wgs_local_index = 0; 1296 wgs->wgs_remote_index = 0; 1297 wg_clear_states(wgs); 1298 wgs->wgs_state = WGS_STATE_UNKNOWN; 1299 } 1300 1301 /* 1302 * wg_get_session_index(wg, wgs) 1303 * 1304 * Choose a session index for wgs->wgs_local_index, and store it 1305 * in wg's table of sessions by index. 1306 * 1307 * wgs must be the unstable session of its peer, and must be 1308 * transitioning out of the UNKNOWN state. 1309 */ 1310 static void 1311 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1312 { 1313 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1314 struct wg_session *wgs0; 1315 uint32_t index; 1316 1317 KASSERT(mutex_owned(wgp->wgp_lock)); 1318 KASSERT(wgs == wgp->wgp_session_unstable); 1319 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1320 wgs->wgs_state); 1321 1322 do { 1323 /* Pick a uniform random index. */ 1324 index = cprng_strong32(); 1325 1326 /* Try to take it. */ 1327 wgs->wgs_local_index = index; 1328 wgs0 = thmap_put(wg->wg_sessions_byindex, 1329 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1330 1331 /* If someone else beat us, start over. */ 1332 } while (__predict_false(wgs0 != wgs)); 1333 } 1334 1335 /* 1336 * wg_put_session_index(wg, wgs) 1337 * 1338 * Remove wgs from the table of sessions by index, wait for any 1339 * passive references to drain, and transition the session to the 1340 * UNKNOWN state. 1341 * 1342 * wgs must be the unstable session of its peer, and must not be 1343 * UNKNOWN or ESTABLISHED. 1344 */ 1345 static void 1346 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1347 { 1348 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1349 1350 KASSERT(mutex_owned(wgp->wgp_lock)); 1351 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1352 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1353 1354 wg_destroy_session(wg, wgs); 1355 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1356 } 1357 1358 /* 1359 * Handshake patterns 1360 * 1361 * [W] 5: "These messages use the "IK" pattern from Noise" 1362 * [N] 7.5. Interactive handshake patterns (fundamental) 1363 * "The first character refers to the initiators static key:" 1364 * "I = Static key for initiator Immediately transmitted to responder, 1365 * despite reduced or absent identity hiding" 1366 * "The second character refers to the responders static key:" 1367 * "K = Static key for responder Known to initiator" 1368 * "IK: 1369 * <- s 1370 * ... 1371 * -> e, es, s, ss 1372 * <- e, ee, se" 1373 * [N] 9.4. Pattern modifiers 1374 * "IKpsk2: 1375 * <- s 1376 * ... 1377 * -> e, es, s, ss 1378 * <- e, ee, se, psk" 1379 */ 1380 static void 1381 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1382 struct wg_session *wgs, struct wg_msg_init *wgmi) 1383 { 1384 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1385 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1386 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1387 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1388 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1389 1390 KASSERT(mutex_owned(wgp->wgp_lock)); 1391 KASSERT(wgs == wgp->wgp_session_unstable); 1392 KASSERTMSG(wgs->wgs_state == WGS_STATE_INIT_ACTIVE, "state=%d", 1393 wgs->wgs_state); 1394 1395 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1396 wgmi->wgmi_sender = wgs->wgs_local_index; 1397 1398 /* [W] 5.4.2: First Message: Initiator to Responder */ 1399 1400 /* Ci := HASH(CONSTRUCTION) */ 1401 /* Hi := HASH(Ci || IDENTIFIER) */ 1402 wg_init_key_and_hash(ckey, hash); 1403 /* Hi := HASH(Hi || Sr^pub) */ 1404 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1405 1406 WG_DUMP_HASH("hash", hash); 1407 1408 /* [N] 2.2: "e" */ 1409 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1410 wg_algo_generate_keypair(pubkey, privkey); 1411 /* Ci := KDF1(Ci, Ei^pub) */ 1412 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1413 /* msg.ephemeral := Ei^pub */ 1414 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1415 /* Hi := HASH(Hi || msg.ephemeral) */ 1416 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1417 1418 WG_DUMP_HASH("ckey", ckey); 1419 WG_DUMP_HASH("hash", hash); 1420 1421 /* [N] 2.2: "es" */ 1422 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1423 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1424 1425 /* [N] 2.2: "s" */ 1426 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1427 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1428 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1429 hash, sizeof(hash)); 1430 /* Hi := HASH(Hi || msg.static) */ 1431 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1432 1433 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1434 1435 /* [N] 2.2: "ss" */ 1436 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1437 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1438 1439 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1440 wg_timestamp_t timestamp; 1441 wg_algo_tai64n(timestamp); 1442 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1443 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1444 /* Hi := HASH(Hi || msg.timestamp) */ 1445 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1446 1447 /* [W] 5.4.4 Cookie MACs */ 1448 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1449 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1450 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1451 /* Need mac1 to decrypt a cookie from a cookie message */ 1452 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1453 sizeof(wgp->wgp_last_sent_mac1)); 1454 wgp->wgp_last_sent_mac1_valid = true; 1455 1456 if (wgp->wgp_latest_cookie_time == 0 || 1457 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1458 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1459 else { 1460 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1461 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1462 (const uint8_t *)wgmi, 1463 offsetof(struct wg_msg_init, wgmi_mac2), 1464 NULL, 0); 1465 } 1466 1467 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1468 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1469 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1470 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1471 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1472 } 1473 1474 static void __noinline 1475 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1476 const struct sockaddr *src) 1477 { 1478 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1479 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1480 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1481 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1482 struct wg_peer *wgp; 1483 struct wg_session *wgs; 1484 int error, ret; 1485 struct psref psref_peer; 1486 uint8_t mac1[WG_MAC_LEN]; 1487 1488 WG_TRACE("init msg received"); 1489 1490 wg_algo_mac_mac1(mac1, sizeof(mac1), 1491 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1492 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1493 1494 /* 1495 * [W] 5.3: Denial of Service Mitigation & Cookies 1496 * "the responder, ..., must always reject messages with an invalid 1497 * msg.mac1" 1498 */ 1499 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1500 WG_DLOG("mac1 is invalid\n"); 1501 return; 1502 } 1503 1504 /* 1505 * [W] 5.4.2: First Message: Initiator to Responder 1506 * "When the responder receives this message, it does the same 1507 * operations so that its final state variables are identical, 1508 * replacing the operands of the DH function to produce equivalent 1509 * values." 1510 * Note that the following comments of operations are just copies of 1511 * the initiator's ones. 1512 */ 1513 1514 /* Ci := HASH(CONSTRUCTION) */ 1515 /* Hi := HASH(Ci || IDENTIFIER) */ 1516 wg_init_key_and_hash(ckey, hash); 1517 /* Hi := HASH(Hi || Sr^pub) */ 1518 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1519 1520 /* [N] 2.2: "e" */ 1521 /* Ci := KDF1(Ci, Ei^pub) */ 1522 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1523 sizeof(wgmi->wgmi_ephemeral)); 1524 /* Hi := HASH(Hi || msg.ephemeral) */ 1525 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1526 1527 WG_DUMP_HASH("ckey", ckey); 1528 1529 /* [N] 2.2: "es" */ 1530 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1531 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1532 1533 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1534 1535 /* [N] 2.2: "s" */ 1536 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1537 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1538 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1539 if (error != 0) { 1540 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1541 "%s: wg_algo_aead_dec for secret key failed\n", 1542 if_name(&wg->wg_if)); 1543 return; 1544 } 1545 /* Hi := HASH(Hi || msg.static) */ 1546 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1547 1548 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1549 if (wgp == NULL) { 1550 WG_DLOG("peer not found\n"); 1551 return; 1552 } 1553 1554 /* 1555 * Lock the peer to serialize access to cookie state. 1556 * 1557 * XXX Can we safely avoid holding the lock across DH? Take it 1558 * just to verify mac2 and then unlock/DH/lock? 1559 */ 1560 mutex_enter(wgp->wgp_lock); 1561 1562 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1563 WG_TRACE("under load"); 1564 /* 1565 * [W] 5.3: Denial of Service Mitigation & Cookies 1566 * "the responder, ..., and when under load may reject messages 1567 * with an invalid msg.mac2. If the responder receives a 1568 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1569 * and is under load, it may respond with a cookie reply 1570 * message" 1571 */ 1572 uint8_t zero[WG_MAC_LEN] = {0}; 1573 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1574 WG_TRACE("sending a cookie message: no cookie included"); 1575 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1576 wgmi->wgmi_mac1, src); 1577 goto out; 1578 } 1579 if (!wgp->wgp_last_sent_cookie_valid) { 1580 WG_TRACE("sending a cookie message: no cookie sent ever"); 1581 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1582 wgmi->wgmi_mac1, src); 1583 goto out; 1584 } 1585 uint8_t mac2[WG_MAC_LEN]; 1586 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1587 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1588 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1589 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1590 WG_DLOG("mac2 is invalid\n"); 1591 goto out; 1592 } 1593 WG_TRACE("under load, but continue to sending"); 1594 } 1595 1596 /* [N] 2.2: "ss" */ 1597 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1598 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1599 1600 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1601 wg_timestamp_t timestamp; 1602 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1603 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1604 hash, sizeof(hash)); 1605 if (error != 0) { 1606 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1607 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1608 if_name(&wg->wg_if), wgp->wgp_name); 1609 goto out; 1610 } 1611 /* Hi := HASH(Hi || msg.timestamp) */ 1612 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1613 1614 /* 1615 * [W] 5.1 "The responder keeps track of the greatest timestamp 1616 * received per peer and discards packets containing 1617 * timestamps less than or equal to it." 1618 */ 1619 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1620 sizeof(timestamp)); 1621 if (ret <= 0) { 1622 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1623 "%s: peer %s: invalid init msg: timestamp is old\n", 1624 if_name(&wg->wg_if), wgp->wgp_name); 1625 goto out; 1626 } 1627 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1628 1629 /* 1630 * Message is good -- we're committing to handle it now, unless 1631 * we were already initiating a session. 1632 */ 1633 wgs = wgp->wgp_session_unstable; 1634 switch (wgs->wgs_state) { 1635 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1636 break; 1637 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1638 /* XXX Who wins if both sides send INIT? */ 1639 WG_TRACE("Session already initializing, ignoring the message"); 1640 goto out; 1641 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1642 WG_TRACE("Session already initializing, destroying old states"); 1643 /* 1644 * XXX Avoid this -- just resend our response -- if the 1645 * INIT message is identical to the previous one. 1646 */ 1647 wg_put_session_index(wg, wgs); 1648 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1649 wgs->wgs_state); 1650 break; 1651 case WGS_STATE_ESTABLISHED: /* can't happen */ 1652 panic("unstable session can't be established"); 1653 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1654 WG_TRACE("Session destroying, but force to clear"); 1655 wg_put_session_index(wg, wgs); 1656 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1657 wgs->wgs_state); 1658 break; 1659 default: 1660 panic("invalid session state: %d", wgs->wgs_state); 1661 } 1662 1663 /* 1664 * Assign a fresh session index. 1665 */ 1666 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1667 wgs->wgs_state); 1668 wg_get_session_index(wg, wgs); 1669 1670 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1671 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1672 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1673 sizeof(wgmi->wgmi_ephemeral)); 1674 1675 wg_update_endpoint_if_necessary(wgp, src); 1676 1677 /* 1678 * Count the time of the INIT message as the time of 1679 * establishment -- this is used to decide when to erase keys, 1680 * and we want to start counting as soon as we have generated 1681 * keys. 1682 * 1683 * No need for atomic store because the session can't be used 1684 * in the rx or tx paths yet -- not until we transition to 1685 * INTI_PASSIVE. 1686 */ 1687 wgs->wgs_time_established = time_uptime32; 1688 wg_schedule_session_dtor_timer(wgp); 1689 1690 /* 1691 * Respond to the initiator with our ephemeral public key. 1692 */ 1693 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1694 1695 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]:" 1696 " calculate keys as responder\n", 1697 wgs->wgs_local_index, wgs->wgs_remote_index); 1698 wg_calculate_keys(wgs, false); 1699 wg_clear_states(wgs); 1700 1701 /* 1702 * Session is ready to receive data now that we have received 1703 * the peer initiator's ephemeral key pair, generated our 1704 * responder's ephemeral key pair, and derived a session key. 1705 * 1706 * Transition from UNKNOWN to INIT_PASSIVE to publish it to the 1707 * data rx path, wg_handle_msg_data, where the 1708 * atomic_load_acquire matching this atomic_store_release 1709 * happens. 1710 * 1711 * (Session is not, however, ready to send data until the peer 1712 * has acknowledged our response by sending its first data 1713 * packet. So don't swap the sessions yet.) 1714 */ 1715 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_INIT_PASSIVE\n", 1716 wgs->wgs_local_index, wgs->wgs_remote_index); 1717 atomic_store_release(&wgs->wgs_state, WGS_STATE_INIT_PASSIVE); 1718 WG_TRACE("WGS_STATE_INIT_PASSIVE"); 1719 1720 out: 1721 mutex_exit(wgp->wgp_lock); 1722 wg_put_peer(wgp, &psref_peer); 1723 } 1724 1725 static struct socket * 1726 wg_get_so_by_af(struct wg_softc *wg, const int af) 1727 { 1728 1729 switch (af) { 1730 #ifdef INET 1731 case AF_INET: 1732 return wg->wg_so4; 1733 #endif 1734 #ifdef INET6 1735 case AF_INET6: 1736 return wg->wg_so6; 1737 #endif 1738 default: 1739 panic("wg: no such af: %d", af); 1740 } 1741 } 1742 1743 static struct socket * 1744 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1745 { 1746 1747 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1748 } 1749 1750 static struct wg_sockaddr * 1751 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1752 { 1753 struct wg_sockaddr *wgsa; 1754 int s; 1755 1756 s = pserialize_read_enter(); 1757 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1758 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1759 pserialize_read_exit(s); 1760 1761 return wgsa; 1762 } 1763 1764 static void 1765 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1766 { 1767 1768 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1769 } 1770 1771 static int 1772 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1773 { 1774 int error; 1775 struct socket *so; 1776 struct psref psref; 1777 struct wg_sockaddr *wgsa; 1778 1779 wgsa = wg_get_endpoint_sa(wgp, &psref); 1780 so = wg_get_so_by_peer(wgp, wgsa); 1781 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1782 wg_put_sa(wgp, wgsa, &psref); 1783 1784 return error; 1785 } 1786 1787 static int 1788 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1789 { 1790 int error; 1791 struct mbuf *m; 1792 struct wg_msg_init *wgmi; 1793 struct wg_session *wgs; 1794 1795 KASSERT(mutex_owned(wgp->wgp_lock)); 1796 1797 wgs = wgp->wgp_session_unstable; 1798 /* XXX pull dispatch out into wg_task_send_init_message */ 1799 switch (wgs->wgs_state) { 1800 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1801 break; 1802 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1803 WG_TRACE("Session already initializing, skip starting new one"); 1804 return EBUSY; 1805 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1806 WG_TRACE("Session already initializing, waiting for peer"); 1807 return EBUSY; 1808 case WGS_STATE_ESTABLISHED: /* can't happen */ 1809 panic("unstable session can't be established"); 1810 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1811 WG_TRACE("Session destroying"); 1812 wg_put_session_index(wg, wgs); 1813 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1814 wgs->wgs_state); 1815 break; 1816 } 1817 1818 /* 1819 * Assign a fresh session index. 1820 */ 1821 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1822 wgs->wgs_state); 1823 wg_get_session_index(wg, wgs); 1824 1825 /* 1826 * We have initiated a session. Transition to INIT_ACTIVE. 1827 * This doesn't publish it for use in the data rx path, 1828 * wg_handle_msg_data, or in the data tx path, wg_output -- we 1829 * have to wait for the peer to respond with their ephemeral 1830 * public key before we can derive a session key for tx/rx. 1831 * Hence only atomic_store_relaxed. 1832 */ 1833 WG_DLOG("session[L=%"PRIx32" R=(unknown)] -> WGS_STATE_INIT_ACTIVE\n", 1834 wgs->wgs_local_index); 1835 atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_INIT_ACTIVE); 1836 1837 m = m_gethdr(M_WAIT, MT_DATA); 1838 if (sizeof(*wgmi) > MHLEN) { 1839 m_clget(m, M_WAIT); 1840 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1841 } 1842 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1843 wgmi = mtod(m, struct wg_msg_init *); 1844 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1845 1846 error = wg->wg_ops->send_hs_msg(wgp, m); 1847 if (error == 0) { 1848 WG_TRACE("init msg sent"); 1849 1850 if (wgp->wgp_handshake_start_time == 0) 1851 wgp->wgp_handshake_start_time = time_uptime; 1852 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1853 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1854 } else { 1855 wg_put_session_index(wg, wgs); 1856 /* Initiation failed; toss packet waiting for it if any. */ 1857 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 1858 m_freem(m); 1859 } 1860 1861 return error; 1862 } 1863 1864 static void 1865 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1866 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1867 const struct wg_msg_init *wgmi) 1868 { 1869 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1870 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1871 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1872 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1873 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1874 1875 KASSERT(mutex_owned(wgp->wgp_lock)); 1876 KASSERT(wgs == wgp->wgp_session_unstable); 1877 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 1878 wgs->wgs_state); 1879 1880 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1881 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1882 1883 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1884 wgmr->wgmr_sender = wgs->wgs_local_index; 1885 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1886 1887 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1888 1889 /* [N] 2.2: "e" */ 1890 /* Er^priv, Er^pub := DH-GENERATE() */ 1891 wg_algo_generate_keypair(pubkey, privkey); 1892 /* Cr := KDF1(Cr, Er^pub) */ 1893 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1894 /* msg.ephemeral := Er^pub */ 1895 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1896 /* Hr := HASH(Hr || msg.ephemeral) */ 1897 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1898 1899 WG_DUMP_HASH("ckey", ckey); 1900 WG_DUMP_HASH("hash", hash); 1901 1902 /* [N] 2.2: "ee" */ 1903 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1904 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1905 1906 /* [N] 2.2: "se" */ 1907 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1908 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1909 1910 /* [N] 9.2: "psk" */ 1911 { 1912 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1913 /* Cr, r, k := KDF3(Cr, Q) */ 1914 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1915 sizeof(wgp->wgp_psk)); 1916 /* Hr := HASH(Hr || r) */ 1917 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1918 } 1919 1920 /* msg.empty := AEAD(k, 0, e, Hr) */ 1921 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1922 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1923 /* Hr := HASH(Hr || msg.empty) */ 1924 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1925 1926 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1927 1928 /* [W] 5.4.4: Cookie MACs */ 1929 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1930 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1931 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1932 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1933 /* Need mac1 to decrypt a cookie from a cookie message */ 1934 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1935 sizeof(wgp->wgp_last_sent_mac1)); 1936 wgp->wgp_last_sent_mac1_valid = true; 1937 1938 if (wgp->wgp_latest_cookie_time == 0 || 1939 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1940 /* msg.mac2 := 0^16 */ 1941 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1942 else { 1943 /* msg.mac2 := MAC(Lm, msg_b) */ 1944 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1945 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1946 (const uint8_t *)wgmr, 1947 offsetof(struct wg_msg_resp, wgmr_mac2), 1948 NULL, 0); 1949 } 1950 1951 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1952 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1953 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1954 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1955 wgs->wgs_remote_index = wgmi->wgmi_sender; 1956 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1957 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1958 } 1959 1960 static void 1961 wg_swap_sessions(struct wg_peer *wgp) 1962 { 1963 struct wg_session *wgs, *wgs_prev; 1964 1965 KASSERT(mutex_owned(wgp->wgp_lock)); 1966 1967 wgs = wgp->wgp_session_unstable; 1968 KASSERTMSG(wgs->wgs_state == WGS_STATE_ESTABLISHED, "state=%d", 1969 wgs->wgs_state); 1970 1971 wgs_prev = wgp->wgp_session_stable; 1972 KASSERTMSG((wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1973 wgs_prev->wgs_state == WGS_STATE_UNKNOWN), 1974 "state=%d", wgs_prev->wgs_state); 1975 atomic_store_release(&wgp->wgp_session_stable, wgs); 1976 wgp->wgp_session_unstable = wgs_prev; 1977 } 1978 1979 static void __noinline 1980 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1981 const struct sockaddr *src) 1982 { 1983 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1984 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1985 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1986 struct wg_peer *wgp; 1987 struct wg_session *wgs; 1988 struct psref psref; 1989 int error; 1990 uint8_t mac1[WG_MAC_LEN]; 1991 struct wg_session *wgs_prev; 1992 struct mbuf *m; 1993 1994 wg_algo_mac_mac1(mac1, sizeof(mac1), 1995 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1996 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1997 1998 /* 1999 * [W] 5.3: Denial of Service Mitigation & Cookies 2000 * "the responder, ..., must always reject messages with an invalid 2001 * msg.mac1" 2002 */ 2003 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 2004 WG_DLOG("mac1 is invalid\n"); 2005 return; 2006 } 2007 2008 WG_TRACE("resp msg received"); 2009 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 2010 if (wgs == NULL) { 2011 WG_TRACE("No session found"); 2012 return; 2013 } 2014 2015 wgp = wgs->wgs_peer; 2016 2017 mutex_enter(wgp->wgp_lock); 2018 2019 /* If we weren't waiting for a handshake response, drop it. */ 2020 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 2021 WG_TRACE("peer sent spurious handshake response, ignoring"); 2022 goto out; 2023 } 2024 2025 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 2026 WG_TRACE("under load"); 2027 /* 2028 * [W] 5.3: Denial of Service Mitigation & Cookies 2029 * "the responder, ..., and when under load may reject messages 2030 * with an invalid msg.mac2. If the responder receives a 2031 * message with a valid msg.mac1 yet with an invalid msg.mac2, 2032 * and is under load, it may respond with a cookie reply 2033 * message" 2034 */ 2035 uint8_t zero[WG_MAC_LEN] = {0}; 2036 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 2037 WG_TRACE("sending a cookie message: no cookie included"); 2038 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 2039 wgmr->wgmr_mac1, src); 2040 goto out; 2041 } 2042 if (!wgp->wgp_last_sent_cookie_valid) { 2043 WG_TRACE("sending a cookie message: no cookie sent ever"); 2044 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 2045 wgmr->wgmr_mac1, src); 2046 goto out; 2047 } 2048 uint8_t mac2[WG_MAC_LEN]; 2049 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 2050 WG_COOKIE_LEN, (const uint8_t *)wgmr, 2051 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 2052 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 2053 WG_DLOG("mac2 is invalid\n"); 2054 goto out; 2055 } 2056 WG_TRACE("under load, but continue to sending"); 2057 } 2058 2059 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 2060 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 2061 2062 /* 2063 * [W] 5.4.3 Second Message: Responder to Initiator 2064 * "When the initiator receives this message, it does the same 2065 * operations so that its final state variables are identical, 2066 * replacing the operands of the DH function to produce equivalent 2067 * values." 2068 * Note that the following comments of operations are just copies of 2069 * the initiator's ones. 2070 */ 2071 2072 /* [N] 2.2: "e" */ 2073 /* Cr := KDF1(Cr, Er^pub) */ 2074 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 2075 sizeof(wgmr->wgmr_ephemeral)); 2076 /* Hr := HASH(Hr || msg.ephemeral) */ 2077 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 2078 2079 WG_DUMP_HASH("ckey", ckey); 2080 WG_DUMP_HASH("hash", hash); 2081 2082 /* [N] 2.2: "ee" */ 2083 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 2084 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 2085 wgmr->wgmr_ephemeral); 2086 2087 /* [N] 2.2: "se" */ 2088 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 2089 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 2090 2091 /* [N] 9.2: "psk" */ 2092 { 2093 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 2094 /* Cr, r, k := KDF3(Cr, Q) */ 2095 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 2096 sizeof(wgp->wgp_psk)); 2097 /* Hr := HASH(Hr || r) */ 2098 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 2099 } 2100 2101 { 2102 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 2103 /* msg.empty := AEAD(k, 0, e, Hr) */ 2104 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 2105 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 2106 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 2107 if (error != 0) { 2108 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2109 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 2110 if_name(&wg->wg_if), wgp->wgp_name); 2111 goto out; 2112 } 2113 /* Hr := HASH(Hr || msg.empty) */ 2114 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 2115 } 2116 2117 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 2118 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 2119 wgs->wgs_remote_index = wgmr->wgmr_sender; 2120 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 2121 2122 KASSERTMSG(wgs->wgs_state == WGS_STATE_INIT_ACTIVE, "state=%d", 2123 wgs->wgs_state); 2124 wgs->wgs_time_established = time_uptime32; 2125 wg_schedule_session_dtor_timer(wgp); 2126 wgs->wgs_time_last_data_sent = 0; 2127 wgs->wgs_is_initiator = true; 2128 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]:" 2129 " calculate keys as initiator\n", 2130 wgs->wgs_local_index, wgs->wgs_remote_index); 2131 wg_calculate_keys(wgs, true); 2132 wg_clear_states(wgs); 2133 2134 /* 2135 * Session is ready to receive data now that we have received 2136 * the responder's response. 2137 * 2138 * Transition from INIT_ACTIVE to ESTABLISHED to publish it to 2139 * the data rx path, wg_handle_msg_data. 2140 */ 2141 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32" -> WGS_STATE_ESTABLISHED\n", 2142 wgs->wgs_local_index, wgs->wgs_remote_index); 2143 atomic_store_release(&wgs->wgs_state, WGS_STATE_ESTABLISHED); 2144 WG_TRACE("WGS_STATE_ESTABLISHED"); 2145 2146 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 2147 2148 /* 2149 * Session is ready to send data now that we have received the 2150 * responder's response. 2151 * 2152 * Swap the sessions to publish the new one as the stable 2153 * session for the data tx path, wg_output. 2154 */ 2155 wg_swap_sessions(wgp); 2156 KASSERT(wgs == wgp->wgp_session_stable); 2157 wgs_prev = wgp->wgp_session_unstable; 2158 getnanotime(&wgp->wgp_last_handshake_time); 2159 wgp->wgp_handshake_start_time = 0; 2160 wgp->wgp_last_sent_mac1_valid = false; 2161 wgp->wgp_last_sent_cookie_valid = false; 2162 2163 wg_update_endpoint_if_necessary(wgp, src); 2164 2165 /* 2166 * If we had a data packet queued up, send it; otherwise send a 2167 * keepalive message -- either way we have to send something 2168 * immediately or else the responder will never answer. 2169 */ 2170 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2171 kpreempt_disable(); 2172 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2173 M_SETCTX(m, wgp); 2174 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2175 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2176 if_name(&wg->wg_if)); 2177 m_freem(m); 2178 } 2179 kpreempt_enable(); 2180 } else { 2181 wg_send_keepalive_msg(wgp, wgs); 2182 } 2183 2184 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2185 /* 2186 * Transition ESTABLISHED->DESTROYING. The session 2187 * will remain usable for the data rx path to process 2188 * packets still in flight to us, but we won't use it 2189 * for data tx. 2190 */ 2191 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]" 2192 " -> WGS_STATE_DESTROYING\n", 2193 wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index); 2194 atomic_store_relaxed(&wgs_prev->wgs_state, 2195 WGS_STATE_DESTROYING); 2196 } else { 2197 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2198 "state=%d", wgs_prev->wgs_state); 2199 } 2200 2201 out: 2202 mutex_exit(wgp->wgp_lock); 2203 wg_put_session(wgs, &psref); 2204 } 2205 2206 static int 2207 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2208 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2209 { 2210 int error; 2211 struct mbuf *m; 2212 struct wg_msg_resp *wgmr; 2213 2214 KASSERT(mutex_owned(wgp->wgp_lock)); 2215 KASSERT(wgs == wgp->wgp_session_unstable); 2216 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 2217 wgs->wgs_state); 2218 2219 m = m_gethdr(M_WAIT, MT_DATA); 2220 if (sizeof(*wgmr) > MHLEN) { 2221 m_clget(m, M_WAIT); 2222 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2223 } 2224 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2225 wgmr = mtod(m, struct wg_msg_resp *); 2226 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2227 2228 error = wg->wg_ops->send_hs_msg(wgp, m); 2229 if (error == 0) 2230 WG_TRACE("resp msg sent"); 2231 return error; 2232 } 2233 2234 static struct wg_peer * 2235 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2236 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2237 { 2238 struct wg_peer *wgp; 2239 2240 int s = pserialize_read_enter(); 2241 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2242 if (wgp != NULL) 2243 wg_get_peer(wgp, psref); 2244 pserialize_read_exit(s); 2245 2246 return wgp; 2247 } 2248 2249 static void 2250 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2251 struct wg_msg_cookie *wgmc, const uint32_t sender, 2252 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2253 { 2254 uint8_t cookie[WG_COOKIE_LEN]; 2255 uint8_t key[WG_HASH_LEN]; 2256 uint8_t addr[sizeof(struct in6_addr)]; 2257 size_t addrlen; 2258 uint16_t uh_sport; /* be */ 2259 2260 KASSERT(mutex_owned(wgp->wgp_lock)); 2261 2262 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2263 wgmc->wgmc_receiver = sender; 2264 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2265 2266 /* 2267 * [W] 5.4.7: Under Load: Cookie Reply Message 2268 * "The secret variable, Rm, changes every two minutes to a 2269 * random value" 2270 */ 2271 if ((time_uptime - wgp->wgp_last_cookiesecret_time) > 2272 WG_COOKIESECRET_TIME) { 2273 cprng_strong(kern_cprng, wgp->wgp_cookiesecret, 2274 sizeof(wgp->wgp_cookiesecret), 0); 2275 wgp->wgp_last_cookiesecret_time = time_uptime; 2276 } 2277 2278 switch (src->sa_family) { 2279 case AF_INET: { 2280 const struct sockaddr_in *sin = satocsin(src); 2281 addrlen = sizeof(sin->sin_addr); 2282 memcpy(addr, &sin->sin_addr, addrlen); 2283 uh_sport = sin->sin_port; 2284 break; 2285 } 2286 #ifdef INET6 2287 case AF_INET6: { 2288 const struct sockaddr_in6 *sin6 = satocsin6(src); 2289 addrlen = sizeof(sin6->sin6_addr); 2290 memcpy(addr, &sin6->sin6_addr, addrlen); 2291 uh_sport = sin6->sin6_port; 2292 break; 2293 } 2294 #endif 2295 default: 2296 panic("invalid af=%d", src->sa_family); 2297 } 2298 2299 wg_algo_mac(cookie, sizeof(cookie), 2300 wgp->wgp_cookiesecret, sizeof(wgp->wgp_cookiesecret), 2301 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2302 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2303 sizeof(wg->wg_pubkey)); 2304 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2305 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2306 2307 /* Need to store to calculate mac2 */ 2308 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2309 wgp->wgp_last_sent_cookie_valid = true; 2310 } 2311 2312 static int 2313 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2314 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2315 const struct sockaddr *src) 2316 { 2317 int error; 2318 struct mbuf *m; 2319 struct wg_msg_cookie *wgmc; 2320 2321 KASSERT(mutex_owned(wgp->wgp_lock)); 2322 2323 m = m_gethdr(M_WAIT, MT_DATA); 2324 if (sizeof(*wgmc) > MHLEN) { 2325 m_clget(m, M_WAIT); 2326 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2327 } 2328 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2329 wgmc = mtod(m, struct wg_msg_cookie *); 2330 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2331 2332 error = wg->wg_ops->send_hs_msg(wgp, m); 2333 if (error == 0) 2334 WG_TRACE("cookie msg sent"); 2335 return error; 2336 } 2337 2338 static bool 2339 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2340 { 2341 #ifdef WG_DEBUG_PARAMS 2342 if (wg_force_underload) 2343 return true; 2344 #endif 2345 2346 /* 2347 * XXX we don't have a means of a load estimation. The purpose of 2348 * the mechanism is a DoS mitigation, so we consider frequent handshake 2349 * messages as (a kind of) load; if a message of the same type comes 2350 * to a peer within 1 second, we consider we are under load. 2351 */ 2352 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2353 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2354 return (time_uptime - last) == 0; 2355 } 2356 2357 static void 2358 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2359 { 2360 2361 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2362 2363 /* 2364 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2365 */ 2366 if (initiator) { 2367 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2368 wgs->wgs_chaining_key, NULL, 0); 2369 } else { 2370 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2371 wgs->wgs_chaining_key, NULL, 0); 2372 } 2373 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2374 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2375 } 2376 2377 static uint64_t 2378 wg_session_get_send_counter(struct wg_session *wgs) 2379 { 2380 #ifdef __HAVE_ATOMIC64_LOADSTORE 2381 return atomic_load_relaxed(&wgs->wgs_send_counter); 2382 #else 2383 uint64_t send_counter; 2384 2385 mutex_enter(&wgs->wgs_send_counter_lock); 2386 send_counter = wgs->wgs_send_counter; 2387 mutex_exit(&wgs->wgs_send_counter_lock); 2388 2389 return send_counter; 2390 #endif 2391 } 2392 2393 static uint64_t 2394 wg_session_inc_send_counter(struct wg_session *wgs) 2395 { 2396 #ifdef __HAVE_ATOMIC64_LOADSTORE 2397 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2398 #else 2399 uint64_t send_counter; 2400 2401 mutex_enter(&wgs->wgs_send_counter_lock); 2402 send_counter = wgs->wgs_send_counter++; 2403 mutex_exit(&wgs->wgs_send_counter_lock); 2404 2405 return send_counter; 2406 #endif 2407 } 2408 2409 static void 2410 wg_clear_states(struct wg_session *wgs) 2411 { 2412 2413 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2414 2415 wgs->wgs_send_counter = 0; 2416 sliwin_reset(&wgs->wgs_recvwin->window); 2417 2418 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2419 wgs_clear(handshake_hash); 2420 wgs_clear(chaining_key); 2421 wgs_clear(ephemeral_key_pub); 2422 wgs_clear(ephemeral_key_priv); 2423 wgs_clear(ephemeral_key_peer); 2424 #undef wgs_clear 2425 } 2426 2427 static struct wg_session * 2428 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2429 struct psref *psref) 2430 { 2431 struct wg_session *wgs; 2432 2433 int s = pserialize_read_enter(); 2434 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2435 if (wgs != NULL) { 2436 uint32_t oindex __diagused = 2437 atomic_load_relaxed(&wgs->wgs_local_index); 2438 KASSERTMSG(index == oindex, 2439 "index=%"PRIx32" wgs->wgs_local_index=%"PRIx32, 2440 index, oindex); 2441 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2442 } 2443 pserialize_read_exit(s); 2444 2445 return wgs; 2446 } 2447 2448 static void 2449 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2450 { 2451 struct mbuf *m; 2452 2453 /* 2454 * [W] 6.5 Passive Keepalive 2455 * "A keepalive message is simply a transport data message with 2456 * a zero-length encapsulated encrypted inner-packet." 2457 */ 2458 WG_TRACE(""); 2459 m = m_gethdr(M_WAIT, MT_DATA); 2460 wg_send_data_msg(wgp, wgs, m); 2461 } 2462 2463 static bool 2464 wg_need_to_send_init_message(struct wg_session *wgs) 2465 { 2466 /* 2467 * [W] 6.2 Transport Message Limits 2468 * "if a peer is the initiator of a current secure session, 2469 * WireGuard will send a handshake initiation message to begin 2470 * a new secure session ... if after receiving a transport data 2471 * message, the current secure session is (REJECT-AFTER-TIME 2472 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2473 * not yet acted upon this event." 2474 */ 2475 return wgs->wgs_is_initiator && 2476 atomic_load_relaxed(&wgs->wgs_time_last_data_sent) == 0 && 2477 ((time_uptime32 - 2478 atomic_load_relaxed(&wgs->wgs_time_established)) >= 2479 (wg_reject_after_time - wg_keepalive_timeout - 2480 wg_rekey_timeout)); 2481 } 2482 2483 static void 2484 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2485 { 2486 2487 mutex_enter(wgp->wgp_intr_lock); 2488 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2489 if (wgp->wgp_tasks == 0) 2490 /* 2491 * XXX If the current CPU is already loaded -- e.g., if 2492 * there's already a bunch of handshakes queued up -- 2493 * consider tossing this over to another CPU to 2494 * distribute the load. 2495 */ 2496 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2497 wgp->wgp_tasks |= task; 2498 mutex_exit(wgp->wgp_intr_lock); 2499 } 2500 2501 static void 2502 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2503 { 2504 struct wg_sockaddr *wgsa_prev; 2505 2506 WG_TRACE("Changing endpoint"); 2507 2508 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2509 wgsa_prev = wgp->wgp_endpoint; 2510 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2511 wgp->wgp_endpoint0 = wgsa_prev; 2512 atomic_store_release(&wgp->wgp_endpoint_available, true); 2513 2514 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2515 } 2516 2517 static bool 2518 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2519 { 2520 uint16_t packet_len; 2521 const struct ip *ip; 2522 2523 if (__predict_false(decrypted_len < sizeof(*ip))) { 2524 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2525 sizeof(*ip)); 2526 return false; 2527 } 2528 2529 ip = (const struct ip *)packet; 2530 if (ip->ip_v == 4) 2531 *af = AF_INET; 2532 else if (ip->ip_v == 6) 2533 *af = AF_INET6; 2534 else { 2535 WG_DLOG("ip_v=%d\n", ip->ip_v); 2536 return false; 2537 } 2538 2539 WG_DLOG("af=%d\n", *af); 2540 2541 switch (*af) { 2542 #ifdef INET 2543 case AF_INET: 2544 packet_len = ntohs(ip->ip_len); 2545 break; 2546 #endif 2547 #ifdef INET6 2548 case AF_INET6: { 2549 const struct ip6_hdr *ip6; 2550 2551 if (__predict_false(decrypted_len < sizeof(*ip6))) { 2552 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2553 sizeof(*ip6)); 2554 return false; 2555 } 2556 2557 ip6 = (const struct ip6_hdr *)packet; 2558 packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); 2559 break; 2560 } 2561 #endif 2562 default: 2563 return false; 2564 } 2565 2566 if (packet_len > decrypted_len) { 2567 WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len, 2568 decrypted_len); 2569 return false; 2570 } 2571 2572 return true; 2573 } 2574 2575 static bool 2576 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2577 int af, char *packet) 2578 { 2579 struct sockaddr_storage ss; 2580 struct sockaddr *sa; 2581 struct psref psref; 2582 struct wg_peer *wgp; 2583 bool ok; 2584 2585 /* 2586 * II CRYPTOKEY ROUTING 2587 * "it will only accept it if its source IP resolves in the 2588 * table to the public key used in the secure session for 2589 * decrypting it." 2590 */ 2591 2592 if (af == AF_INET) { 2593 const struct ip *ip = (const struct ip *)packet; 2594 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2595 sockaddr_in_init(sin, &ip->ip_src, 0); 2596 sa = sintosa(sin); 2597 #ifdef INET6 2598 } else { 2599 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2600 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2601 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2602 sa = sin6tosa(sin6); 2603 #endif 2604 } 2605 2606 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2607 ok = (wgp == wgp_expected); 2608 if (wgp != NULL) 2609 wg_put_peer(wgp, &psref); 2610 2611 return ok; 2612 } 2613 2614 static void 2615 wg_session_dtor_timer(void *arg) 2616 { 2617 struct wg_peer *wgp = arg; 2618 2619 WG_TRACE("enter"); 2620 2621 wg_schedule_session_dtor_timer(wgp); 2622 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2623 } 2624 2625 static void 2626 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2627 { 2628 2629 /* 2630 * If the periodic session destructor is already pending to 2631 * handle the previous session, that's fine -- leave it in 2632 * place; it will be scheduled again. 2633 */ 2634 if (callout_pending(&wgp->wgp_session_dtor_timer)) { 2635 WG_DLOG("session dtor already pending\n"); 2636 return; 2637 } 2638 2639 WG_DLOG("scheduling session dtor in %u secs\n", wg_reject_after_time); 2640 callout_schedule(&wgp->wgp_session_dtor_timer, 2641 wg_reject_after_time*hz); 2642 } 2643 2644 static bool 2645 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2646 { 2647 if (sa1->sa_family != sa2->sa_family) 2648 return false; 2649 2650 switch (sa1->sa_family) { 2651 #ifdef INET 2652 case AF_INET: 2653 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2654 #endif 2655 #ifdef INET6 2656 case AF_INET6: 2657 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2658 #endif 2659 default: 2660 return false; 2661 } 2662 } 2663 2664 static void 2665 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2666 const struct sockaddr *src) 2667 { 2668 struct wg_sockaddr *wgsa; 2669 struct psref psref; 2670 2671 wgsa = wg_get_endpoint_sa(wgp, &psref); 2672 2673 #ifdef WG_DEBUG_LOG 2674 char oldaddr[128], newaddr[128]; 2675 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2676 sockaddr_format(src, newaddr, sizeof(newaddr)); 2677 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2678 #endif 2679 2680 /* 2681 * III: "Since the packet has authenticated correctly, the source IP of 2682 * the outer UDP/IP packet is used to update the endpoint for peer..." 2683 */ 2684 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2685 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2686 /* XXX We can't change the endpoint twice in a short period */ 2687 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2688 wg_change_endpoint(wgp, src); 2689 } 2690 } 2691 2692 wg_put_sa(wgp, wgsa, &psref); 2693 } 2694 2695 static void __noinline 2696 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2697 const struct sockaddr *src) 2698 { 2699 struct wg_msg_data *wgmd; 2700 char *encrypted_buf = NULL, *decrypted_buf; 2701 size_t encrypted_len, decrypted_len; 2702 struct wg_session *wgs; 2703 struct wg_peer *wgp; 2704 int state; 2705 uint32_t age; 2706 size_t mlen; 2707 struct psref psref; 2708 int error, af; 2709 bool success, free_encrypted_buf = false, ok; 2710 struct mbuf *n; 2711 2712 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2713 wgmd = mtod(m, struct wg_msg_data *); 2714 2715 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2716 WG_TRACE("data"); 2717 2718 /* Find the putative session, or drop. */ 2719 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2720 if (wgs == NULL) { 2721 WG_TRACE("No session found"); 2722 m_freem(m); 2723 return; 2724 } 2725 2726 /* 2727 * We are only ready to handle data when in INIT_PASSIVE, 2728 * ESTABLISHED, or DESTROYING. All transitions out of that 2729 * state dissociate the session index and drain psrefs. 2730 * 2731 * atomic_load_acquire matches atomic_store_release in either 2732 * wg_handle_msg_init or wg_handle_msg_resp. (The transition 2733 * INIT_PASSIVE to ESTABLISHED in wg_task_establish_session 2734 * doesn't make a difference for this rx path.) 2735 */ 2736 state = atomic_load_acquire(&wgs->wgs_state); 2737 switch (state) { 2738 case WGS_STATE_UNKNOWN: 2739 case WGS_STATE_INIT_ACTIVE: 2740 WG_TRACE("not yet ready for data"); 2741 goto out; 2742 case WGS_STATE_INIT_PASSIVE: 2743 case WGS_STATE_ESTABLISHED: 2744 case WGS_STATE_DESTROYING: 2745 break; 2746 } 2747 2748 /* 2749 * Reject if the session is too old. 2750 */ 2751 age = time_uptime32 - atomic_load_relaxed(&wgs->wgs_time_established); 2752 if (__predict_false(age >= wg_reject_after_time)) { 2753 WG_DLOG("session %"PRIx32" too old, %"PRIu32" sec\n", 2754 wgmd->wgmd_receiver, age); 2755 goto out; 2756 } 2757 2758 /* 2759 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2760 * to update the endpoint if authentication succeeds. 2761 */ 2762 wgp = wgs->wgs_peer; 2763 2764 /* 2765 * Reject outrageously wrong sequence numbers before doing any 2766 * crypto work or taking any locks. 2767 */ 2768 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2769 le64toh(wgmd->wgmd_counter)); 2770 if (error) { 2771 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2772 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2773 if_name(&wg->wg_if), wgp->wgp_name, 2774 le64toh(wgmd->wgmd_counter)); 2775 goto out; 2776 } 2777 2778 /* Ensure the payload and authenticator are contiguous. */ 2779 mlen = m_length(m); 2780 encrypted_len = mlen - sizeof(*wgmd); 2781 if (encrypted_len < WG_AUTHTAG_LEN) { 2782 WG_DLOG("Short encrypted_len: %zu\n", encrypted_len); 2783 goto out; 2784 } 2785 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2786 if (success) { 2787 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2788 } else { 2789 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2790 if (encrypted_buf == NULL) { 2791 WG_DLOG("failed to allocate encrypted_buf\n"); 2792 goto out; 2793 } 2794 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2795 free_encrypted_buf = true; 2796 } 2797 /* m_ensure_contig may change m regardless of its result */ 2798 KASSERT(m->m_len >= sizeof(*wgmd)); 2799 wgmd = mtod(m, struct wg_msg_data *); 2800 2801 #ifdef WG_DEBUG_PACKET 2802 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2803 hexdump(printf, "incoming packet", encrypted_buf, 2804 encrypted_len); 2805 } 2806 #endif 2807 /* 2808 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2809 * a zero-length buffer (XXX). Drop if plaintext is longer 2810 * than MCLBYTES (XXX). 2811 */ 2812 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2813 if (decrypted_len > MCLBYTES) { 2814 /* FIXME handle larger data than MCLBYTES */ 2815 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2816 goto out; 2817 } 2818 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2819 if (n == NULL) { 2820 WG_DLOG("wg_get_mbuf failed\n"); 2821 goto out; 2822 } 2823 decrypted_buf = mtod(n, char *); 2824 2825 /* Decrypt and verify the packet. */ 2826 WG_DLOG("mlen=%zu, encrypted_len=%zu\n", mlen, encrypted_len); 2827 error = wg_algo_aead_dec(decrypted_buf, 2828 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2829 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2830 encrypted_len, NULL, 0); 2831 if (error != 0) { 2832 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2833 "%s: peer %s: failed to wg_algo_aead_dec\n", 2834 if_name(&wg->wg_if), wgp->wgp_name); 2835 m_freem(n); 2836 goto out; 2837 } 2838 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2839 2840 /* Packet is genuine. Reject it if a replay or just too old. */ 2841 mutex_enter(&wgs->wgs_recvwin->lock); 2842 error = sliwin_update(&wgs->wgs_recvwin->window, 2843 le64toh(wgmd->wgmd_counter)); 2844 mutex_exit(&wgs->wgs_recvwin->lock); 2845 if (error) { 2846 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2847 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2848 if_name(&wg->wg_if), wgp->wgp_name, 2849 le64toh(wgmd->wgmd_counter)); 2850 m_freem(n); 2851 goto out; 2852 } 2853 2854 #ifdef WG_DEBUG_PACKET 2855 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2856 hexdump(printf, "tkey_recv", wgs->wgs_tkey_recv, 2857 sizeof(wgs->wgs_tkey_recv)); 2858 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 2859 hexdump(printf, "decrypted_buf", decrypted_buf, 2860 decrypted_len); 2861 } 2862 #endif 2863 /* We're done with m now; free it and chuck the pointers. */ 2864 m_freem(m); 2865 m = NULL; 2866 wgmd = NULL; 2867 2868 /* 2869 * The packet is genuine. Update the peer's endpoint if the 2870 * source address changed. 2871 * 2872 * XXX How to prevent DoS by replaying genuine packets from the 2873 * wrong source address? 2874 */ 2875 wg_update_endpoint_if_necessary(wgp, src); 2876 2877 /* 2878 * Validate the encapsulated packet header and get the address 2879 * family, or drop. 2880 */ 2881 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2882 if (!ok) { 2883 m_freem(n); 2884 goto update_state; 2885 } 2886 2887 /* Submit it into our network stack if routable. */ 2888 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2889 if (ok) { 2890 wg->wg_ops->input(&wg->wg_if, n, af); 2891 } else { 2892 char addrstr[INET6_ADDRSTRLEN]; 2893 memset(addrstr, 0, sizeof(addrstr)); 2894 if (af == AF_INET) { 2895 const struct ip *ip = (const struct ip *)decrypted_buf; 2896 IN_PRINT(addrstr, &ip->ip_src); 2897 #ifdef INET6 2898 } else if (af == AF_INET6) { 2899 const struct ip6_hdr *ip6 = 2900 (const struct ip6_hdr *)decrypted_buf; 2901 IN6_PRINT(addrstr, &ip6->ip6_src); 2902 #endif 2903 } 2904 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2905 "%s: peer %s: invalid source address (%s)\n", 2906 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2907 m_freem(n); 2908 /* 2909 * The inner address is invalid however the session is valid 2910 * so continue the session processing below. 2911 */ 2912 } 2913 n = NULL; 2914 2915 update_state: 2916 /* Update the state machine if necessary. */ 2917 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2918 /* 2919 * We were waiting for the initiator to send their 2920 * first data transport message, and that has happened. 2921 * Schedule a task to establish this session. 2922 */ 2923 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2924 } else { 2925 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2926 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2927 } 2928 /* 2929 * [W] 6.5 Passive Keepalive 2930 * "If a peer has received a validly-authenticated transport 2931 * data message (section 5.4.6), but does not have any packets 2932 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2933 * a keepalive message." 2934 */ 2935 const uint32_t now = time_uptime32; 2936 const uint32_t time_last_data_sent = 2937 atomic_load_relaxed(&wgs->wgs_time_last_data_sent); 2938 WG_DLOG("time_uptime32=%"PRIu32 2939 " wgs_time_last_data_sent=%"PRIu32"\n", 2940 now, time_last_data_sent); 2941 if ((now - time_last_data_sent) >= wg_keepalive_timeout) { 2942 WG_TRACE("Schedule sending keepalive message"); 2943 /* 2944 * We can't send a keepalive message here to avoid 2945 * a deadlock; we already hold the solock of a socket 2946 * that is used to send the message. 2947 */ 2948 wg_schedule_peer_task(wgp, 2949 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2950 } 2951 } 2952 out: 2953 wg_put_session(wgs, &psref); 2954 m_freem(m); 2955 if (free_encrypted_buf) 2956 kmem_intr_free(encrypted_buf, encrypted_len); 2957 } 2958 2959 static void __noinline 2960 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2961 { 2962 struct wg_session *wgs; 2963 struct wg_peer *wgp; 2964 struct psref psref; 2965 int error; 2966 uint8_t key[WG_HASH_LEN]; 2967 uint8_t cookie[WG_COOKIE_LEN]; 2968 2969 WG_TRACE("cookie msg received"); 2970 2971 /* Find the putative session. */ 2972 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2973 if (wgs == NULL) { 2974 WG_TRACE("No session found"); 2975 return; 2976 } 2977 2978 /* Lock the peer so we can update the cookie state. */ 2979 wgp = wgs->wgs_peer; 2980 mutex_enter(wgp->wgp_lock); 2981 2982 if (!wgp->wgp_last_sent_mac1_valid) { 2983 WG_TRACE("No valid mac1 sent (or expired)"); 2984 goto out; 2985 } 2986 2987 /* 2988 * wgp_last_sent_mac1_valid is only set to true when we are 2989 * transitioning to INIT_ACTIVE or INIT_PASSIVE, and always 2990 * cleared on transition out of them. 2991 */ 2992 KASSERTMSG((wgs->wgs_state == WGS_STATE_INIT_ACTIVE || 2993 wgs->wgs_state == WGS_STATE_INIT_PASSIVE), 2994 "state=%d", wgs->wgs_state); 2995 2996 /* Decrypt the cookie and store it for later handshake retry. */ 2997 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2998 sizeof(wgp->wgp_pubkey)); 2999 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 3000 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 3001 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 3002 wgmc->wgmc_salt); 3003 if (error != 0) { 3004 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 3005 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 3006 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 3007 goto out; 3008 } 3009 /* 3010 * [W] 6.6: Interaction with Cookie Reply System 3011 * "it should simply store the decrypted cookie value from the cookie 3012 * reply message, and wait for the expiration of the REKEY-TIMEOUT 3013 * timer for retrying a handshake initiation message." 3014 */ 3015 wgp->wgp_latest_cookie_time = time_uptime; 3016 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 3017 out: 3018 mutex_exit(wgp->wgp_lock); 3019 wg_put_session(wgs, &psref); 3020 } 3021 3022 static struct mbuf * 3023 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 3024 { 3025 struct wg_msg wgm; 3026 size_t mbuflen; 3027 size_t msglen; 3028 3029 /* 3030 * Get the mbuf chain length. It is already guaranteed, by 3031 * wg_overudp_cb, to be large enough for a struct wg_msg. 3032 */ 3033 mbuflen = m_length(m); 3034 KASSERT(mbuflen >= sizeof(struct wg_msg)); 3035 3036 /* 3037 * Copy the message header (32-bit message type) out -- we'll 3038 * worry about contiguity and alignment later. 3039 */ 3040 m_copydata(m, 0, sizeof(wgm), &wgm); 3041 switch (le32toh(wgm.wgm_type)) { 3042 case WG_MSG_TYPE_INIT: 3043 msglen = sizeof(struct wg_msg_init); 3044 break; 3045 case WG_MSG_TYPE_RESP: 3046 msglen = sizeof(struct wg_msg_resp); 3047 break; 3048 case WG_MSG_TYPE_COOKIE: 3049 msglen = sizeof(struct wg_msg_cookie); 3050 break; 3051 case WG_MSG_TYPE_DATA: 3052 msglen = sizeof(struct wg_msg_data); 3053 break; 3054 default: 3055 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 3056 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 3057 le32toh(wgm.wgm_type)); 3058 goto error; 3059 } 3060 3061 /* Verify the mbuf chain is long enough for this type of message. */ 3062 if (__predict_false(mbuflen < msglen)) { 3063 WG_DLOG("Invalid msg size: mbuflen=%zu type=%u\n", mbuflen, 3064 le32toh(wgm.wgm_type)); 3065 goto error; 3066 } 3067 3068 /* Make the message header contiguous if necessary. */ 3069 if (__predict_false(m->m_len < msglen)) { 3070 m = m_pullup(m, msglen); 3071 if (m == NULL) 3072 return NULL; 3073 } 3074 3075 return m; 3076 3077 error: 3078 m_freem(m); 3079 return NULL; 3080 } 3081 3082 static void 3083 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 3084 const struct sockaddr *src) 3085 { 3086 struct wg_msg *wgm; 3087 3088 KASSERT(curlwp->l_pflag & LP_BOUND); 3089 3090 m = wg_validate_msg_header(wg, m); 3091 if (__predict_false(m == NULL)) 3092 return; 3093 3094 KASSERT(m->m_len >= sizeof(struct wg_msg)); 3095 wgm = mtod(m, struct wg_msg *); 3096 switch (le32toh(wgm->wgm_type)) { 3097 case WG_MSG_TYPE_INIT: 3098 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 3099 break; 3100 case WG_MSG_TYPE_RESP: 3101 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 3102 break; 3103 case WG_MSG_TYPE_COOKIE: 3104 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 3105 break; 3106 case WG_MSG_TYPE_DATA: 3107 wg_handle_msg_data(wg, m, src); 3108 /* wg_handle_msg_data frees m for us */ 3109 return; 3110 default: 3111 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 3112 } 3113 3114 m_freem(m); 3115 } 3116 3117 static void 3118 wg_receive_packets(struct wg_softc *wg, const int af) 3119 { 3120 3121 for (;;) { 3122 int error, flags; 3123 struct socket *so; 3124 struct mbuf *m = NULL; 3125 struct uio dummy_uio; 3126 struct mbuf *paddr = NULL; 3127 struct sockaddr *src; 3128 3129 so = wg_get_so_by_af(wg, af); 3130 flags = MSG_DONTWAIT; 3131 dummy_uio.uio_resid = 1000000000; 3132 3133 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 3134 &flags); 3135 if (error || m == NULL) { 3136 //if (error == EWOULDBLOCK) 3137 return; 3138 } 3139 3140 KASSERT(paddr != NULL); 3141 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 3142 src = mtod(paddr, struct sockaddr *); 3143 3144 wg_handle_packet(wg, m, src); 3145 } 3146 } 3147 3148 static void 3149 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 3150 { 3151 3152 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 3153 } 3154 3155 static void 3156 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 3157 { 3158 3159 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 3160 } 3161 3162 static void 3163 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 3164 { 3165 struct wg_session *wgs; 3166 3167 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 3168 3169 KASSERT(mutex_owned(wgp->wgp_lock)); 3170 3171 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 3172 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 3173 if_name(&wg->wg_if)); 3174 /* XXX should do something? */ 3175 return; 3176 } 3177 3178 /* 3179 * If we already have an established session, there's no need 3180 * to initiate a new one -- unless the rekey-after-time or 3181 * rekey-after-messages limits have passed. 3182 */ 3183 wgs = wgp->wgp_session_stable; 3184 if (wgs->wgs_state == WGS_STATE_ESTABLISHED && 3185 !atomic_swap_uint(&wgp->wgp_force_rekey, 0)) 3186 return; 3187 3188 /* 3189 * Ensure we're initiating a new session. If the unstable 3190 * session is already INIT_ACTIVE or INIT_PASSIVE, this does 3191 * nothing. 3192 */ 3193 wg_send_handshake_msg_init(wg, wgp); 3194 } 3195 3196 static void 3197 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 3198 { 3199 struct wg_session *wgs; 3200 3201 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 3202 3203 KASSERT(mutex_owned(wgp->wgp_lock)); 3204 KASSERT(wgp->wgp_handshake_start_time != 0); 3205 3206 wgs = wgp->wgp_session_unstable; 3207 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3208 return; 3209 3210 /* 3211 * XXX no real need to assign a new index here, but we do need 3212 * to transition to UNKNOWN temporarily 3213 */ 3214 wg_put_session_index(wg, wgs); 3215 3216 /* [W] 6.4 Handshake Initiation Retransmission */ 3217 if ((time_uptime - wgp->wgp_handshake_start_time) > 3218 wg_rekey_attempt_time) { 3219 /* Give up handshaking */ 3220 wgp->wgp_handshake_start_time = 0; 3221 WG_TRACE("give up"); 3222 3223 /* 3224 * If a new data packet comes, handshaking will be retried 3225 * and a new session would be established at that time, 3226 * however we don't want to send pending packets then. 3227 */ 3228 wg_purge_pending_packets(wgp); 3229 return; 3230 } 3231 3232 wg_task_send_init_message(wg, wgp); 3233 } 3234 3235 static void 3236 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3237 { 3238 struct wg_session *wgs, *wgs_prev; 3239 struct mbuf *m; 3240 3241 KASSERT(mutex_owned(wgp->wgp_lock)); 3242 3243 wgs = wgp->wgp_session_unstable; 3244 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3245 /* XXX Can this happen? */ 3246 return; 3247 3248 wgs->wgs_time_last_data_sent = 0; 3249 wgs->wgs_is_initiator = false; 3250 3251 /* 3252 * Session was already ready to receive data. Transition from 3253 * INIT_PASSIVE to ESTABLISHED just so we can swap the 3254 * sessions. 3255 * 3256 * atomic_store_relaxed because this doesn't affect the data rx 3257 * path, wg_handle_msg_data -- changing from INIT_PASSIVE to 3258 * ESTABLISHED makes no difference to the data rx path, and the 3259 * transition to INIT_PASSIVE with store-release already 3260 * published the state needed by the data rx path. 3261 */ 3262 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_ESTABLISHED\n", 3263 wgs->wgs_local_index, wgs->wgs_remote_index); 3264 atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_ESTABLISHED); 3265 WG_TRACE("WGS_STATE_ESTABLISHED"); 3266 3267 /* 3268 * Session is ready to send data too now that we have received 3269 * the peer initiator's first data packet. 3270 * 3271 * Swap the sessions to publish the new one as the stable 3272 * session for the data tx path, wg_output. 3273 */ 3274 wg_swap_sessions(wgp); 3275 KASSERT(wgs == wgp->wgp_session_stable); 3276 wgs_prev = wgp->wgp_session_unstable; 3277 getnanotime(&wgp->wgp_last_handshake_time); 3278 wgp->wgp_handshake_start_time = 0; 3279 wgp->wgp_last_sent_mac1_valid = false; 3280 wgp->wgp_last_sent_cookie_valid = false; 3281 3282 /* If we had a data packet queued up, send it. */ 3283 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3284 kpreempt_disable(); 3285 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3286 M_SETCTX(m, wgp); 3287 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3288 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3289 if_name(&wg->wg_if)); 3290 m_freem(m); 3291 } 3292 kpreempt_enable(); 3293 } 3294 3295 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3296 /* 3297 * Transition ESTABLISHED->DESTROYING. The session 3298 * will remain usable for the data rx path to process 3299 * packets still in flight to us, but we won't use it 3300 * for data tx. 3301 */ 3302 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]" 3303 " -> WGS_STATE_DESTROYING\n", 3304 wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index); 3305 atomic_store_relaxed(&wgs_prev->wgs_state, 3306 WGS_STATE_DESTROYING); 3307 } else { 3308 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3309 "state=%d", wgs_prev->wgs_state); 3310 WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]" 3311 " -> WGS_STATE_UNKNOWN\n", 3312 wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index); 3313 wgs_prev->wgs_local_index = 0; /* paranoia */ 3314 wgs_prev->wgs_remote_index = 0; /* paranoia */ 3315 wg_clear_states(wgs_prev); /* paranoia */ 3316 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3317 } 3318 } 3319 3320 static void 3321 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3322 { 3323 3324 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3325 3326 KASSERT(mutex_owned(wgp->wgp_lock)); 3327 3328 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3329 pserialize_perform(wgp->wgp_psz); 3330 mutex_exit(wgp->wgp_lock); 3331 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3332 wg_psref_class); 3333 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3334 wg_psref_class); 3335 mutex_enter(wgp->wgp_lock); 3336 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3337 } 3338 } 3339 3340 static void 3341 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3342 { 3343 struct wg_session *wgs; 3344 3345 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3346 3347 KASSERT(mutex_owned(wgp->wgp_lock)); 3348 3349 wgs = wgp->wgp_session_stable; 3350 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3351 return; 3352 3353 wg_send_keepalive_msg(wgp, wgs); 3354 } 3355 3356 static void 3357 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3358 { 3359 struct wg_session *wgs; 3360 uint32_t age; 3361 3362 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3363 3364 KASSERT(mutex_owned(wgp->wgp_lock)); 3365 3366 /* 3367 * If theres's any previous unstable session, i.e., one that 3368 * was ESTABLISHED and is now DESTROYING, older than 3369 * reject-after-time, destroy it. Upcoming sessions are still 3370 * in INIT_ACTIVE or INIT_PASSIVE -- we don't touch those here. 3371 * 3372 * No atomic for access to wgs_time_established because it is 3373 * only updated under wgp_lock. 3374 */ 3375 wgs = wgp->wgp_session_unstable; 3376 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 3377 if (wgs->wgs_state == WGS_STATE_DESTROYING && 3378 ((age = (time_uptime32 - wgs->wgs_time_established)) >= 3379 wg_reject_after_time)) { 3380 WG_DLOG("destroying past session %"PRIu32" sec old\n", age); 3381 wg_put_session_index(wg, wgs); 3382 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 3383 wgs->wgs_state); 3384 } 3385 3386 /* 3387 * If theres's any ESTABLISHED stable session older than 3388 * reject-after-time, destroy it. (The stable session can also 3389 * be in UNKNOWN state -- nothing to do in that case) 3390 */ 3391 wgs = wgp->wgp_session_stable; 3392 KASSERT(wgs->wgs_state != WGS_STATE_INIT_ACTIVE); 3393 KASSERT(wgs->wgs_state != WGS_STATE_INIT_PASSIVE); 3394 KASSERT(wgs->wgs_state != WGS_STATE_DESTROYING); 3395 if (wgs->wgs_state == WGS_STATE_ESTABLISHED && 3396 ((age = (time_uptime32 - wgs->wgs_time_established)) >= 3397 wg_reject_after_time)) { 3398 WG_DLOG("destroying current session %"PRIu32" sec old\n", age); 3399 atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_DESTROYING); 3400 wg_put_session_index(wg, wgs); 3401 KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d", 3402 wgs->wgs_state); 3403 } 3404 3405 /* 3406 * If there's no sessions left, no need to have the timer run 3407 * until the next time around -- halt it. 3408 * 3409 * It is only ever scheduled with wgp_lock held or in the 3410 * callout itself, and callout_halt prevents rescheudling 3411 * itself, so this never races with rescheduling. 3412 */ 3413 if (wgp->wgp_session_unstable->wgs_state == WGS_STATE_UNKNOWN && 3414 wgp->wgp_session_stable->wgs_state == WGS_STATE_UNKNOWN) 3415 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3416 } 3417 3418 static void 3419 wg_peer_work(struct work *wk, void *cookie) 3420 { 3421 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3422 struct wg_softc *wg = wgp->wgp_sc; 3423 unsigned int tasks; 3424 3425 mutex_enter(wgp->wgp_intr_lock); 3426 while ((tasks = wgp->wgp_tasks) != 0) { 3427 wgp->wgp_tasks = 0; 3428 mutex_exit(wgp->wgp_intr_lock); 3429 3430 mutex_enter(wgp->wgp_lock); 3431 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3432 wg_task_send_init_message(wg, wgp); 3433 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3434 wg_task_retry_handshake(wg, wgp); 3435 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3436 wg_task_establish_session(wg, wgp); 3437 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3438 wg_task_endpoint_changed(wg, wgp); 3439 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3440 wg_task_send_keepalive_message(wg, wgp); 3441 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3442 wg_task_destroy_prev_session(wg, wgp); 3443 mutex_exit(wgp->wgp_lock); 3444 3445 mutex_enter(wgp->wgp_intr_lock); 3446 } 3447 mutex_exit(wgp->wgp_intr_lock); 3448 } 3449 3450 static void 3451 wg_job(struct threadpool_job *job) 3452 { 3453 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3454 int bound, upcalls; 3455 3456 mutex_enter(wg->wg_intr_lock); 3457 while ((upcalls = wg->wg_upcalls) != 0) { 3458 wg->wg_upcalls = 0; 3459 mutex_exit(wg->wg_intr_lock); 3460 bound = curlwp_bind(); 3461 if (ISSET(upcalls, WG_UPCALL_INET)) 3462 wg_receive_packets(wg, AF_INET); 3463 if (ISSET(upcalls, WG_UPCALL_INET6)) 3464 wg_receive_packets(wg, AF_INET6); 3465 curlwp_bindx(bound); 3466 mutex_enter(wg->wg_intr_lock); 3467 } 3468 threadpool_job_done(job); 3469 mutex_exit(wg->wg_intr_lock); 3470 } 3471 3472 static int 3473 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3474 { 3475 int error; 3476 uint16_t old_port = wg->wg_listen_port; 3477 3478 if (port != 0 && old_port == port) 3479 return 0; 3480 3481 struct sockaddr_in _sin, *sin = &_sin; 3482 sin->sin_len = sizeof(*sin); 3483 sin->sin_family = AF_INET; 3484 sin->sin_addr.s_addr = INADDR_ANY; 3485 sin->sin_port = htons(port); 3486 3487 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3488 if (error != 0) 3489 return error; 3490 3491 #ifdef INET6 3492 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3493 sin6->sin6_len = sizeof(*sin6); 3494 sin6->sin6_family = AF_INET6; 3495 sin6->sin6_addr = in6addr_any; 3496 sin6->sin6_port = htons(port); 3497 3498 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3499 if (error != 0) 3500 return error; 3501 #endif 3502 3503 wg->wg_listen_port = port; 3504 3505 return 0; 3506 } 3507 3508 static void 3509 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3510 { 3511 struct wg_softc *wg = cookie; 3512 int reason; 3513 3514 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3515 WG_UPCALL_INET : 3516 WG_UPCALL_INET6; 3517 3518 mutex_enter(wg->wg_intr_lock); 3519 wg->wg_upcalls |= reason; 3520 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3521 mutex_exit(wg->wg_intr_lock); 3522 } 3523 3524 static int 3525 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3526 struct sockaddr *src, void *arg) 3527 { 3528 struct wg_softc *wg = arg; 3529 struct wg_msg wgm; 3530 struct mbuf *m = *mp; 3531 3532 WG_TRACE("enter"); 3533 3534 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3535 KASSERT(offset <= m_length(m)); 3536 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3537 /* drop on the floor */ 3538 m_freem(m); 3539 return -1; 3540 } 3541 3542 /* 3543 * Copy the message header (32-bit message type) out -- we'll 3544 * worry about contiguity and alignment later. 3545 */ 3546 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3547 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3548 3549 /* 3550 * Handle DATA packets promptly as they arrive, if they are in 3551 * an active session. Other packets may require expensive 3552 * public-key crypto and are not as sensitive to latency, so 3553 * defer them to the worker thread. 3554 */ 3555 switch (le32toh(wgm.wgm_type)) { 3556 case WG_MSG_TYPE_DATA: 3557 /* handle immediately */ 3558 m_adj(m, offset); 3559 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3560 m = m_pullup(m, sizeof(struct wg_msg_data)); 3561 if (m == NULL) 3562 return -1; 3563 } 3564 wg_handle_msg_data(wg, m, src); 3565 *mp = NULL; 3566 return 1; 3567 case WG_MSG_TYPE_INIT: 3568 case WG_MSG_TYPE_RESP: 3569 case WG_MSG_TYPE_COOKIE: 3570 /* pass through to so_receive in wg_receive_packets */ 3571 return 0; 3572 default: 3573 /* drop on the floor */ 3574 m_freem(m); 3575 return -1; 3576 } 3577 } 3578 3579 static int 3580 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3581 { 3582 int error; 3583 struct socket *so; 3584 3585 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3586 if (error != 0) 3587 return error; 3588 3589 solock(so); 3590 so->so_upcallarg = wg; 3591 so->so_upcall = wg_so_upcall; 3592 so->so_rcv.sb_flags |= SB_UPCALL; 3593 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3594 sounlock(so); 3595 3596 *sop = so; 3597 3598 return 0; 3599 } 3600 3601 static bool 3602 wg_session_hit_limits(struct wg_session *wgs) 3603 { 3604 uint32_t time_established = 3605 atomic_load_relaxed(&wgs->wgs_time_established); 3606 3607 /* 3608 * [W] 6.2: Transport Message Limits 3609 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3610 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3611 * comes first, WireGuard will refuse to send any more transport data 3612 * messages using the current secure session, ..." 3613 */ 3614 KASSERT(time_established != 0 || time_uptime > UINT32_MAX); 3615 if ((time_uptime32 - time_established) > wg_reject_after_time) { 3616 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3617 return true; 3618 } else if (wg_session_get_send_counter(wgs) > 3619 wg_reject_after_messages) { 3620 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3621 return true; 3622 } 3623 3624 return false; 3625 } 3626 3627 static void 3628 wgintr(void *cookie) 3629 { 3630 struct wg_peer *wgp; 3631 struct wg_session *wgs; 3632 struct mbuf *m; 3633 struct psref psref; 3634 3635 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3636 wgp = M_GETCTX(m, struct wg_peer *); 3637 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3638 WG_TRACE("no stable session"); 3639 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3640 goto next0; 3641 } 3642 if (__predict_false(wg_session_hit_limits(wgs))) { 3643 WG_TRACE("stable session hit limits"); 3644 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3645 goto next1; 3646 } 3647 wg_send_data_msg(wgp, wgs, m); 3648 m = NULL; /* consumed */ 3649 next1: wg_put_session(wgs, &psref); 3650 next0: m_freem(m); 3651 /* XXX Yield to avoid userland starvation? */ 3652 } 3653 } 3654 3655 static void 3656 wg_purge_pending_packets(struct wg_peer *wgp) 3657 { 3658 struct mbuf *m; 3659 3660 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 3661 m_freem(m); 3662 pktq_barrier(wg_pktq); 3663 } 3664 3665 static void 3666 wg_handshake_timeout_timer(void *arg) 3667 { 3668 struct wg_peer *wgp = arg; 3669 3670 WG_TRACE("enter"); 3671 3672 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3673 } 3674 3675 static struct wg_peer * 3676 wg_alloc_peer(struct wg_softc *wg) 3677 { 3678 struct wg_peer *wgp; 3679 3680 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3681 3682 wgp->wgp_sc = wg; 3683 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3684 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3685 wg_handshake_timeout_timer, wgp); 3686 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3687 callout_setfunc(&wgp->wgp_session_dtor_timer, 3688 wg_session_dtor_timer, wgp); 3689 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3690 wgp->wgp_endpoint_changing = false; 3691 wgp->wgp_endpoint_available = false; 3692 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3693 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3694 wgp->wgp_psz = pserialize_create(); 3695 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3696 3697 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3698 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3699 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3700 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3701 3702 struct wg_session *wgs; 3703 wgp->wgp_session_stable = 3704 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3705 wgp->wgp_session_unstable = 3706 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3707 wgs = wgp->wgp_session_stable; 3708 wgs->wgs_peer = wgp; 3709 wgs->wgs_state = WGS_STATE_UNKNOWN; 3710 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3711 #ifndef __HAVE_ATOMIC64_LOADSTORE 3712 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3713 #endif 3714 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3715 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3716 3717 wgs = wgp->wgp_session_unstable; 3718 wgs->wgs_peer = wgp; 3719 wgs->wgs_state = WGS_STATE_UNKNOWN; 3720 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3721 #ifndef __HAVE_ATOMIC64_LOADSTORE 3722 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3723 #endif 3724 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3725 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3726 3727 return wgp; 3728 } 3729 3730 static void 3731 wg_destroy_peer(struct wg_peer *wgp) 3732 { 3733 struct wg_session *wgs; 3734 struct wg_softc *wg = wgp->wgp_sc; 3735 3736 /* Prevent new packets from this peer on any source address. */ 3737 rw_enter(wg->wg_rwlock, RW_WRITER); 3738 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3739 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3740 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3741 struct radix_node *rn; 3742 3743 KASSERT(rnh != NULL); 3744 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3745 &wga->wga_sa_mask, rnh); 3746 if (rn == NULL) { 3747 char addrstr[128]; 3748 sockaddr_format(&wga->wga_sa_addr, addrstr, 3749 sizeof(addrstr)); 3750 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3751 if_name(&wg->wg_if), addrstr); 3752 } 3753 } 3754 rw_exit(wg->wg_rwlock); 3755 3756 /* Purge pending packets. */ 3757 wg_purge_pending_packets(wgp); 3758 3759 /* Halt all packet processing and timeouts. */ 3760 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3761 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3762 3763 /* Wait for any queued work to complete. */ 3764 workqueue_wait(wg_wq, &wgp->wgp_work); 3765 3766 wgs = wgp->wgp_session_unstable; 3767 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3768 mutex_enter(wgp->wgp_lock); 3769 wg_destroy_session(wg, wgs); 3770 mutex_exit(wgp->wgp_lock); 3771 } 3772 mutex_destroy(&wgs->wgs_recvwin->lock); 3773 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3774 #ifndef __HAVE_ATOMIC64_LOADSTORE 3775 mutex_destroy(&wgs->wgs_send_counter_lock); 3776 #endif 3777 kmem_free(wgs, sizeof(*wgs)); 3778 3779 wgs = wgp->wgp_session_stable; 3780 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3781 mutex_enter(wgp->wgp_lock); 3782 wg_destroy_session(wg, wgs); 3783 mutex_exit(wgp->wgp_lock); 3784 } 3785 mutex_destroy(&wgs->wgs_recvwin->lock); 3786 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3787 #ifndef __HAVE_ATOMIC64_LOADSTORE 3788 mutex_destroy(&wgs->wgs_send_counter_lock); 3789 #endif 3790 kmem_free(wgs, sizeof(*wgs)); 3791 3792 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3793 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3794 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3795 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3796 3797 pserialize_destroy(wgp->wgp_psz); 3798 mutex_obj_free(wgp->wgp_intr_lock); 3799 mutex_obj_free(wgp->wgp_lock); 3800 3801 kmem_free(wgp, sizeof(*wgp)); 3802 } 3803 3804 static void 3805 wg_destroy_all_peers(struct wg_softc *wg) 3806 { 3807 struct wg_peer *wgp, *wgp0 __diagused; 3808 void *garbage_byname, *garbage_bypubkey; 3809 3810 restart: 3811 garbage_byname = garbage_bypubkey = NULL; 3812 mutex_enter(wg->wg_lock); 3813 WG_PEER_WRITER_FOREACH(wgp, wg) { 3814 if (wgp->wgp_name[0]) { 3815 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3816 strlen(wgp->wgp_name)); 3817 KASSERT(wgp0 == wgp); 3818 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3819 } 3820 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3821 sizeof(wgp->wgp_pubkey)); 3822 KASSERT(wgp0 == wgp); 3823 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3824 WG_PEER_WRITER_REMOVE(wgp); 3825 wg->wg_npeers--; 3826 mutex_enter(wgp->wgp_lock); 3827 pserialize_perform(wgp->wgp_psz); 3828 mutex_exit(wgp->wgp_lock); 3829 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3830 break; 3831 } 3832 mutex_exit(wg->wg_lock); 3833 3834 if (wgp == NULL) 3835 return; 3836 3837 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3838 3839 wg_destroy_peer(wgp); 3840 thmap_gc(wg->wg_peers_byname, garbage_byname); 3841 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3842 3843 goto restart; 3844 } 3845 3846 static int 3847 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3848 { 3849 struct wg_peer *wgp, *wgp0 __diagused; 3850 void *garbage_byname, *garbage_bypubkey; 3851 3852 mutex_enter(wg->wg_lock); 3853 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3854 if (wgp != NULL) { 3855 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3856 sizeof(wgp->wgp_pubkey)); 3857 KASSERT(wgp0 == wgp); 3858 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3859 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3860 WG_PEER_WRITER_REMOVE(wgp); 3861 wg->wg_npeers--; 3862 if (wg->wg_npeers == 0) 3863 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3864 mutex_enter(wgp->wgp_lock); 3865 pserialize_perform(wgp->wgp_psz); 3866 mutex_exit(wgp->wgp_lock); 3867 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3868 } 3869 mutex_exit(wg->wg_lock); 3870 3871 if (wgp == NULL) 3872 return ENOENT; 3873 3874 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3875 3876 wg_destroy_peer(wgp); 3877 thmap_gc(wg->wg_peers_byname, garbage_byname); 3878 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3879 3880 return 0; 3881 } 3882 3883 static int 3884 wg_if_attach(struct wg_softc *wg) 3885 { 3886 3887 wg->wg_if.if_addrlen = 0; 3888 wg->wg_if.if_mtu = WG_MTU; 3889 wg->wg_if.if_flags = IFF_MULTICAST; 3890 wg->wg_if.if_extflags = IFEF_MPSAFE; 3891 wg->wg_if.if_ioctl = wg_ioctl; 3892 wg->wg_if.if_output = wg_output; 3893 wg->wg_if.if_init = wg_init; 3894 #ifdef ALTQ 3895 wg->wg_if.if_start = wg_start; 3896 #endif 3897 wg->wg_if.if_stop = wg_stop; 3898 wg->wg_if.if_type = IFT_OTHER; 3899 wg->wg_if.if_dlt = DLT_NULL; 3900 wg->wg_if.if_softc = wg; 3901 #ifdef ALTQ 3902 IFQ_SET_READY(&wg->wg_if.if_snd); 3903 #endif 3904 if_initialize(&wg->wg_if); 3905 3906 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3907 if_alloc_sadl(&wg->wg_if); 3908 if_register(&wg->wg_if); 3909 3910 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3911 3912 return 0; 3913 } 3914 3915 static void 3916 wg_if_detach(struct wg_softc *wg) 3917 { 3918 struct ifnet *ifp = &wg->wg_if; 3919 3920 bpf_detach(ifp); 3921 if_detach(ifp); 3922 } 3923 3924 static int 3925 wg_clone_create(struct if_clone *ifc, int unit) 3926 { 3927 struct wg_softc *wg; 3928 int error; 3929 3930 wg_guarantee_initialized(); 3931 3932 error = wg_count_inc(); 3933 if (error) 3934 return error; 3935 3936 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3937 3938 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3939 3940 PSLIST_INIT(&wg->wg_peers); 3941 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3942 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3943 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3944 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3945 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3946 wg->wg_rwlock = rw_obj_alloc(); 3947 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3948 "%s", if_name(&wg->wg_if)); 3949 wg->wg_ops = &wg_ops_rumpkernel; 3950 3951 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3952 if (error) 3953 goto fail0; 3954 3955 #ifdef INET 3956 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3957 if (error) 3958 goto fail1; 3959 rn_inithead((void **)&wg->wg_rtable_ipv4, 3960 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3961 #endif 3962 #ifdef INET6 3963 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3964 if (error) 3965 goto fail2; 3966 rn_inithead((void **)&wg->wg_rtable_ipv6, 3967 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3968 #endif 3969 3970 error = wg_if_attach(wg); 3971 if (error) 3972 goto fail3; 3973 3974 return 0; 3975 3976 fail4: __unused 3977 wg_if_detach(wg); 3978 fail3: wg_destroy_all_peers(wg); 3979 #ifdef INET6 3980 solock(wg->wg_so6); 3981 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3982 sounlock(wg->wg_so6); 3983 #endif 3984 #ifdef INET 3985 solock(wg->wg_so4); 3986 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3987 sounlock(wg->wg_so4); 3988 #endif 3989 mutex_enter(wg->wg_intr_lock); 3990 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3991 mutex_exit(wg->wg_intr_lock); 3992 #ifdef INET6 3993 if (wg->wg_rtable_ipv6 != NULL) 3994 free(wg->wg_rtable_ipv6, M_RTABLE); 3995 soclose(wg->wg_so6); 3996 fail2: 3997 #endif 3998 #ifdef INET 3999 if (wg->wg_rtable_ipv4 != NULL) 4000 free(wg->wg_rtable_ipv4, M_RTABLE); 4001 soclose(wg->wg_so4); 4002 fail1: 4003 #endif 4004 threadpool_put(wg->wg_threadpool, PRI_NONE); 4005 fail0: threadpool_job_destroy(&wg->wg_job); 4006 rw_obj_free(wg->wg_rwlock); 4007 mutex_obj_free(wg->wg_intr_lock); 4008 mutex_obj_free(wg->wg_lock); 4009 thmap_destroy(wg->wg_sessions_byindex); 4010 thmap_destroy(wg->wg_peers_byname); 4011 thmap_destroy(wg->wg_peers_bypubkey); 4012 PSLIST_DESTROY(&wg->wg_peers); 4013 kmem_free(wg, sizeof(*wg)); 4014 wg_count_dec(); 4015 return error; 4016 } 4017 4018 static int 4019 wg_clone_destroy(struct ifnet *ifp) 4020 { 4021 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 4022 4023 #ifdef WG_RUMPKERNEL 4024 if (wg_user_mode(wg)) { 4025 rumpuser_wg_destroy(wg->wg_user); 4026 wg->wg_user = NULL; 4027 } 4028 #endif 4029 4030 wg_if_detach(wg); 4031 wg_destroy_all_peers(wg); 4032 #ifdef INET6 4033 solock(wg->wg_so6); 4034 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 4035 sounlock(wg->wg_so6); 4036 #endif 4037 #ifdef INET 4038 solock(wg->wg_so4); 4039 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 4040 sounlock(wg->wg_so4); 4041 #endif 4042 mutex_enter(wg->wg_intr_lock); 4043 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 4044 mutex_exit(wg->wg_intr_lock); 4045 #ifdef INET6 4046 if (wg->wg_rtable_ipv6 != NULL) 4047 free(wg->wg_rtable_ipv6, M_RTABLE); 4048 soclose(wg->wg_so6); 4049 #endif 4050 #ifdef INET 4051 if (wg->wg_rtable_ipv4 != NULL) 4052 free(wg->wg_rtable_ipv4, M_RTABLE); 4053 soclose(wg->wg_so4); 4054 #endif 4055 threadpool_put(wg->wg_threadpool, PRI_NONE); 4056 threadpool_job_destroy(&wg->wg_job); 4057 rw_obj_free(wg->wg_rwlock); 4058 mutex_obj_free(wg->wg_intr_lock); 4059 mutex_obj_free(wg->wg_lock); 4060 thmap_destroy(wg->wg_sessions_byindex); 4061 thmap_destroy(wg->wg_peers_byname); 4062 thmap_destroy(wg->wg_peers_bypubkey); 4063 PSLIST_DESTROY(&wg->wg_peers); 4064 kmem_free(wg, sizeof(*wg)); 4065 wg_count_dec(); 4066 4067 return 0; 4068 } 4069 4070 static struct wg_peer * 4071 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 4072 struct psref *psref) 4073 { 4074 struct radix_node_head *rnh; 4075 struct radix_node *rn; 4076 struct wg_peer *wgp = NULL; 4077 struct wg_allowedip *wga; 4078 4079 #ifdef WG_DEBUG_LOG 4080 char addrstr[128]; 4081 sockaddr_format(sa, addrstr, sizeof(addrstr)); 4082 WG_DLOG("sa=%s\n", addrstr); 4083 #endif 4084 4085 rw_enter(wg->wg_rwlock, RW_READER); 4086 4087 rnh = wg_rnh(wg, sa->sa_family); 4088 if (rnh == NULL) 4089 goto out; 4090 4091 rn = rnh->rnh_matchaddr(sa, rnh); 4092 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 4093 goto out; 4094 4095 WG_TRACE("success"); 4096 4097 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 4098 wgp = wga->wga_peer; 4099 wg_get_peer(wgp, psref); 4100 4101 out: 4102 rw_exit(wg->wg_rwlock); 4103 return wgp; 4104 } 4105 4106 static void 4107 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 4108 struct wg_session *wgs, struct wg_msg_data *wgmd) 4109 { 4110 4111 memset(wgmd, 0, sizeof(*wgmd)); 4112 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 4113 wgmd->wgmd_receiver = wgs->wgs_remote_index; 4114 /* [W] 5.4.6: msg.counter := Nm^send */ 4115 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 4116 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 4117 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 4118 } 4119 4120 static int 4121 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 4122 const struct rtentry *rt) 4123 { 4124 struct wg_softc *wg = ifp->if_softc; 4125 struct wg_peer *wgp = NULL; 4126 struct wg_session *wgs = NULL; 4127 struct psref wgp_psref, wgs_psref; 4128 int bound; 4129 int error; 4130 4131 bound = curlwp_bind(); 4132 4133 /* TODO make the nest limit configurable via sysctl */ 4134 error = if_tunnel_check_nesting(ifp, m, 1); 4135 if (error) { 4136 WGLOG(LOG_ERR, 4137 "%s: tunneling loop detected and packet dropped\n", 4138 if_name(&wg->wg_if)); 4139 goto out0; 4140 } 4141 4142 #ifdef ALTQ 4143 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 4144 & ALTQF_ENABLED; 4145 if (altq) 4146 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 4147 #endif 4148 4149 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 4150 4151 m->m_flags &= ~(M_BCAST|M_MCAST); 4152 4153 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 4154 if (wgp == NULL) { 4155 WG_TRACE("peer not found"); 4156 error = EHOSTUNREACH; 4157 goto out0; 4158 } 4159 4160 /* Clear checksum-offload flags. */ 4161 m->m_pkthdr.csum_flags = 0; 4162 m->m_pkthdr.csum_data = 0; 4163 4164 /* Check whether there's an established session. */ 4165 wgs = wg_get_stable_session(wgp, &wgs_psref); 4166 if (wgs == NULL) { 4167 /* 4168 * No established session. If we're the first to try 4169 * sending data, schedule a handshake and queue the 4170 * packet for when the handshake is done; otherwise 4171 * just drop the packet and let the ongoing handshake 4172 * attempt continue. We could queue more data packets 4173 * but it's not clear that's worthwhile. 4174 */ 4175 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 4176 m = NULL; /* consume */ 4177 WG_TRACE("queued first packet; init handshake"); 4178 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4179 } else { 4180 WG_TRACE("first packet already queued, dropping"); 4181 } 4182 goto out1; 4183 } 4184 4185 /* There's an established session. Toss it in the queue. */ 4186 #ifdef ALTQ 4187 if (altq) { 4188 mutex_enter(ifp->if_snd.ifq_lock); 4189 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 4190 M_SETCTX(m, wgp); 4191 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 4192 m = NULL; /* consume */ 4193 } 4194 mutex_exit(ifp->if_snd.ifq_lock); 4195 if (m == NULL) { 4196 wg_start(ifp); 4197 goto out2; 4198 } 4199 } 4200 #endif 4201 kpreempt_disable(); 4202 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4203 M_SETCTX(m, wgp); 4204 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4205 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4206 if_name(&wg->wg_if)); 4207 error = ENOBUFS; 4208 goto out3; 4209 } 4210 m = NULL; /* consumed */ 4211 error = 0; 4212 out3: kpreempt_enable(); 4213 4214 #ifdef ALTQ 4215 out2: 4216 #endif 4217 wg_put_session(wgs, &wgs_psref); 4218 out1: wg_put_peer(wgp, &wgp_psref); 4219 out0: m_freem(m); 4220 curlwp_bindx(bound); 4221 return error; 4222 } 4223 4224 static int 4225 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 4226 { 4227 struct psref psref; 4228 struct wg_sockaddr *wgsa; 4229 int error; 4230 struct socket *so; 4231 4232 wgsa = wg_get_endpoint_sa(wgp, &psref); 4233 so = wg_get_so_by_peer(wgp, wgsa); 4234 solock(so); 4235 if (wgsatosa(wgsa)->sa_family == AF_INET) { 4236 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 4237 } else { 4238 #ifdef INET6 4239 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 4240 NULL, curlwp); 4241 #else 4242 m_freem(m); 4243 error = EPFNOSUPPORT; 4244 #endif 4245 } 4246 sounlock(so); 4247 wg_put_sa(wgp, wgsa, &psref); 4248 4249 return error; 4250 } 4251 4252 /* Inspired by pppoe_get_mbuf */ 4253 static struct mbuf * 4254 wg_get_mbuf(size_t leading_len, size_t len) 4255 { 4256 struct mbuf *m; 4257 4258 KASSERT(leading_len <= MCLBYTES); 4259 KASSERT(len <= MCLBYTES - leading_len); 4260 4261 m = m_gethdr(M_DONTWAIT, MT_DATA); 4262 if (m == NULL) 4263 return NULL; 4264 if (len + leading_len > MHLEN) { 4265 m_clget(m, M_DONTWAIT); 4266 if ((m->m_flags & M_EXT) == 0) { 4267 m_free(m); 4268 return NULL; 4269 } 4270 } 4271 m->m_data += leading_len; 4272 m->m_pkthdr.len = m->m_len = len; 4273 4274 return m; 4275 } 4276 4277 static int 4278 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 4279 struct mbuf *m) 4280 { 4281 struct wg_softc *wg = wgp->wgp_sc; 4282 int error; 4283 size_t inner_len, padded_len, encrypted_len; 4284 char *padded_buf = NULL; 4285 size_t mlen; 4286 struct wg_msg_data *wgmd; 4287 bool free_padded_buf = false; 4288 struct mbuf *n; 4289 size_t leading_len = max_hdr + sizeof(struct udphdr); 4290 4291 mlen = m_length(m); 4292 inner_len = mlen; 4293 padded_len = roundup(mlen, 16); 4294 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4295 WG_DLOG("inner=%zu, padded=%zu, encrypted_len=%zu\n", 4296 inner_len, padded_len, encrypted_len); 4297 if (mlen != 0) { 4298 bool success; 4299 success = m_ensure_contig(&m, padded_len); 4300 if (success) { 4301 padded_buf = mtod(m, char *); 4302 } else { 4303 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4304 if (padded_buf == NULL) { 4305 error = ENOBUFS; 4306 goto end; 4307 } 4308 free_padded_buf = true; 4309 m_copydata(m, 0, mlen, padded_buf); 4310 } 4311 memset(padded_buf + mlen, 0, padded_len - inner_len); 4312 } 4313 4314 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4315 if (n == NULL) { 4316 error = ENOBUFS; 4317 goto end; 4318 } 4319 KASSERT(n->m_len >= sizeof(*wgmd)); 4320 wgmd = mtod(n, struct wg_msg_data *); 4321 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4322 #ifdef WG_DEBUG_PACKET 4323 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4324 hexdump(printf, "padded_buf", padded_buf, 4325 padded_len); 4326 } 4327 #endif 4328 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4329 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4330 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4331 padded_buf, padded_len, 4332 NULL, 0); 4333 #ifdef WG_DEBUG_PACKET 4334 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4335 hexdump(printf, "tkey_send", wgs->wgs_tkey_send, 4336 sizeof(wgs->wgs_tkey_send)); 4337 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 4338 hexdump(printf, "outgoing packet", 4339 (char *)wgmd + sizeof(*wgmd), encrypted_len); 4340 size_t decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 4341 char *decrypted_buf = kmem_intr_alloc((decrypted_len + 4342 WG_AUTHTAG_LEN/*XXX*/), KM_NOSLEEP); 4343 if (decrypted_buf != NULL) { 4344 error = wg_algo_aead_dec( 4345 1 + decrypted_buf /* force misalignment */, 4346 encrypted_len - WG_AUTHTAG_LEN /* XXX */, 4347 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4348 (char *)wgmd + sizeof(*wgmd), encrypted_len, 4349 NULL, 0); 4350 if (error) { 4351 WG_DLOG("wg_algo_aead_dec failed: %d\n", 4352 error); 4353 } 4354 if (!consttime_memequal(1 + decrypted_buf, 4355 (char *)wgmd + sizeof(*wgmd), 4356 decrypted_len)) { 4357 WG_DLOG("wg_algo_aead_dec returned garbage\n"); 4358 } 4359 kmem_intr_free(decrypted_buf, (decrypted_len + 4360 WG_AUTHTAG_LEN/*XXX*/)); 4361 } 4362 } 4363 #endif 4364 4365 error = wg->wg_ops->send_data_msg(wgp, n); 4366 if (error == 0) { 4367 struct ifnet *ifp = &wg->wg_if; 4368 if_statadd(ifp, if_obytes, mlen); 4369 if_statinc(ifp, if_opackets); 4370 if (wgs->wgs_is_initiator && 4371 ((time_uptime32 - 4372 atomic_load_relaxed(&wgs->wgs_time_established)) >= 4373 wg_rekey_after_time)) { 4374 /* 4375 * [W] 6.2 Transport Message Limits 4376 * "if a peer is the initiator of a current secure 4377 * session, WireGuard will send a handshake initiation 4378 * message to begin a new secure session if, after 4379 * transmitting a transport data message, the current 4380 * secure session is REKEY-AFTER-TIME seconds old," 4381 */ 4382 WG_TRACE("rekey after time"); 4383 atomic_store_relaxed(&wgp->wgp_force_rekey, 1); 4384 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4385 } 4386 const uint32_t now = time_uptime32; 4387 atomic_store_relaxed(&wgs->wgs_time_last_data_sent, 4388 MAX(now, 1)); 4389 if (wg_session_get_send_counter(wgs) >= 4390 wg_rekey_after_messages) { 4391 /* 4392 * [W] 6.2 Transport Message Limits 4393 * "WireGuard will try to create a new session, by 4394 * sending a handshake initiation message (section 4395 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4396 * transport data messages..." 4397 */ 4398 WG_TRACE("rekey after messages"); 4399 atomic_store_relaxed(&wgp->wgp_force_rekey, 1); 4400 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4401 } 4402 } 4403 end: 4404 m_freem(m); 4405 if (free_padded_buf) 4406 kmem_intr_free(padded_buf, padded_len); 4407 return error; 4408 } 4409 4410 static void 4411 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4412 { 4413 pktqueue_t *pktq; 4414 size_t pktlen; 4415 4416 KASSERT(af == AF_INET || af == AF_INET6); 4417 4418 WG_TRACE(""); 4419 4420 m_set_rcvif(m, ifp); 4421 pktlen = m->m_pkthdr.len; 4422 4423 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4424 4425 switch (af) { 4426 case AF_INET: 4427 pktq = ip_pktq; 4428 break; 4429 #ifdef INET6 4430 case AF_INET6: 4431 pktq = ip6_pktq; 4432 break; 4433 #endif 4434 default: 4435 panic("invalid af=%d", af); 4436 } 4437 4438 kpreempt_disable(); 4439 const u_int h = curcpu()->ci_index; 4440 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4441 if_statadd(ifp, if_ibytes, pktlen); 4442 if_statinc(ifp, if_ipackets); 4443 } else { 4444 m_freem(m); 4445 } 4446 kpreempt_enable(); 4447 } 4448 4449 static void 4450 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4451 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4452 { 4453 4454 crypto_scalarmult_base(pubkey, privkey); 4455 } 4456 4457 static int 4458 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4459 { 4460 struct radix_node_head *rnh; 4461 struct radix_node *rn; 4462 int error = 0; 4463 4464 rw_enter(wg->wg_rwlock, RW_WRITER); 4465 rnh = wg_rnh(wg, wga->wga_family); 4466 KASSERT(rnh != NULL); 4467 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4468 wga->wga_nodes); 4469 rw_exit(wg->wg_rwlock); 4470 4471 if (rn == NULL) 4472 error = EEXIST; 4473 4474 return error; 4475 } 4476 4477 static int 4478 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4479 struct wg_peer **wgpp) 4480 { 4481 int error = 0; 4482 const void *pubkey; 4483 size_t pubkey_len; 4484 const void *psk; 4485 size_t psk_len; 4486 const char *name = NULL; 4487 4488 if (prop_dictionary_get_string(peer, "name", &name)) { 4489 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4490 error = EINVAL; 4491 goto out; 4492 } 4493 } 4494 4495 if (!prop_dictionary_get_data(peer, "public_key", 4496 &pubkey, &pubkey_len)) { 4497 error = EINVAL; 4498 goto out; 4499 } 4500 #ifdef WG_DEBUG_DUMP 4501 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4502 char *hex = gethexdump(pubkey, pubkey_len); 4503 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%zu\n%s\n", 4504 pubkey, pubkey_len, hex); 4505 puthexdump(hex, pubkey, pubkey_len); 4506 } 4507 #endif 4508 4509 struct wg_peer *wgp = wg_alloc_peer(wg); 4510 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4511 if (name != NULL) 4512 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4513 4514 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4515 if (psk_len != sizeof(wgp->wgp_psk)) { 4516 error = EINVAL; 4517 goto out; 4518 } 4519 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4520 } 4521 4522 const void *addr; 4523 size_t addr_len; 4524 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4525 4526 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4527 goto skip_endpoint; 4528 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4529 addr_len > sizeof(*wgsatoss(wgsa))) { 4530 error = EINVAL; 4531 goto out; 4532 } 4533 memcpy(wgsatoss(wgsa), addr, addr_len); 4534 switch (wgsa_family(wgsa)) { 4535 case AF_INET: 4536 #ifdef INET6 4537 case AF_INET6: 4538 #endif 4539 break; 4540 default: 4541 error = EPFNOSUPPORT; 4542 goto out; 4543 } 4544 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4545 error = EINVAL; 4546 goto out; 4547 } 4548 { 4549 char addrstr[128]; 4550 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4551 WG_DLOG("addr=%s\n", addrstr); 4552 } 4553 wgp->wgp_endpoint_available = true; 4554 4555 prop_array_t allowedips; 4556 skip_endpoint: 4557 allowedips = prop_dictionary_get(peer, "allowedips"); 4558 if (allowedips == NULL) 4559 goto skip; 4560 4561 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4562 prop_dictionary_t prop_allowedip; 4563 int j = 0; 4564 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4565 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4566 4567 if (!prop_dictionary_get_int(prop_allowedip, "family", 4568 &wga->wga_family)) 4569 continue; 4570 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4571 &addr, &addr_len)) 4572 continue; 4573 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4574 &wga->wga_cidr)) 4575 continue; 4576 4577 switch (wga->wga_family) { 4578 case AF_INET: { 4579 struct sockaddr_in sin; 4580 char addrstr[128]; 4581 struct in_addr mask; 4582 struct sockaddr_in sin_mask; 4583 4584 if (addr_len != sizeof(struct in_addr)) 4585 return EINVAL; 4586 memcpy(&wga->wga_addr4, addr, addr_len); 4587 4588 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4589 0); 4590 sockaddr_copy(&wga->wga_sa_addr, 4591 sizeof(sin), sintosa(&sin)); 4592 4593 sockaddr_format(sintosa(&sin), 4594 addrstr, sizeof(addrstr)); 4595 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4596 4597 in_len2mask(&mask, wga->wga_cidr); 4598 sockaddr_in_init(&sin_mask, &mask, 0); 4599 sockaddr_copy(&wga->wga_sa_mask, 4600 sizeof(sin_mask), sintosa(&sin_mask)); 4601 4602 break; 4603 } 4604 #ifdef INET6 4605 case AF_INET6: { 4606 struct sockaddr_in6 sin6; 4607 char addrstr[128]; 4608 struct in6_addr mask; 4609 struct sockaddr_in6 sin6_mask; 4610 4611 if (addr_len != sizeof(struct in6_addr)) 4612 return EINVAL; 4613 memcpy(&wga->wga_addr6, addr, addr_len); 4614 4615 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4616 0, 0, 0); 4617 sockaddr_copy(&wga->wga_sa_addr, 4618 sizeof(sin6), sin6tosa(&sin6)); 4619 4620 sockaddr_format(sin6tosa(&sin6), 4621 addrstr, sizeof(addrstr)); 4622 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4623 4624 in6_prefixlen2mask(&mask, wga->wga_cidr); 4625 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4626 sockaddr_copy(&wga->wga_sa_mask, 4627 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4628 4629 break; 4630 } 4631 #endif 4632 default: 4633 error = EINVAL; 4634 goto out; 4635 } 4636 wga->wga_peer = wgp; 4637 4638 error = wg_rtable_add_route(wg, wga); 4639 if (error != 0) 4640 goto out; 4641 4642 j++; 4643 } 4644 wgp->wgp_n_allowedips = j; 4645 skip: 4646 *wgpp = wgp; 4647 out: 4648 return error; 4649 } 4650 4651 static int 4652 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4653 { 4654 int error; 4655 char *buf; 4656 4657 WG_DLOG("buf=%p, len=%zu\n", ifd->ifd_data, ifd->ifd_len); 4658 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4659 return E2BIG; 4660 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4661 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4662 if (error != 0) 4663 return error; 4664 buf[ifd->ifd_len] = '\0'; 4665 #ifdef WG_DEBUG_DUMP 4666 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4667 log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len), 4668 (const char *)buf); 4669 } 4670 #endif 4671 *_buf = buf; 4672 return 0; 4673 } 4674 4675 static int 4676 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4677 { 4678 int error; 4679 prop_dictionary_t prop_dict; 4680 char *buf = NULL; 4681 const void *privkey; 4682 size_t privkey_len; 4683 4684 error = wg_alloc_prop_buf(&buf, ifd); 4685 if (error != 0) 4686 return error; 4687 error = EINVAL; 4688 prop_dict = prop_dictionary_internalize(buf); 4689 if (prop_dict == NULL) 4690 goto out; 4691 if (!prop_dictionary_get_data(prop_dict, "private_key", 4692 &privkey, &privkey_len)) 4693 goto out; 4694 #ifdef WG_DEBUG_DUMP 4695 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4696 char *hex = gethexdump(privkey, privkey_len); 4697 log(LOG_DEBUG, "privkey=%p, privkey_len=%zu\n%s\n", 4698 privkey, privkey_len, hex); 4699 puthexdump(hex, privkey, privkey_len); 4700 } 4701 #endif 4702 if (privkey_len != WG_STATIC_KEY_LEN) 4703 goto out; 4704 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4705 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4706 error = 0; 4707 4708 out: 4709 kmem_free(buf, ifd->ifd_len + 1); 4710 return error; 4711 } 4712 4713 static int 4714 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4715 { 4716 int error; 4717 prop_dictionary_t prop_dict; 4718 char *buf = NULL; 4719 uint16_t port; 4720 4721 error = wg_alloc_prop_buf(&buf, ifd); 4722 if (error != 0) 4723 return error; 4724 error = EINVAL; 4725 prop_dict = prop_dictionary_internalize(buf); 4726 if (prop_dict == NULL) 4727 goto out; 4728 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4729 goto out; 4730 4731 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4732 4733 out: 4734 kmem_free(buf, ifd->ifd_len + 1); 4735 return error; 4736 } 4737 4738 static int 4739 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4740 { 4741 int error; 4742 prop_dictionary_t prop_dict; 4743 char *buf = NULL; 4744 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4745 4746 error = wg_alloc_prop_buf(&buf, ifd); 4747 if (error != 0) 4748 return error; 4749 error = EINVAL; 4750 prop_dict = prop_dictionary_internalize(buf); 4751 if (prop_dict == NULL) 4752 goto out; 4753 4754 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4755 if (error != 0) 4756 goto out; 4757 4758 mutex_enter(wg->wg_lock); 4759 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4760 sizeof(wgp->wgp_pubkey)) != NULL || 4761 (wgp->wgp_name[0] && 4762 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4763 strlen(wgp->wgp_name)) != NULL)) { 4764 mutex_exit(wg->wg_lock); 4765 wg_destroy_peer(wgp); 4766 error = EEXIST; 4767 goto out; 4768 } 4769 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4770 sizeof(wgp->wgp_pubkey), wgp); 4771 KASSERT(wgp0 == wgp); 4772 if (wgp->wgp_name[0]) { 4773 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4774 strlen(wgp->wgp_name), wgp); 4775 KASSERT(wgp0 == wgp); 4776 } 4777 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4778 wg->wg_npeers++; 4779 mutex_exit(wg->wg_lock); 4780 4781 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4782 4783 out: 4784 kmem_free(buf, ifd->ifd_len + 1); 4785 return error; 4786 } 4787 4788 static int 4789 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4790 { 4791 int error; 4792 prop_dictionary_t prop_dict; 4793 char *buf = NULL; 4794 const char *name; 4795 4796 error = wg_alloc_prop_buf(&buf, ifd); 4797 if (error != 0) 4798 return error; 4799 error = EINVAL; 4800 prop_dict = prop_dictionary_internalize(buf); 4801 if (prop_dict == NULL) 4802 goto out; 4803 4804 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4805 goto out; 4806 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4807 goto out; 4808 4809 error = wg_destroy_peer_name(wg, name); 4810 out: 4811 kmem_free(buf, ifd->ifd_len + 1); 4812 return error; 4813 } 4814 4815 static bool 4816 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4817 { 4818 int au = cmd == SIOCGDRVSPEC ? 4819 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4820 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4821 return kauth_authorize_network(kauth_cred_get(), 4822 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4823 (void *)cmd, NULL) == 0; 4824 } 4825 4826 static int 4827 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4828 { 4829 int error = ENOMEM; 4830 prop_dictionary_t prop_dict; 4831 prop_array_t peers = NULL; 4832 char *buf; 4833 struct wg_peer *wgp; 4834 int s, i; 4835 4836 prop_dict = prop_dictionary_create(); 4837 if (prop_dict == NULL) 4838 goto error; 4839 4840 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4841 if (!prop_dictionary_set_data(prop_dict, "private_key", 4842 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4843 goto error; 4844 } 4845 4846 if (wg->wg_listen_port != 0) { 4847 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4848 wg->wg_listen_port)) 4849 goto error; 4850 } 4851 4852 if (wg->wg_npeers == 0) 4853 goto skip_peers; 4854 4855 peers = prop_array_create(); 4856 if (peers == NULL) 4857 goto error; 4858 4859 s = pserialize_read_enter(); 4860 i = 0; 4861 WG_PEER_READER_FOREACH(wgp, wg) { 4862 struct wg_sockaddr *wgsa; 4863 struct psref wgp_psref, wgsa_psref; 4864 prop_dictionary_t prop_peer; 4865 4866 wg_get_peer(wgp, &wgp_psref); 4867 pserialize_read_exit(s); 4868 4869 prop_peer = prop_dictionary_create(); 4870 if (prop_peer == NULL) 4871 goto next; 4872 4873 if (strlen(wgp->wgp_name) > 0) { 4874 if (!prop_dictionary_set_string(prop_peer, "name", 4875 wgp->wgp_name)) 4876 goto next; 4877 } 4878 4879 if (!prop_dictionary_set_data(prop_peer, "public_key", 4880 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4881 goto next; 4882 4883 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4884 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4885 sizeof(wgp->wgp_psk))) { 4886 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4887 if (!prop_dictionary_set_data(prop_peer, 4888 "preshared_key", 4889 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4890 goto next; 4891 } 4892 } 4893 4894 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4895 CTASSERT(AF_UNSPEC == 0); 4896 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4897 !prop_dictionary_set_data(prop_peer, "endpoint", 4898 wgsatoss(wgsa), 4899 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4900 wg_put_sa(wgp, wgsa, &wgsa_psref); 4901 goto next; 4902 } 4903 wg_put_sa(wgp, wgsa, &wgsa_psref); 4904 4905 const struct timespec *t = &wgp->wgp_last_handshake_time; 4906 4907 if (!prop_dictionary_set_uint64(prop_peer, 4908 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4909 goto next; 4910 if (!prop_dictionary_set_uint32(prop_peer, 4911 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4912 goto next; 4913 4914 if (wgp->wgp_n_allowedips == 0) 4915 goto skip_allowedips; 4916 4917 prop_array_t allowedips = prop_array_create(); 4918 if (allowedips == NULL) 4919 goto next; 4920 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4921 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4922 prop_dictionary_t prop_allowedip; 4923 4924 prop_allowedip = prop_dictionary_create(); 4925 if (prop_allowedip == NULL) 4926 break; 4927 4928 if (!prop_dictionary_set_int(prop_allowedip, "family", 4929 wga->wga_family)) 4930 goto _next; 4931 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4932 wga->wga_cidr)) 4933 goto _next; 4934 4935 switch (wga->wga_family) { 4936 case AF_INET: 4937 if (!prop_dictionary_set_data(prop_allowedip, 4938 "ip", &wga->wga_addr4, 4939 sizeof(wga->wga_addr4))) 4940 goto _next; 4941 break; 4942 #ifdef INET6 4943 case AF_INET6: 4944 if (!prop_dictionary_set_data(prop_allowedip, 4945 "ip", &wga->wga_addr6, 4946 sizeof(wga->wga_addr6))) 4947 goto _next; 4948 break; 4949 #endif 4950 default: 4951 break; 4952 } 4953 prop_array_set(allowedips, j, prop_allowedip); 4954 _next: 4955 prop_object_release(prop_allowedip); 4956 } 4957 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4958 prop_object_release(allowedips); 4959 4960 skip_allowedips: 4961 4962 prop_array_set(peers, i, prop_peer); 4963 next: 4964 if (prop_peer) 4965 prop_object_release(prop_peer); 4966 i++; 4967 4968 s = pserialize_read_enter(); 4969 wg_put_peer(wgp, &wgp_psref); 4970 } 4971 pserialize_read_exit(s); 4972 4973 prop_dictionary_set(prop_dict, "peers", peers); 4974 prop_object_release(peers); 4975 peers = NULL; 4976 4977 skip_peers: 4978 buf = prop_dictionary_externalize(prop_dict); 4979 if (buf == NULL) 4980 goto error; 4981 if (ifd->ifd_len < (strlen(buf) + 1)) { 4982 error = EINVAL; 4983 goto error; 4984 } 4985 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4986 4987 free(buf, 0); 4988 error: 4989 if (peers != NULL) 4990 prop_object_release(peers); 4991 if (prop_dict != NULL) 4992 prop_object_release(prop_dict); 4993 4994 return error; 4995 } 4996 4997 static int 4998 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4999 { 5000 struct wg_softc *wg = ifp->if_softc; 5001 struct ifreq *ifr = data; 5002 struct ifaddr *ifa = data; 5003 struct ifdrv *ifd = data; 5004 int error = 0; 5005 5006 switch (cmd) { 5007 case SIOCINITIFADDR: 5008 if (ifa->ifa_addr->sa_family != AF_LINK && 5009 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 5010 (IFF_UP | IFF_RUNNING)) { 5011 ifp->if_flags |= IFF_UP; 5012 error = if_init(ifp); 5013 } 5014 return error; 5015 case SIOCADDMULTI: 5016 case SIOCDELMULTI: 5017 switch (ifr->ifr_addr.sa_family) { 5018 case AF_INET: /* IP supports Multicast */ 5019 break; 5020 #ifdef INET6 5021 case AF_INET6: /* IP6 supports Multicast */ 5022 break; 5023 #endif 5024 default: /* Other protocols doesn't support Multicast */ 5025 error = EAFNOSUPPORT; 5026 break; 5027 } 5028 return error; 5029 case SIOCSDRVSPEC: 5030 if (!wg_is_authorized(wg, cmd)) { 5031 return EPERM; 5032 } 5033 switch (ifd->ifd_cmd) { 5034 case WG_IOCTL_SET_PRIVATE_KEY: 5035 error = wg_ioctl_set_private_key(wg, ifd); 5036 break; 5037 case WG_IOCTL_SET_LISTEN_PORT: 5038 error = wg_ioctl_set_listen_port(wg, ifd); 5039 break; 5040 case WG_IOCTL_ADD_PEER: 5041 error = wg_ioctl_add_peer(wg, ifd); 5042 break; 5043 case WG_IOCTL_DELETE_PEER: 5044 error = wg_ioctl_delete_peer(wg, ifd); 5045 break; 5046 default: 5047 error = EINVAL; 5048 break; 5049 } 5050 return error; 5051 case SIOCGDRVSPEC: 5052 return wg_ioctl_get(wg, ifd); 5053 case SIOCSIFFLAGS: 5054 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 5055 break; 5056 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 5057 case IFF_RUNNING: 5058 /* 5059 * If interface is marked down and it is running, 5060 * then stop and disable it. 5061 */ 5062 if_stop(ifp, 1); 5063 break; 5064 case IFF_UP: 5065 /* 5066 * If interface is marked up and it is stopped, then 5067 * start it. 5068 */ 5069 error = if_init(ifp); 5070 break; 5071 default: 5072 break; 5073 } 5074 return error; 5075 #ifdef WG_RUMPKERNEL 5076 case SIOCSLINKSTR: 5077 error = wg_ioctl_linkstr(wg, ifd); 5078 if (error == 0) 5079 wg->wg_ops = &wg_ops_rumpuser; 5080 return error; 5081 #endif 5082 default: 5083 break; 5084 } 5085 5086 error = ifioctl_common(ifp, cmd, data); 5087 5088 #ifdef WG_RUMPKERNEL 5089 if (!wg_user_mode(wg)) 5090 return error; 5091 5092 /* Do the same to the corresponding tun device on the host */ 5093 /* 5094 * XXX Actually the command has not been handled yet. It 5095 * will be handled via pr_ioctl form doifioctl later. 5096 */ 5097 switch (cmd) { 5098 case SIOCAIFADDR: 5099 case SIOCDIFADDR: { 5100 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 5101 struct in_aliasreq *ifra = &_ifra; 5102 KASSERT(error == ENOTTY); 5103 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 5104 IFNAMSIZ); 5105 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 5106 if (error == 0) 5107 error = ENOTTY; 5108 break; 5109 } 5110 #ifdef INET6 5111 case SIOCAIFADDR_IN6: 5112 case SIOCDIFADDR_IN6: { 5113 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 5114 struct in6_aliasreq *ifra = &_ifra; 5115 KASSERT(error == ENOTTY); 5116 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 5117 IFNAMSIZ); 5118 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 5119 if (error == 0) 5120 error = ENOTTY; 5121 break; 5122 } 5123 #endif 5124 } 5125 #endif /* WG_RUMPKERNEL */ 5126 5127 return error; 5128 } 5129 5130 static int 5131 wg_init(struct ifnet *ifp) 5132 { 5133 5134 ifp->if_flags |= IFF_RUNNING; 5135 5136 /* TODO flush pending packets. */ 5137 return 0; 5138 } 5139 5140 #ifdef ALTQ 5141 static void 5142 wg_start(struct ifnet *ifp) 5143 { 5144 struct mbuf *m; 5145 5146 for (;;) { 5147 IFQ_DEQUEUE(&ifp->if_snd, m); 5148 if (m == NULL) 5149 break; 5150 5151 kpreempt_disable(); 5152 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 5153 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 5154 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 5155 if_name(ifp)); 5156 m_freem(m); 5157 } 5158 kpreempt_enable(); 5159 } 5160 } 5161 #endif 5162 5163 static void 5164 wg_stop(struct ifnet *ifp, int disable) 5165 { 5166 5167 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 5168 ifp->if_flags &= ~IFF_RUNNING; 5169 5170 /* Need to do something? */ 5171 } 5172 5173 #ifdef WG_DEBUG_PARAMS 5174 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 5175 { 5176 const struct sysctlnode *node = NULL; 5177 5178 sysctl_createv(clog, 0, NULL, &node, 5179 CTLFLAG_PERMANENT, 5180 CTLTYPE_NODE, "wg", 5181 SYSCTL_DESCR("wg(4)"), 5182 NULL, 0, NULL, 0, 5183 CTL_NET, CTL_CREATE, CTL_EOL); 5184 sysctl_createv(clog, 0, &node, NULL, 5185 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5186 CTLTYPE_QUAD, "rekey_after_messages", 5187 SYSCTL_DESCR("session liftime by messages"), 5188 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 5189 sysctl_createv(clog, 0, &node, NULL, 5190 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5191 CTLTYPE_INT, "rekey_after_time", 5192 SYSCTL_DESCR("session liftime"), 5193 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 5194 sysctl_createv(clog, 0, &node, NULL, 5195 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5196 CTLTYPE_INT, "rekey_timeout", 5197 SYSCTL_DESCR("session handshake retry time"), 5198 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 5199 sysctl_createv(clog, 0, &node, NULL, 5200 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5201 CTLTYPE_INT, "rekey_attempt_time", 5202 SYSCTL_DESCR("session handshake timeout"), 5203 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 5204 sysctl_createv(clog, 0, &node, NULL, 5205 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5206 CTLTYPE_INT, "keepalive_timeout", 5207 SYSCTL_DESCR("keepalive timeout"), 5208 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 5209 sysctl_createv(clog, 0, &node, NULL, 5210 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5211 CTLTYPE_BOOL, "force_underload", 5212 SYSCTL_DESCR("force to detemine under load"), 5213 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 5214 sysctl_createv(clog, 0, &node, NULL, 5215 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5216 CTLTYPE_INT, "debug", 5217 SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"), 5218 NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL); 5219 } 5220 #endif 5221 5222 #ifdef WG_RUMPKERNEL 5223 static bool 5224 wg_user_mode(struct wg_softc *wg) 5225 { 5226 5227 return wg->wg_user != NULL; 5228 } 5229 5230 static int 5231 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 5232 { 5233 struct ifnet *ifp = &wg->wg_if; 5234 int error; 5235 5236 if (ifp->if_flags & IFF_UP) 5237 return EBUSY; 5238 5239 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 5240 /* XXX do nothing */ 5241 return 0; 5242 } else if (ifd->ifd_cmd != 0) { 5243 return EINVAL; 5244 } else if (wg->wg_user != NULL) { 5245 return EBUSY; 5246 } 5247 5248 /* Assume \0 included */ 5249 if (ifd->ifd_len > IFNAMSIZ) { 5250 return E2BIG; 5251 } else if (ifd->ifd_len < 1) { 5252 return EINVAL; 5253 } 5254 5255 char tun_name[IFNAMSIZ]; 5256 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 5257 if (error != 0) 5258 return error; 5259 5260 if (strncmp(tun_name, "tun", 3) != 0) 5261 return EINVAL; 5262 5263 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 5264 5265 return error; 5266 } 5267 5268 static int 5269 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 5270 { 5271 int error; 5272 struct psref psref; 5273 struct wg_sockaddr *wgsa; 5274 struct wg_softc *wg = wgp->wgp_sc; 5275 struct iovec iov[1]; 5276 5277 wgsa = wg_get_endpoint_sa(wgp, &psref); 5278 5279 iov[0].iov_base = mtod(m, void *); 5280 iov[0].iov_len = m->m_len; 5281 5282 /* Send messages to a peer via an ordinary socket. */ 5283 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 5284 5285 wg_put_sa(wgp, wgsa, &psref); 5286 5287 m_freem(m); 5288 5289 return error; 5290 } 5291 5292 static void 5293 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 5294 { 5295 struct wg_softc *wg = ifp->if_softc; 5296 struct iovec iov[2]; 5297 struct sockaddr_storage ss; 5298 5299 KASSERT(af == AF_INET || af == AF_INET6); 5300 5301 WG_TRACE(""); 5302 5303 if (af == AF_INET) { 5304 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 5305 struct ip *ip; 5306 5307 KASSERT(m->m_len >= sizeof(struct ip)); 5308 ip = mtod(m, struct ip *); 5309 sockaddr_in_init(sin, &ip->ip_dst, 0); 5310 } else { 5311 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 5312 struct ip6_hdr *ip6; 5313 5314 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 5315 ip6 = mtod(m, struct ip6_hdr *); 5316 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 5317 } 5318 5319 iov[0].iov_base = &ss; 5320 iov[0].iov_len = ss.ss_len; 5321 iov[1].iov_base = mtod(m, void *); 5322 iov[1].iov_len = m->m_len; 5323 5324 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5325 5326 /* Send decrypted packets to users via a tun. */ 5327 rumpuser_wg_send_user(wg->wg_user, iov, 2); 5328 5329 m_freem(m); 5330 } 5331 5332 static int 5333 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 5334 { 5335 int error; 5336 uint16_t old_port = wg->wg_listen_port; 5337 5338 if (port != 0 && old_port == port) 5339 return 0; 5340 5341 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5342 if (error == 0) 5343 wg->wg_listen_port = port; 5344 return error; 5345 } 5346 5347 /* 5348 * Receive user packets. 5349 */ 5350 void 5351 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5352 { 5353 struct ifnet *ifp = &wg->wg_if; 5354 struct mbuf *m; 5355 const struct sockaddr *dst; 5356 5357 WG_TRACE(""); 5358 5359 dst = iov[0].iov_base; 5360 5361 m = m_gethdr(M_DONTWAIT, MT_DATA); 5362 if (m == NULL) 5363 return; 5364 m->m_len = m->m_pkthdr.len = 0; 5365 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5366 5367 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5368 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5369 5370 (void)wg_output(ifp, m, dst, NULL); 5371 } 5372 5373 /* 5374 * Receive packets from a peer. 5375 */ 5376 void 5377 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5378 { 5379 struct mbuf *m; 5380 const struct sockaddr *src; 5381 int bound; 5382 5383 WG_TRACE(""); 5384 5385 src = iov[0].iov_base; 5386 5387 m = m_gethdr(M_DONTWAIT, MT_DATA); 5388 if (m == NULL) 5389 return; 5390 m->m_len = m->m_pkthdr.len = 0; 5391 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5392 5393 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5394 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5395 5396 bound = curlwp_bind(); 5397 wg_handle_packet(wg, m, src); 5398 curlwp_bindx(bound); 5399 } 5400 #endif /* WG_RUMPKERNEL */ 5401 5402 /* 5403 * Module infrastructure 5404 */ 5405 #include "if_module.h" 5406 5407 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5408
Indexes created Thu Sep 25 02:09:54 GMT 2025