if_wg.c revision 1.85
1 /* $NetBSD: if_wg.c,v 1.85 2024/07/25 00:07:33 christos Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.85 2024/07/25 00:07:33 christos Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <net/bpf.h> 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/if_wg.h> 90 #include <net/pktqueue.h> 91 #include <net/route.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip.h> 97 #include <netinet/ip_var.h> 98 #include <netinet/udp.h> 99 #include <netinet/udp_var.h> 100 101 #ifdef INET6 102 #include <netinet/ip6.h> 103 #include <netinet6/in6_pcb.h> 104 #include <netinet6/in6_var.h> 105 #include <netinet6/ip6_var.h> 106 #include <netinet6/udp6_var.h> 107 #endif /* INET6 */ 108 109 #include <prop/proplib.h> 110 111 #include <crypto/blake2/blake2s.h> 112 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 113 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 114 #include <crypto/sodium/crypto_scalarmult.h> 115 116 #include "ioconf.h" 117 118 #ifdef WG_RUMPKERNEL 119 #include "wg_user.h" 120 #endif 121 122 /* 123 * Data structures 124 * - struct wg_softc is an instance of wg interfaces 125 * - It has a list of peers (struct wg_peer) 126 * - It has a threadpool job that sends/receives handshake messages and 127 * runs event handlers 128 * - It has its own two routing tables: one is for IPv4 and the other IPv6 129 * - struct wg_peer is a representative of a peer 130 * - It has a struct work to handle handshakes and timer tasks 131 * - It has a pair of session instances (struct wg_session) 132 * - It has a pair of endpoint instances (struct wg_sockaddr) 133 * - Normally one endpoint is used and the second one is used only on 134 * a peer migration (a change of peer's IP address) 135 * - It has a list of IP addresses and sub networks called allowedips 136 * (struct wg_allowedip) 137 * - A packets sent over a session is allowed if its destination matches 138 * any IP addresses or sub networks of the list 139 * - struct wg_session represents a session of a secure tunnel with a peer 140 * - Two instances of sessions belong to a peer; a stable session and a 141 * unstable session 142 * - A handshake process of a session always starts with a unstable instance 143 * - Once a session is established, its instance becomes stable and the 144 * other becomes unstable instead 145 * - Data messages are always sent via a stable session 146 * 147 * Locking notes: 148 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 149 * - Changes to the peer list are serialized by wg_lock 150 * - The peer list may be read with pserialize(9) and psref(9) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 152 * => XXX replace by pserialize when routing table is psz-safe 153 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 154 * only in thread context and serializes: 155 * - the stable and unstable session pointers 156 * - all unstable session state 157 * - Packet processing may be done in softint context: 158 * - The stable session can be read under pserialize(9) or psref(9) 159 * - The stable session is always ESTABLISHED 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 * - Lock order: wg_lock -> wgp_lock 164 */ 165 166 167 #define WGLOG(level, fmt, args...) \ 168 log(level, "%s: " fmt, __func__, ##args) 169 170 #define WG_DEBUG 171 172 /* Debug options */ 173 #ifdef WG_DEBUG 174 /* Output debug logs */ 175 #ifndef WG_DEBUG_LOG 176 #define WG_DEBUG_LOG 177 #endif 178 /* Output trace logs */ 179 #ifndef WG_DEBUG_TRACE 180 #define WG_DEBUG_TRACE 181 #endif 182 /* Output hash values, etc. */ 183 #ifndef WG_DEBUG_DUMP 184 #define WG_DEBUG_DUMP 185 #endif 186 /* debug packets */ 187 #ifndef WG_DEBUG_PACKET 188 #define WG_DEBUG_PACKET 189 #endif 190 /* Make some internal parameters configurable for testing and debugging */ 191 #ifndef WG_DEBUG_PARAMS 192 #define WG_DEBUG_PARAMS 193 #endif 194 #endif /* WG_DEBUG */ 195 196 #ifndef WG_DEBUG 197 # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \ 198 defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS) 199 # define WG_DEBUG 200 # endif 201 #endif 202 203 #ifdef WG_DEBUG 204 int wg_debug; 205 #define WG_DEBUG_FLAGS_LOG 1 206 #define WG_DEBUG_FLAGS_TRACE 2 207 #define WG_DEBUG_FLAGS_DUMP 4 208 #define WG_DEBUG_FLAGS_PACKET 8 209 #endif 210 211 212 #ifdef WG_DEBUG_TRACE 213 #define WG_TRACE(msg) do { \ 214 if (wg_debug & WG_DEBUG_FLAGS_TRACE) \ 215 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \ 216 } while (0) 217 #else 218 #define WG_TRACE(msg) __nothing 219 #endif 220 221 #ifdef WG_DEBUG_LOG 222 #define WG_DLOG(fmt, args...) do { \ 223 if (wg_debug & WG_DEBUG_FLAGS_LOG) \ 224 log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \ 225 } while (0) 226 #else 227 #define WG_DLOG(fmt, args...) __nothing 228 #endif 229 230 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 231 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 232 &(wgprc)->wgprc_curpps, 1)) { \ 233 log(level, fmt, ##args); \ 234 } \ 235 } while (0) 236 237 #ifdef WG_DEBUG_PARAMS 238 static bool wg_force_underload = false; 239 #endif 240 241 #ifdef WG_DEBUG_DUMP 242 243 static char * 244 gethexdump(const void *vp, size_t n) 245 { 246 char *buf; 247 const uint8_t *p = vp; 248 size_t i; 249 250 if (n > (SIZE_MAX - 1) / 3) 251 return NULL; 252 buf = kmem_alloc(3 * n + 1, KM_NOSLEEP); 253 if (buf == NULL) 254 return NULL; 255 for (i = 0; i < n; i++) 256 snprintf(buf + 3 * i, 3 + 1, " %02hhx", p[i]); 257 return buf; 258 } 259 260 static void 261 puthexdump(char *buf, const void *p, size_t n) 262 { 263 264 if (buf == NULL) 265 return; 266 kmem_free(buf, 3*n + 1); 267 } 268 269 #ifdef WG_RUMPKERNEL 270 static void 271 wg_dump_buf(const char *func, const char *buf, const size_t size) 272 { 273 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 274 return; 275 276 char *hex = gethexdump(buf, size); 277 278 log(LOG_DEBUG, "%s: %s\n", func, hex ? hex : "(enomem)"); 279 puthexdump(hex, buf, size); 280 } 281 #endif 282 283 static void 284 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 285 const size_t size) 286 { 287 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 288 return; 289 290 char *hex = gethexdump(hash, size); 291 292 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex ? hex : "(enomem)"); 293 puthexdump(hex, hash, size); 294 } 295 296 #define WG_DUMP_HASH(name, hash) \ 297 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 298 #define WG_DUMP_HASH48(name, hash) \ 299 wg_dump_hash(__func__, name, hash, 48) 300 #define WG_DUMP_BUF(buf, size) \ 301 wg_dump_buf(__func__, buf, size) 302 #else 303 #define WG_DUMP_HASH(name, hash) __nothing 304 #define WG_DUMP_HASH48(name, hash) __nothing 305 #define WG_DUMP_BUF(buf, size) __nothing 306 #endif /* WG_DEBUG_DUMP */ 307 308 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 309 #define WG_MAX_PROPLEN 32766 310 311 #define WG_MTU 1420 312 #define WG_ALLOWEDIPS 16 313 314 #define CURVE25519_KEY_LEN 32 315 #define TAI64N_LEN sizeof(uint32_t) * 3 316 #define POLY1305_AUTHTAG_LEN 16 317 #define HMAC_BLOCK_LEN 64 318 319 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 320 /* [N] 4.3: Hash functions */ 321 #define NOISE_DHLEN 32 322 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 323 #define NOISE_HASHLEN 32 324 #define NOISE_BLOCKLEN 64 325 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 326 /* [N] 5.1: "k" */ 327 #define NOISE_CIPHER_KEY_LEN 32 328 /* 329 * [N] 9.2: "psk" 330 * "... psk is a 32-byte secret value provided by the application." 331 */ 332 #define NOISE_PRESHARED_KEY_LEN 32 333 334 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 335 #define WG_TIMESTAMP_LEN TAI64N_LEN 336 337 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 338 339 #define WG_COOKIE_LEN 16 340 #define WG_MAC_LEN 16 341 #define WG_RANDVAL_LEN 24 342 343 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 344 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 345 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 346 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 347 #define WG_HASH_LEN NOISE_HASHLEN 348 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 349 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 350 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 351 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 352 #define WG_DATA_KEY_LEN 32 353 #define WG_SALT_LEN 24 354 355 /* 356 * The protocol messages 357 */ 358 struct wg_msg { 359 uint32_t wgm_type; 360 } __packed; 361 362 /* [W] 5.4.2 First Message: Initiator to Responder */ 363 struct wg_msg_init { 364 uint32_t wgmi_type; 365 uint32_t wgmi_sender; 366 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 367 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 368 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 369 uint8_t wgmi_mac1[WG_MAC_LEN]; 370 uint8_t wgmi_mac2[WG_MAC_LEN]; 371 } __packed; 372 373 /* [W] 5.4.3 Second Message: Responder to Initiator */ 374 struct wg_msg_resp { 375 uint32_t wgmr_type; 376 uint32_t wgmr_sender; 377 uint32_t wgmr_receiver; 378 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 379 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 380 uint8_t wgmr_mac1[WG_MAC_LEN]; 381 uint8_t wgmr_mac2[WG_MAC_LEN]; 382 } __packed; 383 384 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 385 struct wg_msg_data { 386 uint32_t wgmd_type; 387 uint32_t wgmd_receiver; 388 uint64_t wgmd_counter; 389 uint32_t wgmd_packet[0]; 390 } __packed; 391 392 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 393 struct wg_msg_cookie { 394 uint32_t wgmc_type; 395 uint32_t wgmc_receiver; 396 uint8_t wgmc_salt[WG_SALT_LEN]; 397 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 398 } __packed; 399 400 #define WG_MSG_TYPE_INIT 1 401 #define WG_MSG_TYPE_RESP 2 402 #define WG_MSG_TYPE_COOKIE 3 403 #define WG_MSG_TYPE_DATA 4 404 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 405 406 /* Sliding windows */ 407 408 #define SLIWIN_BITS 2048u 409 #define SLIWIN_TYPE uint32_t 410 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 411 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 412 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 413 414 struct sliwin { 415 SLIWIN_TYPE B[SLIWIN_WORDS]; 416 uint64_t T; 417 }; 418 419 static void 420 sliwin_reset(struct sliwin *W) 421 { 422 423 memset(W, 0, sizeof(*W)); 424 } 425 426 static int 427 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 428 { 429 430 /* 431 * If it's more than one window older than the highest sequence 432 * number we've seen, reject. 433 */ 434 #ifdef __HAVE_ATOMIC64_LOADSTORE 435 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 436 return EAUTH; 437 #endif 438 439 /* 440 * Otherwise, we need to take the lock to decide, so don't 441 * reject just yet. Caller must serialize a call to 442 * sliwin_update in this case. 443 */ 444 return 0; 445 } 446 447 static int 448 sliwin_update(struct sliwin *W, uint64_t S) 449 { 450 unsigned word, bit; 451 452 /* 453 * If it's more than one window older than the highest sequence 454 * number we've seen, reject. 455 */ 456 if (S + SLIWIN_NPKT < W->T) 457 return EAUTH; 458 459 /* 460 * If it's higher than the highest sequence number we've seen, 461 * advance the window. 462 */ 463 if (S > W->T) { 464 uint64_t i = W->T / SLIWIN_BPW; 465 uint64_t j = S / SLIWIN_BPW; 466 unsigned k; 467 468 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 469 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 470 #ifdef __HAVE_ATOMIC64_LOADSTORE 471 atomic_store_relaxed(&W->T, S); 472 #else 473 W->T = S; 474 #endif 475 } 476 477 /* Test and set the bit -- if already set, reject. */ 478 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 479 bit = S % SLIWIN_BPW; 480 if (W->B[word] & (1UL << bit)) 481 return EAUTH; 482 W->B[word] |= 1U << bit; 483 484 /* Accept! */ 485 return 0; 486 } 487 488 struct wg_session { 489 struct wg_peer *wgs_peer; 490 struct psref_target 491 wgs_psref; 492 493 int wgs_state; 494 #define WGS_STATE_UNKNOWN 0 495 #define WGS_STATE_INIT_ACTIVE 1 496 #define WGS_STATE_INIT_PASSIVE 2 497 #define WGS_STATE_ESTABLISHED 3 498 #define WGS_STATE_DESTROYING 4 499 500 time_t wgs_time_established; 501 time_t wgs_time_last_data_sent; 502 bool wgs_is_initiator; 503 504 uint32_t wgs_local_index; 505 uint32_t wgs_remote_index; 506 #ifdef __HAVE_ATOMIC64_LOADSTORE 507 volatile uint64_t 508 wgs_send_counter; 509 #else 510 kmutex_t wgs_send_counter_lock; 511 uint64_t wgs_send_counter; 512 #endif 513 514 struct { 515 kmutex_t lock; 516 struct sliwin window; 517 } *wgs_recvwin; 518 519 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 520 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 521 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 522 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 523 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 524 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 525 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 526 }; 527 528 struct wg_sockaddr { 529 union { 530 struct sockaddr_storage _ss; 531 struct sockaddr _sa; 532 struct sockaddr_in _sin; 533 struct sockaddr_in6 _sin6; 534 }; 535 struct psref_target wgsa_psref; 536 }; 537 538 #define wgsatoss(wgsa) (&(wgsa)->_ss) 539 #define wgsatosa(wgsa) (&(wgsa)->_sa) 540 #define wgsatosin(wgsa) (&(wgsa)->_sin) 541 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 542 543 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 544 545 struct wg_peer; 546 struct wg_allowedip { 547 struct radix_node wga_nodes[2]; 548 struct wg_sockaddr _wga_sa_addr; 549 struct wg_sockaddr _wga_sa_mask; 550 #define wga_sa_addr _wga_sa_addr._sa 551 #define wga_sa_mask _wga_sa_mask._sa 552 553 int wga_family; 554 uint8_t wga_cidr; 555 union { 556 struct in_addr _ip4; 557 struct in6_addr _ip6; 558 } wga_addr; 559 #define wga_addr4 wga_addr._ip4 560 #define wga_addr6 wga_addr._ip6 561 562 struct wg_peer *wga_peer; 563 }; 564 565 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 566 567 struct wg_ppsratecheck { 568 struct timeval wgprc_lasttime; 569 int wgprc_curpps; 570 }; 571 572 struct wg_softc; 573 struct wg_peer { 574 struct wg_softc *wgp_sc; 575 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 576 struct pslist_entry wgp_peerlist_entry; 577 pserialize_t wgp_psz; 578 struct psref_target wgp_psref; 579 kmutex_t *wgp_lock; 580 kmutex_t *wgp_intr_lock; 581 582 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 583 struct wg_sockaddr *wgp_endpoint; 584 struct wg_sockaddr *wgp_endpoint0; 585 volatile unsigned wgp_endpoint_changing; 586 bool wgp_endpoint_available; 587 588 /* The preshared key (optional) */ 589 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 590 591 struct wg_session *wgp_session_stable; 592 struct wg_session *wgp_session_unstable; 593 594 /* first outgoing packet awaiting session initiation */ 595 struct mbuf *wgp_pending; 596 597 /* timestamp in big-endian */ 598 wg_timestamp_t wgp_timestamp_latest_init; 599 600 struct timespec wgp_last_handshake_time; 601 602 callout_t wgp_rekey_timer; 603 callout_t wgp_handshake_timeout_timer; 604 callout_t wgp_session_dtor_timer; 605 606 time_t wgp_handshake_start_time; 607 608 int wgp_n_allowedips; 609 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 610 611 time_t wgp_latest_cookie_time; 612 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 613 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 614 bool wgp_last_sent_mac1_valid; 615 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 616 bool wgp_last_sent_cookie_valid; 617 618 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 619 620 time_t wgp_last_genrandval_time; 621 uint32_t wgp_randval; 622 623 struct wg_ppsratecheck wgp_ppsratecheck; 624 625 struct work wgp_work; 626 unsigned int wgp_tasks; 627 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 628 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 629 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 630 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 631 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 632 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 633 }; 634 635 struct wg_ops; 636 637 struct wg_softc { 638 struct ifnet wg_if; 639 LIST_ENTRY(wg_softc) wg_list; 640 kmutex_t *wg_lock; 641 kmutex_t *wg_intr_lock; 642 krwlock_t *wg_rwlock; 643 644 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 645 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 646 647 int wg_npeers; 648 struct pslist_head wg_peers; 649 struct thmap *wg_peers_bypubkey; 650 struct thmap *wg_peers_byname; 651 struct thmap *wg_sessions_byindex; 652 uint16_t wg_listen_port; 653 654 struct threadpool *wg_threadpool; 655 656 struct threadpool_job wg_job; 657 int wg_upcalls; 658 #define WG_UPCALL_INET __BIT(0) 659 #define WG_UPCALL_INET6 __BIT(1) 660 661 #ifdef INET 662 struct socket *wg_so4; 663 struct radix_node_head *wg_rtable_ipv4; 664 #endif 665 #ifdef INET6 666 struct socket *wg_so6; 667 struct radix_node_head *wg_rtable_ipv6; 668 #endif 669 670 struct wg_ppsratecheck wg_ppsratecheck; 671 672 struct wg_ops *wg_ops; 673 674 #ifdef WG_RUMPKERNEL 675 struct wg_user *wg_user; 676 #endif 677 }; 678 679 /* [W] 6.1 Preliminaries */ 680 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 681 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 682 #define WG_REKEY_AFTER_TIME 120 683 #define WG_REJECT_AFTER_TIME 180 684 #define WG_REKEY_ATTEMPT_TIME 90 685 #define WG_REKEY_TIMEOUT 5 686 #define WG_KEEPALIVE_TIMEOUT 10 687 688 #define WG_COOKIE_TIME 120 689 #define WG_RANDVAL_TIME (2 * 60) 690 691 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 692 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 693 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 694 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 695 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 696 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 697 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 698 699 static struct mbuf * 700 wg_get_mbuf(size_t, size_t); 701 702 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 703 struct mbuf *); 704 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 705 const uint32_t, const uint8_t [WG_MAC_LEN], 706 const struct sockaddr *); 707 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 708 struct wg_session *, const struct wg_msg_init *); 709 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 710 711 static struct wg_peer * 712 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 713 struct psref *); 714 static struct wg_peer * 715 wg_lookup_peer_by_pubkey(struct wg_softc *, 716 const uint8_t [WG_STATIC_KEY_LEN], struct psref *); 717 718 static struct wg_session * 719 wg_lookup_session_by_index(struct wg_softc *, 720 const uint32_t, struct psref *); 721 722 static void wg_update_endpoint_if_necessary(struct wg_peer *, 723 const struct sockaddr *); 724 725 static void wg_schedule_rekey_timer(struct wg_peer *); 726 static void wg_schedule_session_dtor_timer(struct wg_peer *); 727 728 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 729 static void wg_calculate_keys(struct wg_session *, const bool); 730 731 static void wg_clear_states(struct wg_session *); 732 733 static void wg_get_peer(struct wg_peer *, struct psref *); 734 static void wg_put_peer(struct wg_peer *, struct psref *); 735 736 static int wg_send_so(struct wg_peer *, struct mbuf *); 737 static int wg_send_udp(struct wg_peer *, struct mbuf *); 738 static int wg_output(struct ifnet *, struct mbuf *, 739 const struct sockaddr *, const struct rtentry *); 740 static void wg_input(struct ifnet *, struct mbuf *, const int); 741 static int wg_ioctl(struct ifnet *, u_long, void *); 742 static int wg_bind_port(struct wg_softc *, const uint16_t); 743 static int wg_init(struct ifnet *); 744 #ifdef ALTQ 745 static void wg_start(struct ifnet *); 746 #endif 747 static void wg_stop(struct ifnet *, int); 748 749 static void wg_peer_work(struct work *, void *); 750 static void wg_job(struct threadpool_job *); 751 static void wgintr(void *); 752 static void wg_purge_pending_packets(struct wg_peer *); 753 754 static int wg_clone_create(struct if_clone *, int); 755 static int wg_clone_destroy(struct ifnet *); 756 757 struct wg_ops { 758 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 759 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 760 void (*input)(struct ifnet *, struct mbuf *, const int); 761 int (*bind_port)(struct wg_softc *, const uint16_t); 762 }; 763 764 struct wg_ops wg_ops_rumpkernel = { 765 .send_hs_msg = wg_send_so, 766 .send_data_msg = wg_send_udp, 767 .input = wg_input, 768 .bind_port = wg_bind_port, 769 }; 770 771 #ifdef WG_RUMPKERNEL 772 static bool wg_user_mode(struct wg_softc *); 773 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 774 775 static int wg_send_user(struct wg_peer *, struct mbuf *); 776 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 777 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 778 779 struct wg_ops wg_ops_rumpuser = { 780 .send_hs_msg = wg_send_user, 781 .send_data_msg = wg_send_user, 782 .input = wg_input_user, 783 .bind_port = wg_bind_port_user, 784 }; 785 #endif 786 787 #define WG_PEER_READER_FOREACH(wgp, wg) \ 788 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 789 wgp_peerlist_entry) 790 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 791 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 792 wgp_peerlist_entry) 793 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 794 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 795 #define WG_PEER_WRITER_REMOVE(wgp) \ 796 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 797 798 struct wg_route { 799 struct radix_node wgr_nodes[2]; 800 struct wg_peer *wgr_peer; 801 }; 802 803 static struct radix_node_head * 804 wg_rnh(struct wg_softc *wg, const int family) 805 { 806 807 switch (family) { 808 case AF_INET: 809 return wg->wg_rtable_ipv4; 810 #ifdef INET6 811 case AF_INET6: 812 return wg->wg_rtable_ipv6; 813 #endif 814 default: 815 return NULL; 816 } 817 } 818 819 820 /* 821 * Global variables 822 */ 823 static volatile unsigned wg_count __cacheline_aligned; 824 825 struct psref_class *wg_psref_class __read_mostly; 826 827 static struct if_clone wg_cloner = 828 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 829 830 static struct pktqueue *wg_pktq __read_mostly; 831 static struct workqueue *wg_wq __read_mostly; 832 833 void wgattach(int); 834 /* ARGSUSED */ 835 void 836 wgattach(int count) 837 { 838 /* 839 * Nothing to do here, initialization is handled by the 840 * module initialization code in wginit() below). 841 */ 842 } 843 844 static void 845 wginit(void) 846 { 847 848 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 849 850 if_clone_attach(&wg_cloner); 851 } 852 853 /* 854 * XXX Kludge: This should just happen in wginit, but workqueue_create 855 * cannot be run until after CPUs have been detected, and wginit runs 856 * before configure. 857 */ 858 static int 859 wginitqueues(void) 860 { 861 int error __diagused; 862 863 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 864 KASSERT(wg_pktq != NULL); 865 866 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 867 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 868 KASSERT(error == 0); 869 870 return 0; 871 } 872 873 static void 874 wg_guarantee_initialized(void) 875 { 876 static ONCE_DECL(init); 877 int error __diagused; 878 879 error = RUN_ONCE(&init, wginitqueues); 880 KASSERT(error == 0); 881 } 882 883 static int 884 wg_count_inc(void) 885 { 886 unsigned o, n; 887 888 do { 889 o = atomic_load_relaxed(&wg_count); 890 if (o == UINT_MAX) 891 return ENFILE; 892 n = o + 1; 893 } while (atomic_cas_uint(&wg_count, o, n) != o); 894 895 return 0; 896 } 897 898 static void 899 wg_count_dec(void) 900 { 901 unsigned c __diagused; 902 903 c = atomic_dec_uint_nv(&wg_count); 904 KASSERT(c != UINT_MAX); 905 } 906 907 static int 908 wgdetach(void) 909 { 910 911 /* Prevent new interface creation. */ 912 if_clone_detach(&wg_cloner); 913 914 /* Check whether there are any existing interfaces. */ 915 if (atomic_load_relaxed(&wg_count)) { 916 /* Back out -- reattach the cloner. */ 917 if_clone_attach(&wg_cloner); 918 return EBUSY; 919 } 920 921 /* No interfaces left. Nuke it. */ 922 workqueue_destroy(wg_wq); 923 pktq_destroy(wg_pktq); 924 psref_class_destroy(wg_psref_class); 925 926 return 0; 927 } 928 929 static void 930 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 931 uint8_t hash[WG_HASH_LEN]) 932 { 933 /* [W] 5.4: CONSTRUCTION */ 934 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 935 /* [W] 5.4: IDENTIFIER */ 936 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 937 struct blake2s state; 938 939 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 940 signature, strlen(signature)); 941 942 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 943 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 944 945 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 946 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 947 blake2s_update(&state, id, strlen(id)); 948 blake2s_final(&state, hash); 949 950 WG_DUMP_HASH("ckey", ckey); 951 WG_DUMP_HASH("hash", hash); 952 } 953 954 static void 955 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 956 const size_t inputsize) 957 { 958 struct blake2s state; 959 960 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 961 blake2s_update(&state, hash, WG_HASH_LEN); 962 blake2s_update(&state, input, inputsize); 963 blake2s_final(&state, hash); 964 } 965 966 static void 967 wg_algo_mac(uint8_t out[], const size_t outsize, 968 const uint8_t key[], const size_t keylen, 969 const uint8_t input1[], const size_t input1len, 970 const uint8_t input2[], const size_t input2len) 971 { 972 struct blake2s state; 973 974 blake2s_init(&state, outsize, key, keylen); 975 976 blake2s_update(&state, input1, input1len); 977 if (input2 != NULL) 978 blake2s_update(&state, input2, input2len); 979 blake2s_final(&state, out); 980 } 981 982 static void 983 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 984 const uint8_t input1[], const size_t input1len, 985 const uint8_t input2[], const size_t input2len) 986 { 987 struct blake2s state; 988 /* [W] 5.4: LABEL-MAC1 */ 989 const char *label = "mac1----"; 990 uint8_t key[WG_HASH_LEN]; 991 992 blake2s_init(&state, sizeof(key), NULL, 0); 993 blake2s_update(&state, label, strlen(label)); 994 blake2s_update(&state, input1, input1len); 995 blake2s_final(&state, key); 996 997 blake2s_init(&state, outsize, key, sizeof(key)); 998 if (input2 != NULL) 999 blake2s_update(&state, input2, input2len); 1000 blake2s_final(&state, out); 1001 } 1002 1003 static void 1004 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 1005 const uint8_t input1[], const size_t input1len) 1006 { 1007 struct blake2s state; 1008 /* [W] 5.4: LABEL-COOKIE */ 1009 const char *label = "cookie--"; 1010 1011 blake2s_init(&state, outsize, NULL, 0); 1012 blake2s_update(&state, label, strlen(label)); 1013 blake2s_update(&state, input1, input1len); 1014 blake2s_final(&state, out); 1015 } 1016 1017 static void 1018 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 1019 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 1020 { 1021 1022 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1023 1024 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 1025 crypto_scalarmult_base(pubkey, privkey); 1026 } 1027 1028 static void 1029 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 1030 const uint8_t privkey[WG_STATIC_KEY_LEN], 1031 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 1032 { 1033 1034 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1035 1036 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1037 KASSERT(ret == 0); 1038 } 1039 1040 static void 1041 wg_algo_hmac(uint8_t out[], const size_t outlen, 1042 const uint8_t key[], const size_t keylen, 1043 const uint8_t in[], const size_t inlen) 1044 { 1045 #define IPAD 0x36 1046 #define OPAD 0x5c 1047 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1048 uint8_t ipad[HMAC_BLOCK_LEN]; 1049 uint8_t opad[HMAC_BLOCK_LEN]; 1050 size_t i; 1051 struct blake2s state; 1052 1053 KASSERT(outlen == WG_HASH_LEN); 1054 KASSERT(keylen <= HMAC_BLOCK_LEN); 1055 1056 memcpy(hmackey, key, keylen); 1057 1058 for (i = 0; i < sizeof(hmackey); i++) { 1059 ipad[i] = hmackey[i] ^ IPAD; 1060 opad[i] = hmackey[i] ^ OPAD; 1061 } 1062 1063 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1064 blake2s_update(&state, ipad, sizeof(ipad)); 1065 blake2s_update(&state, in, inlen); 1066 blake2s_final(&state, out); 1067 1068 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1069 blake2s_update(&state, opad, sizeof(opad)); 1070 blake2s_update(&state, out, WG_HASH_LEN); 1071 blake2s_final(&state, out); 1072 #undef IPAD 1073 #undef OPAD 1074 } 1075 1076 static void 1077 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1078 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1079 const uint8_t input[], const size_t inputlen) 1080 { 1081 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1082 uint8_t one[1]; 1083 1084 /* 1085 * [N] 4.3: "an input_key_material byte sequence with length 1086 * either zero bytes, 32 bytes, or DHLEN bytes." 1087 */ 1088 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1089 1090 WG_DUMP_HASH("ckey", ckey); 1091 if (input != NULL) 1092 WG_DUMP_HASH("input", input); 1093 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1094 input, inputlen); 1095 WG_DUMP_HASH("tmp1", tmp1); 1096 one[0] = 1; 1097 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1098 one, sizeof(one)); 1099 WG_DUMP_HASH("out1", out1); 1100 if (out2 == NULL) 1101 return; 1102 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1103 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1104 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1105 tmp2, sizeof(tmp2)); 1106 WG_DUMP_HASH("out2", out2); 1107 if (out3 == NULL) 1108 return; 1109 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1110 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1111 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1112 tmp2, sizeof(tmp2)); 1113 WG_DUMP_HASH("out3", out3); 1114 } 1115 1116 static void __noinline 1117 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1118 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1119 const uint8_t local_key[WG_STATIC_KEY_LEN], 1120 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1121 { 1122 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1123 1124 wg_algo_dh(dhout, local_key, remote_key); 1125 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1126 1127 WG_DUMP_HASH("dhout", dhout); 1128 WG_DUMP_HASH("ckey", ckey); 1129 if (cipher_key != NULL) 1130 WG_DUMP_HASH("cipher_key", cipher_key); 1131 } 1132 1133 static void 1134 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1135 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1136 const uint8_t auth[], size_t authlen) 1137 { 1138 uint8_t nonce[(32 + 64) / 8] = {0}; 1139 long long unsigned int outsize; 1140 int error __diagused; 1141 1142 le64enc(&nonce[4], counter); 1143 1144 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1145 plainsize, auth, authlen, NULL, nonce, key); 1146 KASSERT(error == 0); 1147 KASSERT(outsize == expected_outsize); 1148 } 1149 1150 static int 1151 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1152 const uint64_t counter, const uint8_t encrypted[], 1153 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1154 { 1155 uint8_t nonce[(32 + 64) / 8] = {0}; 1156 long long unsigned int outsize; 1157 int error; 1158 1159 le64enc(&nonce[4], counter); 1160 1161 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1162 encrypted, encryptedsize, auth, authlen, nonce, key); 1163 if (error == 0) 1164 KASSERT(outsize == expected_outsize); 1165 return error; 1166 } 1167 1168 static void 1169 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1170 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1171 const uint8_t auth[], size_t authlen, 1172 const uint8_t nonce[WG_SALT_LEN]) 1173 { 1174 long long unsigned int outsize; 1175 int error __diagused; 1176 1177 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1178 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1179 plain, plainsize, auth, authlen, NULL, nonce, key); 1180 KASSERT(error == 0); 1181 KASSERT(outsize == expected_outsize); 1182 } 1183 1184 static int 1185 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1186 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1187 const uint8_t auth[], size_t authlen, 1188 const uint8_t nonce[WG_SALT_LEN]) 1189 { 1190 long long unsigned int outsize; 1191 int error; 1192 1193 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1194 encrypted, encryptedsize, auth, authlen, nonce, key); 1195 if (error == 0) 1196 KASSERT(outsize == expected_outsize); 1197 return error; 1198 } 1199 1200 static void 1201 wg_algo_tai64n(wg_timestamp_t timestamp) 1202 { 1203 struct timespec ts; 1204 1205 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1206 getnanotime(&ts); 1207 /* TAI64 label in external TAI64 format */ 1208 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1209 /* second beginning from 1970 TAI */ 1210 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1211 /* nanosecond in big-endian format */ 1212 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1213 } 1214 1215 /* 1216 * wg_get_stable_session(wgp, psref) 1217 * 1218 * Get a passive reference to the current stable session, or 1219 * return NULL if there is no current stable session. 1220 * 1221 * The pointer is always there but the session is not necessarily 1222 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1223 * the session may transition from ESTABLISHED to DESTROYING while 1224 * holding the passive reference. 1225 */ 1226 static struct wg_session * 1227 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1228 { 1229 int s; 1230 struct wg_session *wgs; 1231 1232 s = pserialize_read_enter(); 1233 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1234 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1235 wgs = NULL; 1236 else 1237 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1238 pserialize_read_exit(s); 1239 1240 return wgs; 1241 } 1242 1243 static void 1244 wg_put_session(struct wg_session *wgs, struct psref *psref) 1245 { 1246 1247 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1248 } 1249 1250 static void 1251 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1252 { 1253 struct wg_peer *wgp = wgs->wgs_peer; 1254 struct wg_session *wgs0 __diagused; 1255 void *garbage; 1256 1257 KASSERT(mutex_owned(wgp->wgp_lock)); 1258 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1259 1260 /* Remove the session from the table. */ 1261 wgs0 = thmap_del(wg->wg_sessions_byindex, 1262 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1263 KASSERT(wgs0 == wgs); 1264 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1265 1266 /* Wait for passive references to drain. */ 1267 pserialize_perform(wgp->wgp_psz); 1268 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1269 1270 /* Free memory, zero state, and transition to UNKNOWN. */ 1271 thmap_gc(wg->wg_sessions_byindex, garbage); 1272 wg_clear_states(wgs); 1273 wgs->wgs_state = WGS_STATE_UNKNOWN; 1274 } 1275 1276 /* 1277 * wg_get_session_index(wg, wgs) 1278 * 1279 * Choose a session index for wgs->wgs_local_index, and store it 1280 * in wg's table of sessions by index. 1281 * 1282 * wgs must be the unstable session of its peer, and must be 1283 * transitioning out of the UNKNOWN state. 1284 */ 1285 static void 1286 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1287 { 1288 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1289 struct wg_session *wgs0; 1290 uint32_t index; 1291 1292 KASSERT(mutex_owned(wgp->wgp_lock)); 1293 KASSERT(wgs == wgp->wgp_session_unstable); 1294 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1295 1296 do { 1297 /* Pick a uniform random index. */ 1298 index = cprng_strong32(); 1299 1300 /* Try to take it. */ 1301 wgs->wgs_local_index = index; 1302 wgs0 = thmap_put(wg->wg_sessions_byindex, 1303 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1304 1305 /* If someone else beat us, start over. */ 1306 } while (__predict_false(wgs0 != wgs)); 1307 } 1308 1309 /* 1310 * wg_put_session_index(wg, wgs) 1311 * 1312 * Remove wgs from the table of sessions by index, wait for any 1313 * passive references to drain, and transition the session to the 1314 * UNKNOWN state. 1315 * 1316 * wgs must be the unstable session of its peer, and must not be 1317 * UNKNOWN or ESTABLISHED. 1318 */ 1319 static void 1320 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1321 { 1322 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1323 1324 KASSERT(mutex_owned(wgp->wgp_lock)); 1325 KASSERT(wgs == wgp->wgp_session_unstable); 1326 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1327 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1328 1329 wg_destroy_session(wg, wgs); 1330 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1331 } 1332 1333 /* 1334 * Handshake patterns 1335 * 1336 * [W] 5: "These messages use the "IK" pattern from Noise" 1337 * [N] 7.5. Interactive handshake patterns (fundamental) 1338 * "The first character refers to the initiators static key:" 1339 * "I = Static key for initiator Immediately transmitted to responder, 1340 * despite reduced or absent identity hiding" 1341 * "The second character refers to the responders static key:" 1342 * "K = Static key for responder Known to initiator" 1343 * "IK: 1344 * <- s 1345 * ... 1346 * -> e, es, s, ss 1347 * <- e, ee, se" 1348 * [N] 9.4. Pattern modifiers 1349 * "IKpsk2: 1350 * <- s 1351 * ... 1352 * -> e, es, s, ss 1353 * <- e, ee, se, psk" 1354 */ 1355 static void 1356 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1357 struct wg_session *wgs, struct wg_msg_init *wgmi) 1358 { 1359 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1360 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1361 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1362 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1363 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1364 1365 KASSERT(mutex_owned(wgp->wgp_lock)); 1366 KASSERT(wgs == wgp->wgp_session_unstable); 1367 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1368 1369 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1370 wgmi->wgmi_sender = wgs->wgs_local_index; 1371 1372 /* [W] 5.4.2: First Message: Initiator to Responder */ 1373 1374 /* Ci := HASH(CONSTRUCTION) */ 1375 /* Hi := HASH(Ci || IDENTIFIER) */ 1376 wg_init_key_and_hash(ckey, hash); 1377 /* Hi := HASH(Hi || Sr^pub) */ 1378 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1379 1380 WG_DUMP_HASH("hash", hash); 1381 1382 /* [N] 2.2: "e" */ 1383 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1384 wg_algo_generate_keypair(pubkey, privkey); 1385 /* Ci := KDF1(Ci, Ei^pub) */ 1386 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1387 /* msg.ephemeral := Ei^pub */ 1388 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1389 /* Hi := HASH(Hi || msg.ephemeral) */ 1390 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1391 1392 WG_DUMP_HASH("ckey", ckey); 1393 WG_DUMP_HASH("hash", hash); 1394 1395 /* [N] 2.2: "es" */ 1396 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1397 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1398 1399 /* [N] 2.2: "s" */ 1400 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1401 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1402 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1403 hash, sizeof(hash)); 1404 /* Hi := HASH(Hi || msg.static) */ 1405 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1406 1407 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1408 1409 /* [N] 2.2: "ss" */ 1410 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1411 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1412 1413 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1414 wg_timestamp_t timestamp; 1415 wg_algo_tai64n(timestamp); 1416 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1417 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1418 /* Hi := HASH(Hi || msg.timestamp) */ 1419 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1420 1421 /* [W] 5.4.4 Cookie MACs */ 1422 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1423 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1424 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1425 /* Need mac1 to decrypt a cookie from a cookie message */ 1426 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1427 sizeof(wgp->wgp_last_sent_mac1)); 1428 wgp->wgp_last_sent_mac1_valid = true; 1429 1430 if (wgp->wgp_latest_cookie_time == 0 || 1431 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1432 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1433 else { 1434 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1435 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1436 (const uint8_t *)wgmi, 1437 offsetof(struct wg_msg_init, wgmi_mac2), 1438 NULL, 0); 1439 } 1440 1441 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1442 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1443 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1444 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1445 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1446 } 1447 1448 static void __noinline 1449 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1450 const struct sockaddr *src) 1451 { 1452 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1453 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1454 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1455 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1456 struct wg_peer *wgp; 1457 struct wg_session *wgs; 1458 int error, ret; 1459 struct psref psref_peer; 1460 uint8_t mac1[WG_MAC_LEN]; 1461 1462 WG_TRACE("init msg received"); 1463 1464 wg_algo_mac_mac1(mac1, sizeof(mac1), 1465 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1466 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1467 1468 /* 1469 * [W] 5.3: Denial of Service Mitigation & Cookies 1470 * "the responder, ..., must always reject messages with an invalid 1471 * msg.mac1" 1472 */ 1473 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1474 WG_DLOG("mac1 is invalid\n"); 1475 return; 1476 } 1477 1478 /* 1479 * [W] 5.4.2: First Message: Initiator to Responder 1480 * "When the responder receives this message, it does the same 1481 * operations so that its final state variables are identical, 1482 * replacing the operands of the DH function to produce equivalent 1483 * values." 1484 * Note that the following comments of operations are just copies of 1485 * the initiator's ones. 1486 */ 1487 1488 /* Ci := HASH(CONSTRUCTION) */ 1489 /* Hi := HASH(Ci || IDENTIFIER) */ 1490 wg_init_key_and_hash(ckey, hash); 1491 /* Hi := HASH(Hi || Sr^pub) */ 1492 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1493 1494 /* [N] 2.2: "e" */ 1495 /* Ci := KDF1(Ci, Ei^pub) */ 1496 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1497 sizeof(wgmi->wgmi_ephemeral)); 1498 /* Hi := HASH(Hi || msg.ephemeral) */ 1499 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1500 1501 WG_DUMP_HASH("ckey", ckey); 1502 1503 /* [N] 2.2: "es" */ 1504 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1505 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1506 1507 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1508 1509 /* [N] 2.2: "s" */ 1510 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1511 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1512 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1513 if (error != 0) { 1514 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1515 "%s: wg_algo_aead_dec for secret key failed\n", 1516 if_name(&wg->wg_if)); 1517 return; 1518 } 1519 /* Hi := HASH(Hi || msg.static) */ 1520 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1521 1522 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1523 if (wgp == NULL) { 1524 WG_DLOG("peer not found\n"); 1525 return; 1526 } 1527 1528 /* 1529 * Lock the peer to serialize access to cookie state. 1530 * 1531 * XXX Can we safely avoid holding the lock across DH? Take it 1532 * just to verify mac2 and then unlock/DH/lock? 1533 */ 1534 mutex_enter(wgp->wgp_lock); 1535 1536 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1537 WG_TRACE("under load"); 1538 /* 1539 * [W] 5.3: Denial of Service Mitigation & Cookies 1540 * "the responder, ..., and when under load may reject messages 1541 * with an invalid msg.mac2. If the responder receives a 1542 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1543 * and is under load, it may respond with a cookie reply 1544 * message" 1545 */ 1546 uint8_t zero[WG_MAC_LEN] = {0}; 1547 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1548 WG_TRACE("sending a cookie message: no cookie included"); 1549 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1550 wgmi->wgmi_mac1, src); 1551 goto out; 1552 } 1553 if (!wgp->wgp_last_sent_cookie_valid) { 1554 WG_TRACE("sending a cookie message: no cookie sent ever"); 1555 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1556 wgmi->wgmi_mac1, src); 1557 goto out; 1558 } 1559 uint8_t mac2[WG_MAC_LEN]; 1560 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1561 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1562 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1563 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1564 WG_DLOG("mac2 is invalid\n"); 1565 goto out; 1566 } 1567 WG_TRACE("under load, but continue to sending"); 1568 } 1569 1570 /* [N] 2.2: "ss" */ 1571 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1572 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1573 1574 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1575 wg_timestamp_t timestamp; 1576 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1577 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1578 hash, sizeof(hash)); 1579 if (error != 0) { 1580 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1581 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1582 if_name(&wg->wg_if), wgp->wgp_name); 1583 goto out; 1584 } 1585 /* Hi := HASH(Hi || msg.timestamp) */ 1586 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1587 1588 /* 1589 * [W] 5.1 "The responder keeps track of the greatest timestamp 1590 * received per peer and discards packets containing 1591 * timestamps less than or equal to it." 1592 */ 1593 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1594 sizeof(timestamp)); 1595 if (ret <= 0) { 1596 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1597 "%s: peer %s: invalid init msg: timestamp is old\n", 1598 if_name(&wg->wg_if), wgp->wgp_name); 1599 goto out; 1600 } 1601 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1602 1603 /* 1604 * Message is good -- we're committing to handle it now, unless 1605 * we were already initiating a session. 1606 */ 1607 wgs = wgp->wgp_session_unstable; 1608 switch (wgs->wgs_state) { 1609 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1610 wg_get_session_index(wg, wgs); 1611 break; 1612 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1613 WG_TRACE("Session already initializing, ignoring the message"); 1614 goto out; 1615 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1616 WG_TRACE("Session already initializing, destroying old states"); 1617 wg_clear_states(wgs); 1618 /* keep session index */ 1619 break; 1620 case WGS_STATE_ESTABLISHED: /* can't happen */ 1621 panic("unstable session can't be established"); 1622 break; 1623 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1624 WG_TRACE("Session destroying, but force to clear"); 1625 callout_stop(&wgp->wgp_session_dtor_timer); 1626 wg_clear_states(wgs); 1627 /* keep session index */ 1628 break; 1629 default: 1630 panic("invalid session state: %d", wgs->wgs_state); 1631 } 1632 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1633 1634 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1635 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1636 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1637 sizeof(wgmi->wgmi_ephemeral)); 1638 1639 wg_update_endpoint_if_necessary(wgp, src); 1640 1641 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1642 1643 wg_calculate_keys(wgs, false); 1644 wg_clear_states(wgs); 1645 1646 out: 1647 mutex_exit(wgp->wgp_lock); 1648 wg_put_peer(wgp, &psref_peer); 1649 } 1650 1651 static struct socket * 1652 wg_get_so_by_af(struct wg_softc *wg, const int af) 1653 { 1654 1655 switch (af) { 1656 #ifdef INET 1657 case AF_INET: 1658 return wg->wg_so4; 1659 #endif 1660 #ifdef INET6 1661 case AF_INET6: 1662 return wg->wg_so6; 1663 #endif 1664 default: 1665 panic("wg: no such af: %d", af); 1666 } 1667 } 1668 1669 static struct socket * 1670 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1671 { 1672 1673 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1674 } 1675 1676 static struct wg_sockaddr * 1677 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1678 { 1679 struct wg_sockaddr *wgsa; 1680 int s; 1681 1682 s = pserialize_read_enter(); 1683 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1684 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1685 pserialize_read_exit(s); 1686 1687 return wgsa; 1688 } 1689 1690 static void 1691 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1692 { 1693 1694 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1695 } 1696 1697 static int 1698 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1699 { 1700 int error; 1701 struct socket *so; 1702 struct psref psref; 1703 struct wg_sockaddr *wgsa; 1704 1705 wgsa = wg_get_endpoint_sa(wgp, &psref); 1706 so = wg_get_so_by_peer(wgp, wgsa); 1707 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1708 wg_put_sa(wgp, wgsa, &psref); 1709 1710 return error; 1711 } 1712 1713 static int 1714 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1715 { 1716 int error; 1717 struct mbuf *m; 1718 struct wg_msg_init *wgmi; 1719 struct wg_session *wgs; 1720 1721 KASSERT(mutex_owned(wgp->wgp_lock)); 1722 1723 wgs = wgp->wgp_session_unstable; 1724 /* XXX pull dispatch out into wg_task_send_init_message */ 1725 switch (wgs->wgs_state) { 1726 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1727 wg_get_session_index(wg, wgs); 1728 break; 1729 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1730 WG_TRACE("Session already initializing, skip starting new one"); 1731 return EBUSY; 1732 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1733 WG_TRACE("Session already initializing, destroying old states"); 1734 wg_clear_states(wgs); 1735 /* keep session index */ 1736 break; 1737 case WGS_STATE_ESTABLISHED: /* can't happen */ 1738 panic("unstable session can't be established"); 1739 break; 1740 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1741 WG_TRACE("Session destroying"); 1742 /* XXX should wait? */ 1743 return EBUSY; 1744 } 1745 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1746 1747 m = m_gethdr(M_WAIT, MT_DATA); 1748 if (sizeof(*wgmi) > MHLEN) { 1749 m_clget(m, M_WAIT); 1750 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1751 } 1752 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1753 wgmi = mtod(m, struct wg_msg_init *); 1754 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1755 1756 error = wg->wg_ops->send_hs_msg(wgp, m); 1757 if (error == 0) { 1758 WG_TRACE("init msg sent"); 1759 1760 if (wgp->wgp_handshake_start_time == 0) 1761 wgp->wgp_handshake_start_time = time_uptime; 1762 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1763 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1764 } else { 1765 wg_put_session_index(wg, wgs); 1766 /* Initiation failed; toss packet waiting for it if any. */ 1767 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 1768 m_freem(m); 1769 } 1770 1771 return error; 1772 } 1773 1774 static void 1775 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1776 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1777 const struct wg_msg_init *wgmi) 1778 { 1779 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1780 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1781 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1782 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1783 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1784 1785 KASSERT(mutex_owned(wgp->wgp_lock)); 1786 KASSERT(wgs == wgp->wgp_session_unstable); 1787 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1788 1789 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1790 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1791 1792 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1793 wgmr->wgmr_sender = wgs->wgs_local_index; 1794 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1795 1796 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1797 1798 /* [N] 2.2: "e" */ 1799 /* Er^priv, Er^pub := DH-GENERATE() */ 1800 wg_algo_generate_keypair(pubkey, privkey); 1801 /* Cr := KDF1(Cr, Er^pub) */ 1802 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1803 /* msg.ephemeral := Er^pub */ 1804 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1805 /* Hr := HASH(Hr || msg.ephemeral) */ 1806 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1807 1808 WG_DUMP_HASH("ckey", ckey); 1809 WG_DUMP_HASH("hash", hash); 1810 1811 /* [N] 2.2: "ee" */ 1812 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1813 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1814 1815 /* [N] 2.2: "se" */ 1816 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1817 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1818 1819 /* [N] 9.2: "psk" */ 1820 { 1821 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1822 /* Cr, r, k := KDF3(Cr, Q) */ 1823 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1824 sizeof(wgp->wgp_psk)); 1825 /* Hr := HASH(Hr || r) */ 1826 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1827 } 1828 1829 /* msg.empty := AEAD(k, 0, e, Hr) */ 1830 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1831 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1832 /* Hr := HASH(Hr || msg.empty) */ 1833 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1834 1835 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1836 1837 /* [W] 5.4.4: Cookie MACs */ 1838 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1839 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1840 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1841 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1842 /* Need mac1 to decrypt a cookie from a cookie message */ 1843 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1844 sizeof(wgp->wgp_last_sent_mac1)); 1845 wgp->wgp_last_sent_mac1_valid = true; 1846 1847 if (wgp->wgp_latest_cookie_time == 0 || 1848 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1849 /* msg.mac2 := 0^16 */ 1850 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1851 else { 1852 /* msg.mac2 := MAC(Lm, msg_b) */ 1853 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1854 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1855 (const uint8_t *)wgmr, 1856 offsetof(struct wg_msg_resp, wgmr_mac2), 1857 NULL, 0); 1858 } 1859 1860 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1861 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1862 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1863 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1864 wgs->wgs_remote_index = wgmi->wgmi_sender; 1865 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1866 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1867 } 1868 1869 static void 1870 wg_swap_sessions(struct wg_peer *wgp) 1871 { 1872 struct wg_session *wgs, *wgs_prev; 1873 1874 KASSERT(mutex_owned(wgp->wgp_lock)); 1875 1876 wgs = wgp->wgp_session_unstable; 1877 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1878 1879 wgs_prev = wgp->wgp_session_stable; 1880 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1881 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1882 atomic_store_release(&wgp->wgp_session_stable, wgs); 1883 wgp->wgp_session_unstable = wgs_prev; 1884 } 1885 1886 static void __noinline 1887 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1888 const struct sockaddr *src) 1889 { 1890 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1891 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1892 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1893 struct wg_peer *wgp; 1894 struct wg_session *wgs; 1895 struct psref psref; 1896 int error; 1897 uint8_t mac1[WG_MAC_LEN]; 1898 struct wg_session *wgs_prev; 1899 struct mbuf *m; 1900 1901 wg_algo_mac_mac1(mac1, sizeof(mac1), 1902 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1903 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1904 1905 /* 1906 * [W] 5.3: Denial of Service Mitigation & Cookies 1907 * "the responder, ..., must always reject messages with an invalid 1908 * msg.mac1" 1909 */ 1910 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1911 WG_DLOG("mac1 is invalid\n"); 1912 return; 1913 } 1914 1915 WG_TRACE("resp msg received"); 1916 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1917 if (wgs == NULL) { 1918 WG_TRACE("No session found"); 1919 return; 1920 } 1921 1922 wgp = wgs->wgs_peer; 1923 1924 mutex_enter(wgp->wgp_lock); 1925 1926 /* If we weren't waiting for a handshake response, drop it. */ 1927 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1928 WG_TRACE("peer sent spurious handshake response, ignoring"); 1929 goto out; 1930 } 1931 1932 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1933 WG_TRACE("under load"); 1934 /* 1935 * [W] 5.3: Denial of Service Mitigation & Cookies 1936 * "the responder, ..., and when under load may reject messages 1937 * with an invalid msg.mac2. If the responder receives a 1938 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1939 * and is under load, it may respond with a cookie reply 1940 * message" 1941 */ 1942 uint8_t zero[WG_MAC_LEN] = {0}; 1943 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1944 WG_TRACE("sending a cookie message: no cookie included"); 1945 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1946 wgmr->wgmr_mac1, src); 1947 goto out; 1948 } 1949 if (!wgp->wgp_last_sent_cookie_valid) { 1950 WG_TRACE("sending a cookie message: no cookie sent ever"); 1951 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1952 wgmr->wgmr_mac1, src); 1953 goto out; 1954 } 1955 uint8_t mac2[WG_MAC_LEN]; 1956 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1957 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1958 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1959 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1960 WG_DLOG("mac2 is invalid\n"); 1961 goto out; 1962 } 1963 WG_TRACE("under load, but continue to sending"); 1964 } 1965 1966 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1967 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1968 1969 /* 1970 * [W] 5.4.3 Second Message: Responder to Initiator 1971 * "When the initiator receives this message, it does the same 1972 * operations so that its final state variables are identical, 1973 * replacing the operands of the DH function to produce equivalent 1974 * values." 1975 * Note that the following comments of operations are just copies of 1976 * the initiator's ones. 1977 */ 1978 1979 /* [N] 2.2: "e" */ 1980 /* Cr := KDF1(Cr, Er^pub) */ 1981 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1982 sizeof(wgmr->wgmr_ephemeral)); 1983 /* Hr := HASH(Hr || msg.ephemeral) */ 1984 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1985 1986 WG_DUMP_HASH("ckey", ckey); 1987 WG_DUMP_HASH("hash", hash); 1988 1989 /* [N] 2.2: "ee" */ 1990 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1991 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1992 wgmr->wgmr_ephemeral); 1993 1994 /* [N] 2.2: "se" */ 1995 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1996 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1997 1998 /* [N] 9.2: "psk" */ 1999 { 2000 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 2001 /* Cr, r, k := KDF3(Cr, Q) */ 2002 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 2003 sizeof(wgp->wgp_psk)); 2004 /* Hr := HASH(Hr || r) */ 2005 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 2006 } 2007 2008 { 2009 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 2010 /* msg.empty := AEAD(k, 0, e, Hr) */ 2011 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 2012 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 2013 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 2014 if (error != 0) { 2015 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2016 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 2017 if_name(&wg->wg_if), wgp->wgp_name); 2018 goto out; 2019 } 2020 /* Hr := HASH(Hr || msg.empty) */ 2021 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 2022 } 2023 2024 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 2025 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 2026 wgs->wgs_remote_index = wgmr->wgmr_sender; 2027 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 2028 2029 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 2030 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2031 wgs->wgs_time_established = time_uptime; 2032 wgs->wgs_time_last_data_sent = 0; 2033 wgs->wgs_is_initiator = true; 2034 wg_calculate_keys(wgs, true); 2035 wg_clear_states(wgs); 2036 WG_TRACE("WGS_STATE_ESTABLISHED"); 2037 2038 callout_stop(&wgp->wgp_handshake_timeout_timer); 2039 2040 wg_swap_sessions(wgp); 2041 KASSERT(wgs == wgp->wgp_session_stable); 2042 wgs_prev = wgp->wgp_session_unstable; 2043 getnanotime(&wgp->wgp_last_handshake_time); 2044 wgp->wgp_handshake_start_time = 0; 2045 wgp->wgp_last_sent_mac1_valid = false; 2046 wgp->wgp_last_sent_cookie_valid = false; 2047 2048 wg_schedule_rekey_timer(wgp); 2049 2050 wg_update_endpoint_if_necessary(wgp, src); 2051 2052 /* 2053 * If we had a data packet queued up, send it; otherwise send a 2054 * keepalive message -- either way we have to send something 2055 * immediately or else the responder will never answer. 2056 */ 2057 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2058 kpreempt_disable(); 2059 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2060 M_SETCTX(m, wgp); 2061 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2062 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2063 if_name(&wg->wg_if)); 2064 m_freem(m); 2065 } 2066 kpreempt_enable(); 2067 } else { 2068 wg_send_keepalive_msg(wgp, wgs); 2069 } 2070 2071 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2072 /* Wait for wg_get_stable_session to drain. */ 2073 pserialize_perform(wgp->wgp_psz); 2074 2075 /* Transition ESTABLISHED->DESTROYING. */ 2076 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2077 2078 /* We can't destroy the old session immediately */ 2079 wg_schedule_session_dtor_timer(wgp); 2080 } else { 2081 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2082 "state=%d", wgs_prev->wgs_state); 2083 } 2084 2085 out: 2086 mutex_exit(wgp->wgp_lock); 2087 wg_put_session(wgs, &psref); 2088 } 2089 2090 static int 2091 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2092 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2093 { 2094 int error; 2095 struct mbuf *m; 2096 struct wg_msg_resp *wgmr; 2097 2098 KASSERT(mutex_owned(wgp->wgp_lock)); 2099 KASSERT(wgs == wgp->wgp_session_unstable); 2100 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 2101 2102 m = m_gethdr(M_WAIT, MT_DATA); 2103 if (sizeof(*wgmr) > MHLEN) { 2104 m_clget(m, M_WAIT); 2105 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2106 } 2107 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2108 wgmr = mtod(m, struct wg_msg_resp *); 2109 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2110 2111 error = wg->wg_ops->send_hs_msg(wgp, m); 2112 if (error == 0) 2113 WG_TRACE("resp msg sent"); 2114 return error; 2115 } 2116 2117 static struct wg_peer * 2118 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2119 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2120 { 2121 struct wg_peer *wgp; 2122 2123 int s = pserialize_read_enter(); 2124 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2125 if (wgp != NULL) 2126 wg_get_peer(wgp, psref); 2127 pserialize_read_exit(s); 2128 2129 return wgp; 2130 } 2131 2132 static void 2133 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2134 struct wg_msg_cookie *wgmc, const uint32_t sender, 2135 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2136 { 2137 uint8_t cookie[WG_COOKIE_LEN]; 2138 uint8_t key[WG_HASH_LEN]; 2139 uint8_t addr[sizeof(struct in6_addr)]; 2140 size_t addrlen; 2141 uint16_t uh_sport; /* be */ 2142 2143 KASSERT(mutex_owned(wgp->wgp_lock)); 2144 2145 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2146 wgmc->wgmc_receiver = sender; 2147 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2148 2149 /* 2150 * [W] 5.4.7: Under Load: Cookie Reply Message 2151 * "The secret variable, Rm, changes every two minutes to a 2152 * random value" 2153 */ 2154 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2155 wgp->wgp_randval = cprng_strong32(); 2156 wgp->wgp_last_genrandval_time = time_uptime; 2157 } 2158 2159 switch (src->sa_family) { 2160 case AF_INET: { 2161 const struct sockaddr_in *sin = satocsin(src); 2162 addrlen = sizeof(sin->sin_addr); 2163 memcpy(addr, &sin->sin_addr, addrlen); 2164 uh_sport = sin->sin_port; 2165 break; 2166 } 2167 #ifdef INET6 2168 case AF_INET6: { 2169 const struct sockaddr_in6 *sin6 = satocsin6(src); 2170 addrlen = sizeof(sin6->sin6_addr); 2171 memcpy(addr, &sin6->sin6_addr, addrlen); 2172 uh_sport = sin6->sin6_port; 2173 break; 2174 } 2175 #endif 2176 default: 2177 panic("invalid af=%d", src->sa_family); 2178 } 2179 2180 wg_algo_mac(cookie, sizeof(cookie), 2181 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2182 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2183 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2184 sizeof(wg->wg_pubkey)); 2185 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2186 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2187 2188 /* Need to store to calculate mac2 */ 2189 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2190 wgp->wgp_last_sent_cookie_valid = true; 2191 } 2192 2193 static int 2194 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2195 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2196 const struct sockaddr *src) 2197 { 2198 int error; 2199 struct mbuf *m; 2200 struct wg_msg_cookie *wgmc; 2201 2202 KASSERT(mutex_owned(wgp->wgp_lock)); 2203 2204 m = m_gethdr(M_WAIT, MT_DATA); 2205 if (sizeof(*wgmc) > MHLEN) { 2206 m_clget(m, M_WAIT); 2207 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2208 } 2209 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2210 wgmc = mtod(m, struct wg_msg_cookie *); 2211 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2212 2213 error = wg->wg_ops->send_hs_msg(wgp, m); 2214 if (error == 0) 2215 WG_TRACE("cookie msg sent"); 2216 return error; 2217 } 2218 2219 static bool 2220 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2221 { 2222 #ifdef WG_DEBUG_PARAMS 2223 if (wg_force_underload) 2224 return true; 2225 #endif 2226 2227 /* 2228 * XXX we don't have a means of a load estimation. The purpose of 2229 * the mechanism is a DoS mitigation, so we consider frequent handshake 2230 * messages as (a kind of) load; if a message of the same type comes 2231 * to a peer within 1 second, we consider we are under load. 2232 */ 2233 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2234 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2235 return (time_uptime - last) == 0; 2236 } 2237 2238 static void 2239 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2240 { 2241 2242 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2243 2244 /* 2245 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2246 */ 2247 if (initiator) { 2248 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2249 wgs->wgs_chaining_key, NULL, 0); 2250 } else { 2251 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2252 wgs->wgs_chaining_key, NULL, 0); 2253 } 2254 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2255 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2256 } 2257 2258 static uint64_t 2259 wg_session_get_send_counter(struct wg_session *wgs) 2260 { 2261 #ifdef __HAVE_ATOMIC64_LOADSTORE 2262 return atomic_load_relaxed(&wgs->wgs_send_counter); 2263 #else 2264 uint64_t send_counter; 2265 2266 mutex_enter(&wgs->wgs_send_counter_lock); 2267 send_counter = wgs->wgs_send_counter; 2268 mutex_exit(&wgs->wgs_send_counter_lock); 2269 2270 return send_counter; 2271 #endif 2272 } 2273 2274 static uint64_t 2275 wg_session_inc_send_counter(struct wg_session *wgs) 2276 { 2277 #ifdef __HAVE_ATOMIC64_LOADSTORE 2278 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2279 #else 2280 uint64_t send_counter; 2281 2282 mutex_enter(&wgs->wgs_send_counter_lock); 2283 send_counter = wgs->wgs_send_counter++; 2284 mutex_exit(&wgs->wgs_send_counter_lock); 2285 2286 return send_counter; 2287 #endif 2288 } 2289 2290 static void 2291 wg_clear_states(struct wg_session *wgs) 2292 { 2293 2294 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2295 2296 wgs->wgs_send_counter = 0; 2297 sliwin_reset(&wgs->wgs_recvwin->window); 2298 2299 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2300 wgs_clear(handshake_hash); 2301 wgs_clear(chaining_key); 2302 wgs_clear(ephemeral_key_pub); 2303 wgs_clear(ephemeral_key_priv); 2304 wgs_clear(ephemeral_key_peer); 2305 #undef wgs_clear 2306 } 2307 2308 static struct wg_session * 2309 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2310 struct psref *psref) 2311 { 2312 struct wg_session *wgs; 2313 2314 int s = pserialize_read_enter(); 2315 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2316 if (wgs != NULL) { 2317 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2318 WGS_STATE_UNKNOWN); 2319 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2320 } 2321 pserialize_read_exit(s); 2322 2323 return wgs; 2324 } 2325 2326 static void 2327 wg_schedule_rekey_timer(struct wg_peer *wgp) 2328 { 2329 int timeout = MIN(wg_rekey_after_time, (unsigned)(INT_MAX / hz)); 2330 2331 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2332 } 2333 2334 static void 2335 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2336 { 2337 struct mbuf *m; 2338 2339 /* 2340 * [W] 6.5 Passive Keepalive 2341 * "A keepalive message is simply a transport data message with 2342 * a zero-length encapsulated encrypted inner-packet." 2343 */ 2344 WG_TRACE(""); 2345 m = m_gethdr(M_WAIT, MT_DATA); 2346 wg_send_data_msg(wgp, wgs, m); 2347 } 2348 2349 static bool 2350 wg_need_to_send_init_message(struct wg_session *wgs) 2351 { 2352 /* 2353 * [W] 6.2 Transport Message Limits 2354 * "if a peer is the initiator of a current secure session, 2355 * WireGuard will send a handshake initiation message to begin 2356 * a new secure session ... if after receiving a transport data 2357 * message, the current secure session is (REJECT-AFTER-TIME 2358 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2359 * not yet acted upon this event." 2360 */ 2361 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2362 (time_uptime - wgs->wgs_time_established) >= 2363 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2364 } 2365 2366 static void 2367 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2368 { 2369 2370 mutex_enter(wgp->wgp_intr_lock); 2371 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2372 if (wgp->wgp_tasks == 0) 2373 /* 2374 * XXX If the current CPU is already loaded -- e.g., if 2375 * there's already a bunch of handshakes queued up -- 2376 * consider tossing this over to another CPU to 2377 * distribute the load. 2378 */ 2379 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2380 wgp->wgp_tasks |= task; 2381 mutex_exit(wgp->wgp_intr_lock); 2382 } 2383 2384 static void 2385 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2386 { 2387 struct wg_sockaddr *wgsa_prev; 2388 2389 WG_TRACE("Changing endpoint"); 2390 2391 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2392 wgsa_prev = wgp->wgp_endpoint; 2393 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2394 wgp->wgp_endpoint0 = wgsa_prev; 2395 atomic_store_release(&wgp->wgp_endpoint_available, true); 2396 2397 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2398 } 2399 2400 static bool 2401 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2402 { 2403 uint16_t packet_len; 2404 const struct ip *ip; 2405 2406 if (__predict_false(decrypted_len < sizeof(*ip))) { 2407 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2408 sizeof(*ip)); 2409 return false; 2410 } 2411 2412 ip = (const struct ip *)packet; 2413 if (ip->ip_v == 4) 2414 *af = AF_INET; 2415 else if (ip->ip_v == 6) 2416 *af = AF_INET6; 2417 else { 2418 WG_DLOG("ip_v=%d\n", ip->ip_v); 2419 return false; 2420 } 2421 2422 WG_DLOG("af=%d\n", *af); 2423 2424 switch (*af) { 2425 #ifdef INET 2426 case AF_INET: 2427 packet_len = ntohs(ip->ip_len); 2428 break; 2429 #endif 2430 #ifdef INET6 2431 case AF_INET6: { 2432 const struct ip6_hdr *ip6; 2433 2434 if (__predict_false(decrypted_len < sizeof(*ip6))) { 2435 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2436 sizeof(*ip6)); 2437 return false; 2438 } 2439 2440 ip6 = (const struct ip6_hdr *)packet; 2441 packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); 2442 break; 2443 } 2444 #endif 2445 default: 2446 return false; 2447 } 2448 2449 if (packet_len > decrypted_len) { 2450 WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len, 2451 decrypted_len); 2452 return false; 2453 } 2454 2455 return true; 2456 } 2457 2458 static bool 2459 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2460 int af, char *packet) 2461 { 2462 struct sockaddr_storage ss; 2463 struct sockaddr *sa; 2464 struct psref psref; 2465 struct wg_peer *wgp; 2466 bool ok; 2467 2468 /* 2469 * II CRYPTOKEY ROUTING 2470 * "it will only accept it if its source IP resolves in the 2471 * table to the public key used in the secure session for 2472 * decrypting it." 2473 */ 2474 2475 if (af == AF_INET) { 2476 const struct ip *ip = (const struct ip *)packet; 2477 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2478 sockaddr_in_init(sin, &ip->ip_src, 0); 2479 sa = sintosa(sin); 2480 #ifdef INET6 2481 } else { 2482 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2483 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2484 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2485 sa = sin6tosa(sin6); 2486 #endif 2487 } 2488 2489 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2490 ok = (wgp == wgp_expected); 2491 if (wgp != NULL) 2492 wg_put_peer(wgp, &psref); 2493 2494 return ok; 2495 } 2496 2497 static void 2498 wg_session_dtor_timer(void *arg) 2499 { 2500 struct wg_peer *wgp = arg; 2501 2502 WG_TRACE("enter"); 2503 2504 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2505 } 2506 2507 static void 2508 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2509 { 2510 2511 /* 1 second grace period */ 2512 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2513 } 2514 2515 static bool 2516 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2517 { 2518 if (sa1->sa_family != sa2->sa_family) 2519 return false; 2520 2521 switch (sa1->sa_family) { 2522 #ifdef INET 2523 case AF_INET: 2524 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2525 #endif 2526 #ifdef INET6 2527 case AF_INET6: 2528 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2529 #endif 2530 default: 2531 return false; 2532 } 2533 } 2534 2535 static void 2536 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2537 const struct sockaddr *src) 2538 { 2539 struct wg_sockaddr *wgsa; 2540 struct psref psref; 2541 2542 wgsa = wg_get_endpoint_sa(wgp, &psref); 2543 2544 #ifdef WG_DEBUG_LOG 2545 char oldaddr[128], newaddr[128]; 2546 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2547 sockaddr_format(src, newaddr, sizeof(newaddr)); 2548 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2549 #endif 2550 2551 /* 2552 * III: "Since the packet has authenticated correctly, the source IP of 2553 * the outer UDP/IP packet is used to update the endpoint for peer..." 2554 */ 2555 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2556 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2557 /* XXX We can't change the endpoint twice in a short period */ 2558 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2559 wg_change_endpoint(wgp, src); 2560 } 2561 } 2562 2563 wg_put_sa(wgp, wgsa, &psref); 2564 } 2565 2566 static void __noinline 2567 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2568 const struct sockaddr *src) 2569 { 2570 struct wg_msg_data *wgmd; 2571 char *encrypted_buf = NULL, *decrypted_buf; 2572 size_t encrypted_len, decrypted_len; 2573 struct wg_session *wgs; 2574 struct wg_peer *wgp; 2575 int state; 2576 size_t mlen; 2577 struct psref psref; 2578 int error, af; 2579 bool success, free_encrypted_buf = false, ok; 2580 struct mbuf *n; 2581 2582 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2583 wgmd = mtod(m, struct wg_msg_data *); 2584 2585 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2586 WG_TRACE("data"); 2587 2588 /* Find the putative session, or drop. */ 2589 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2590 if (wgs == NULL) { 2591 WG_TRACE("No session found"); 2592 m_freem(m); 2593 return; 2594 } 2595 2596 /* 2597 * We are only ready to handle data when in INIT_PASSIVE, 2598 * ESTABLISHED, or DESTROYING. All transitions out of that 2599 * state dissociate the session index and drain psrefs. 2600 */ 2601 state = atomic_load_relaxed(&wgs->wgs_state); 2602 switch (state) { 2603 case WGS_STATE_UNKNOWN: 2604 panic("wg session %p in unknown state has session index %u", 2605 wgs, wgmd->wgmd_receiver); 2606 case WGS_STATE_INIT_ACTIVE: 2607 WG_TRACE("not yet ready for data"); 2608 goto out; 2609 case WGS_STATE_INIT_PASSIVE: 2610 case WGS_STATE_ESTABLISHED: 2611 case WGS_STATE_DESTROYING: 2612 break; 2613 } 2614 2615 /* 2616 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2617 * to update the endpoint if authentication succeeds. 2618 */ 2619 wgp = wgs->wgs_peer; 2620 2621 /* 2622 * Reject outrageously wrong sequence numbers before doing any 2623 * crypto work or taking any locks. 2624 */ 2625 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2626 le64toh(wgmd->wgmd_counter)); 2627 if (error) { 2628 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2629 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2630 if_name(&wg->wg_if), wgp->wgp_name, 2631 le64toh(wgmd->wgmd_counter)); 2632 goto out; 2633 } 2634 2635 /* Ensure the payload and authenticator are contiguous. */ 2636 mlen = m_length(m); 2637 encrypted_len = mlen - sizeof(*wgmd); 2638 if (encrypted_len < WG_AUTHTAG_LEN) { 2639 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2640 goto out; 2641 } 2642 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2643 if (success) { 2644 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2645 } else { 2646 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2647 if (encrypted_buf == NULL) { 2648 WG_DLOG("failed to allocate encrypted_buf\n"); 2649 goto out; 2650 } 2651 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2652 free_encrypted_buf = true; 2653 } 2654 /* m_ensure_contig may change m regardless of its result */ 2655 KASSERT(m->m_len >= sizeof(*wgmd)); 2656 wgmd = mtod(m, struct wg_msg_data *); 2657 2658 /* 2659 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2660 * a zero-length buffer (XXX). Drop if plaintext is longer 2661 * than MCLBYTES (XXX). 2662 */ 2663 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2664 if (decrypted_len > MCLBYTES) { 2665 /* FIXME handle larger data than MCLBYTES */ 2666 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2667 goto out; 2668 } 2669 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2670 if (n == NULL) { 2671 WG_DLOG("wg_get_mbuf failed\n"); 2672 goto out; 2673 } 2674 decrypted_buf = mtod(n, char *); 2675 2676 /* Decrypt and verify the packet. */ 2677 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2678 error = wg_algo_aead_dec(decrypted_buf, 2679 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2680 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2681 encrypted_len, NULL, 0); 2682 if (error != 0) { 2683 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2684 "%s: peer %s: failed to wg_algo_aead_dec\n", 2685 if_name(&wg->wg_if), wgp->wgp_name); 2686 m_freem(n); 2687 goto out; 2688 } 2689 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2690 2691 /* Packet is genuine. Reject it if a replay or just too old. */ 2692 mutex_enter(&wgs->wgs_recvwin->lock); 2693 error = sliwin_update(&wgs->wgs_recvwin->window, 2694 le64toh(wgmd->wgmd_counter)); 2695 mutex_exit(&wgs->wgs_recvwin->lock); 2696 if (error) { 2697 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2698 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2699 if_name(&wg->wg_if), wgp->wgp_name, 2700 le64toh(wgmd->wgmd_counter)); 2701 m_freem(n); 2702 goto out; 2703 } 2704 2705 #ifdef WG_DEBUG_PACKET 2706 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2707 char *hex = gethexdump(wgmd, sizeof(*wgmd)); 2708 if (hex != NULL) { 2709 log(LOG_DEBUG, "wgmd=%p, sizeof(*wgmd)=%zu\n%s\n", 2710 wgmd, sizeof(*wgmd), hex); 2711 puthexdump(hex, wgmd, sizeof(*wgmd)); 2712 } 2713 hex = gethexdump(decrypted_buf, decrypted_len); 2714 if (hex != NULL) { 2715 log(LOG_DEBUG, "decrypted_buf=%p, " 2716 "decrypted_len=%zu\n%s\n", 2717 decrypted_buf, decrypted_len, hex); 2718 puthexdump(hex, decrypted_buf, decrypted_len); 2719 } 2720 } 2721 #endif 2722 /* We're done with m now; free it and chuck the pointers. */ 2723 m_freem(m); 2724 m = NULL; 2725 wgmd = NULL; 2726 2727 /* 2728 * Validate the encapsulated packet header and get the address 2729 * family, or drop. 2730 */ 2731 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2732 if (!ok) { 2733 m_freem(n); 2734 goto out; 2735 } 2736 2737 /* 2738 * The packet is genuine. Update the peer's endpoint if the 2739 * source address changed. 2740 * 2741 * XXX How to prevent DoS by replaying genuine packets from the 2742 * wrong source address? 2743 */ 2744 wg_update_endpoint_if_necessary(wgp, src); 2745 2746 /* Submit it into our network stack if routable. */ 2747 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2748 if (ok) { 2749 wg->wg_ops->input(&wg->wg_if, n, af); 2750 } else { 2751 char addrstr[INET6_ADDRSTRLEN]; 2752 memset(addrstr, 0, sizeof(addrstr)); 2753 if (af == AF_INET) { 2754 const struct ip *ip = (const struct ip *)decrypted_buf; 2755 IN_PRINT(addrstr, &ip->ip_src); 2756 #ifdef INET6 2757 } else if (af == AF_INET6) { 2758 const struct ip6_hdr *ip6 = 2759 (const struct ip6_hdr *)decrypted_buf; 2760 IN6_PRINT(addrstr, &ip6->ip6_src); 2761 #endif 2762 } 2763 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2764 "%s: peer %s: invalid source address (%s)\n", 2765 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2766 m_freem(n); 2767 /* 2768 * The inner address is invalid however the session is valid 2769 * so continue the session processing below. 2770 */ 2771 } 2772 n = NULL; 2773 2774 /* Update the state machine if necessary. */ 2775 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2776 /* 2777 * We were waiting for the initiator to send their 2778 * first data transport message, and that has happened. 2779 * Schedule a task to establish this session. 2780 */ 2781 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2782 } else { 2783 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2784 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2785 } 2786 /* 2787 * [W] 6.5 Passive Keepalive 2788 * "If a peer has received a validly-authenticated transport 2789 * data message (section 5.4.6), but does not have any packets 2790 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2791 * a keepalive message." 2792 */ 2793 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2794 (uintmax_t)time_uptime, 2795 (uintmax_t)wgs->wgs_time_last_data_sent); 2796 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2797 wg_keepalive_timeout) { 2798 WG_TRACE("Schedule sending keepalive message"); 2799 /* 2800 * We can't send a keepalive message here to avoid 2801 * a deadlock; we already hold the solock of a socket 2802 * that is used to send the message. 2803 */ 2804 wg_schedule_peer_task(wgp, 2805 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2806 } 2807 } 2808 out: 2809 wg_put_session(wgs, &psref); 2810 m_freem(m); 2811 if (free_encrypted_buf) 2812 kmem_intr_free(encrypted_buf, encrypted_len); 2813 } 2814 2815 static void __noinline 2816 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2817 { 2818 struct wg_session *wgs; 2819 struct wg_peer *wgp; 2820 struct psref psref; 2821 int error; 2822 uint8_t key[WG_HASH_LEN]; 2823 uint8_t cookie[WG_COOKIE_LEN]; 2824 2825 WG_TRACE("cookie msg received"); 2826 2827 /* Find the putative session. */ 2828 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2829 if (wgs == NULL) { 2830 WG_TRACE("No session found"); 2831 return; 2832 } 2833 2834 /* Lock the peer so we can update the cookie state. */ 2835 wgp = wgs->wgs_peer; 2836 mutex_enter(wgp->wgp_lock); 2837 2838 if (!wgp->wgp_last_sent_mac1_valid) { 2839 WG_TRACE("No valid mac1 sent (or expired)"); 2840 goto out; 2841 } 2842 2843 /* Decrypt the cookie and store it for later handshake retry. */ 2844 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2845 sizeof(wgp->wgp_pubkey)); 2846 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2847 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2848 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2849 wgmc->wgmc_salt); 2850 if (error != 0) { 2851 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2852 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 2853 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 2854 goto out; 2855 } 2856 /* 2857 * [W] 6.6: Interaction with Cookie Reply System 2858 * "it should simply store the decrypted cookie value from the cookie 2859 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2860 * timer for retrying a handshake initiation message." 2861 */ 2862 wgp->wgp_latest_cookie_time = time_uptime; 2863 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2864 out: 2865 mutex_exit(wgp->wgp_lock); 2866 wg_put_session(wgs, &psref); 2867 } 2868 2869 static struct mbuf * 2870 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2871 { 2872 struct wg_msg wgm; 2873 size_t mbuflen; 2874 size_t msglen; 2875 2876 /* 2877 * Get the mbuf chain length. It is already guaranteed, by 2878 * wg_overudp_cb, to be large enough for a struct wg_msg. 2879 */ 2880 mbuflen = m_length(m); 2881 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2882 2883 /* 2884 * Copy the message header (32-bit message type) out -- we'll 2885 * worry about contiguity and alignment later. 2886 */ 2887 m_copydata(m, 0, sizeof(wgm), &wgm); 2888 switch (le32toh(wgm.wgm_type)) { 2889 case WG_MSG_TYPE_INIT: 2890 msglen = sizeof(struct wg_msg_init); 2891 break; 2892 case WG_MSG_TYPE_RESP: 2893 msglen = sizeof(struct wg_msg_resp); 2894 break; 2895 case WG_MSG_TYPE_COOKIE: 2896 msglen = sizeof(struct wg_msg_cookie); 2897 break; 2898 case WG_MSG_TYPE_DATA: 2899 msglen = sizeof(struct wg_msg_data); 2900 break; 2901 default: 2902 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2903 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 2904 le32toh(wgm.wgm_type)); 2905 goto error; 2906 } 2907 2908 /* Verify the mbuf chain is long enough for this type of message. */ 2909 if (__predict_false(mbuflen < msglen)) { 2910 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2911 le32toh(wgm.wgm_type)); 2912 goto error; 2913 } 2914 2915 /* Make the message header contiguous if necessary. */ 2916 if (__predict_false(m->m_len < msglen)) { 2917 m = m_pullup(m, msglen); 2918 if (m == NULL) 2919 return NULL; 2920 } 2921 2922 return m; 2923 2924 error: 2925 m_freem(m); 2926 return NULL; 2927 } 2928 2929 static void 2930 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2931 const struct sockaddr *src) 2932 { 2933 struct wg_msg *wgm; 2934 2935 KASSERT(curlwp->l_pflag & LP_BOUND); 2936 2937 m = wg_validate_msg_header(wg, m); 2938 if (__predict_false(m == NULL)) 2939 return; 2940 2941 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2942 wgm = mtod(m, struct wg_msg *); 2943 switch (le32toh(wgm->wgm_type)) { 2944 case WG_MSG_TYPE_INIT: 2945 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2946 break; 2947 case WG_MSG_TYPE_RESP: 2948 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2949 break; 2950 case WG_MSG_TYPE_COOKIE: 2951 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2952 break; 2953 case WG_MSG_TYPE_DATA: 2954 wg_handle_msg_data(wg, m, src); 2955 /* wg_handle_msg_data frees m for us */ 2956 return; 2957 default: 2958 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2959 } 2960 2961 m_freem(m); 2962 } 2963 2964 static void 2965 wg_receive_packets(struct wg_softc *wg, const int af) 2966 { 2967 2968 for (;;) { 2969 int error, flags; 2970 struct socket *so; 2971 struct mbuf *m = NULL; 2972 struct uio dummy_uio; 2973 struct mbuf *paddr = NULL; 2974 struct sockaddr *src; 2975 2976 so = wg_get_so_by_af(wg, af); 2977 flags = MSG_DONTWAIT; 2978 dummy_uio.uio_resid = 1000000000; 2979 2980 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2981 &flags); 2982 if (error || m == NULL) { 2983 //if (error == EWOULDBLOCK) 2984 return; 2985 } 2986 2987 KASSERT(paddr != NULL); 2988 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2989 src = mtod(paddr, struct sockaddr *); 2990 2991 wg_handle_packet(wg, m, src); 2992 } 2993 } 2994 2995 static void 2996 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2997 { 2998 2999 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 3000 } 3001 3002 static void 3003 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 3004 { 3005 3006 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 3007 } 3008 3009 static void 3010 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 3011 { 3012 struct wg_session *wgs; 3013 3014 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 3015 3016 KASSERT(mutex_owned(wgp->wgp_lock)); 3017 3018 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 3019 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 3020 if_name(&wg->wg_if)); 3021 /* XXX should do something? */ 3022 return; 3023 } 3024 3025 wgs = wgp->wgp_session_stable; 3026 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 3027 /* XXX What if the unstable session is already INIT_ACTIVE? */ 3028 wg_send_handshake_msg_init(wg, wgp); 3029 } else { 3030 /* rekey */ 3031 wgs = wgp->wgp_session_unstable; 3032 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3033 wg_send_handshake_msg_init(wg, wgp); 3034 } 3035 } 3036 3037 static void 3038 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 3039 { 3040 struct wg_session *wgs; 3041 3042 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 3043 3044 KASSERT(mutex_owned(wgp->wgp_lock)); 3045 KASSERT(wgp->wgp_handshake_start_time != 0); 3046 3047 wgs = wgp->wgp_session_unstable; 3048 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3049 return; 3050 3051 /* 3052 * XXX no real need to assign a new index here, but we do need 3053 * to transition to UNKNOWN temporarily 3054 */ 3055 wg_put_session_index(wg, wgs); 3056 3057 /* [W] 6.4 Handshake Initiation Retransmission */ 3058 if ((time_uptime - wgp->wgp_handshake_start_time) > 3059 wg_rekey_attempt_time) { 3060 /* Give up handshaking */ 3061 wgp->wgp_handshake_start_time = 0; 3062 WG_TRACE("give up"); 3063 3064 /* 3065 * If a new data packet comes, handshaking will be retried 3066 * and a new session would be established at that time, 3067 * however we don't want to send pending packets then. 3068 */ 3069 wg_purge_pending_packets(wgp); 3070 return; 3071 } 3072 3073 wg_task_send_init_message(wg, wgp); 3074 } 3075 3076 static void 3077 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3078 { 3079 struct wg_session *wgs, *wgs_prev; 3080 struct mbuf *m; 3081 3082 KASSERT(mutex_owned(wgp->wgp_lock)); 3083 3084 wgs = wgp->wgp_session_unstable; 3085 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3086 /* XXX Can this happen? */ 3087 return; 3088 3089 wgs->wgs_state = WGS_STATE_ESTABLISHED; 3090 wgs->wgs_time_established = time_uptime; 3091 wgs->wgs_time_last_data_sent = 0; 3092 wgs->wgs_is_initiator = false; 3093 WG_TRACE("WGS_STATE_ESTABLISHED"); 3094 3095 wg_swap_sessions(wgp); 3096 KASSERT(wgs == wgp->wgp_session_stable); 3097 wgs_prev = wgp->wgp_session_unstable; 3098 getnanotime(&wgp->wgp_last_handshake_time); 3099 wgp->wgp_handshake_start_time = 0; 3100 wgp->wgp_last_sent_mac1_valid = false; 3101 wgp->wgp_last_sent_cookie_valid = false; 3102 3103 /* If we had a data packet queued up, send it. */ 3104 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3105 kpreempt_disable(); 3106 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3107 M_SETCTX(m, wgp); 3108 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3109 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3110 if_name(&wg->wg_if)); 3111 m_freem(m); 3112 } 3113 kpreempt_enable(); 3114 } 3115 3116 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3117 /* Wait for wg_get_stable_session to drain. */ 3118 pserialize_perform(wgp->wgp_psz); 3119 3120 /* Transition ESTABLISHED->DESTROYING. */ 3121 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 3122 3123 /* We can't destroy the old session immediately */ 3124 wg_schedule_session_dtor_timer(wgp); 3125 } else { 3126 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3127 "state=%d", wgs_prev->wgs_state); 3128 wg_clear_states(wgs_prev); 3129 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3130 } 3131 } 3132 3133 static void 3134 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3135 { 3136 3137 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3138 3139 KASSERT(mutex_owned(wgp->wgp_lock)); 3140 3141 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3142 pserialize_perform(wgp->wgp_psz); 3143 mutex_exit(wgp->wgp_lock); 3144 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3145 wg_psref_class); 3146 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3147 wg_psref_class); 3148 mutex_enter(wgp->wgp_lock); 3149 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3150 } 3151 } 3152 3153 static void 3154 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3155 { 3156 struct wg_session *wgs; 3157 3158 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3159 3160 KASSERT(mutex_owned(wgp->wgp_lock)); 3161 3162 wgs = wgp->wgp_session_stable; 3163 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3164 return; 3165 3166 wg_send_keepalive_msg(wgp, wgs); 3167 } 3168 3169 static void 3170 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3171 { 3172 struct wg_session *wgs; 3173 3174 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3175 3176 KASSERT(mutex_owned(wgp->wgp_lock)); 3177 3178 wgs = wgp->wgp_session_unstable; 3179 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 3180 wg_put_session_index(wg, wgs); 3181 } 3182 } 3183 3184 static void 3185 wg_peer_work(struct work *wk, void *cookie) 3186 { 3187 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3188 struct wg_softc *wg = wgp->wgp_sc; 3189 unsigned int tasks; 3190 3191 mutex_enter(wgp->wgp_intr_lock); 3192 while ((tasks = wgp->wgp_tasks) != 0) { 3193 wgp->wgp_tasks = 0; 3194 mutex_exit(wgp->wgp_intr_lock); 3195 3196 mutex_enter(wgp->wgp_lock); 3197 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3198 wg_task_send_init_message(wg, wgp); 3199 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3200 wg_task_retry_handshake(wg, wgp); 3201 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3202 wg_task_establish_session(wg, wgp); 3203 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3204 wg_task_endpoint_changed(wg, wgp); 3205 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3206 wg_task_send_keepalive_message(wg, wgp); 3207 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3208 wg_task_destroy_prev_session(wg, wgp); 3209 mutex_exit(wgp->wgp_lock); 3210 3211 mutex_enter(wgp->wgp_intr_lock); 3212 } 3213 mutex_exit(wgp->wgp_intr_lock); 3214 } 3215 3216 static void 3217 wg_job(struct threadpool_job *job) 3218 { 3219 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3220 int bound, upcalls; 3221 3222 mutex_enter(wg->wg_intr_lock); 3223 while ((upcalls = wg->wg_upcalls) != 0) { 3224 wg->wg_upcalls = 0; 3225 mutex_exit(wg->wg_intr_lock); 3226 bound = curlwp_bind(); 3227 if (ISSET(upcalls, WG_UPCALL_INET)) 3228 wg_receive_packets(wg, AF_INET); 3229 if (ISSET(upcalls, WG_UPCALL_INET6)) 3230 wg_receive_packets(wg, AF_INET6); 3231 curlwp_bindx(bound); 3232 mutex_enter(wg->wg_intr_lock); 3233 } 3234 threadpool_job_done(job); 3235 mutex_exit(wg->wg_intr_lock); 3236 } 3237 3238 static int 3239 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3240 { 3241 int error; 3242 uint16_t old_port = wg->wg_listen_port; 3243 3244 if (port != 0 && old_port == port) 3245 return 0; 3246 3247 struct sockaddr_in _sin, *sin = &_sin; 3248 sin->sin_len = sizeof(*sin); 3249 sin->sin_family = AF_INET; 3250 sin->sin_addr.s_addr = INADDR_ANY; 3251 sin->sin_port = htons(port); 3252 3253 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3254 if (error != 0) 3255 return error; 3256 3257 #ifdef INET6 3258 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3259 sin6->sin6_len = sizeof(*sin6); 3260 sin6->sin6_family = AF_INET6; 3261 sin6->sin6_addr = in6addr_any; 3262 sin6->sin6_port = htons(port); 3263 3264 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3265 if (error != 0) 3266 return error; 3267 #endif 3268 3269 wg->wg_listen_port = port; 3270 3271 return 0; 3272 } 3273 3274 static void 3275 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3276 { 3277 struct wg_softc *wg = cookie; 3278 int reason; 3279 3280 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3281 WG_UPCALL_INET : 3282 WG_UPCALL_INET6; 3283 3284 mutex_enter(wg->wg_intr_lock); 3285 wg->wg_upcalls |= reason; 3286 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3287 mutex_exit(wg->wg_intr_lock); 3288 } 3289 3290 static int 3291 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3292 struct sockaddr *src, void *arg) 3293 { 3294 struct wg_softc *wg = arg; 3295 struct wg_msg wgm; 3296 struct mbuf *m = *mp; 3297 3298 WG_TRACE("enter"); 3299 3300 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3301 KASSERT(offset <= m_length(m)); 3302 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3303 /* drop on the floor */ 3304 m_freem(m); 3305 return -1; 3306 } 3307 3308 /* 3309 * Copy the message header (32-bit message type) out -- we'll 3310 * worry about contiguity and alignment later. 3311 */ 3312 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3313 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3314 3315 /* 3316 * Handle DATA packets promptly as they arrive. Other packets 3317 * may require expensive public-key crypto and are not as 3318 * sensitive to latency, so defer them to the worker thread. 3319 */ 3320 switch (le32toh(wgm.wgm_type)) { 3321 case WG_MSG_TYPE_DATA: 3322 /* handle immediately */ 3323 m_adj(m, offset); 3324 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3325 m = m_pullup(m, sizeof(struct wg_msg_data)); 3326 if (m == NULL) 3327 return -1; 3328 } 3329 wg_handle_msg_data(wg, m, src); 3330 *mp = NULL; 3331 return 1; 3332 case WG_MSG_TYPE_INIT: 3333 case WG_MSG_TYPE_RESP: 3334 case WG_MSG_TYPE_COOKIE: 3335 /* pass through to so_receive in wg_receive_packets */ 3336 return 0; 3337 default: 3338 /* drop on the floor */ 3339 m_freem(m); 3340 return -1; 3341 } 3342 } 3343 3344 static int 3345 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3346 { 3347 int error; 3348 struct socket *so; 3349 3350 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3351 if (error != 0) 3352 return error; 3353 3354 solock(so); 3355 so->so_upcallarg = wg; 3356 so->so_upcall = wg_so_upcall; 3357 so->so_rcv.sb_flags |= SB_UPCALL; 3358 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3359 sounlock(so); 3360 3361 *sop = so; 3362 3363 return 0; 3364 } 3365 3366 static bool 3367 wg_session_hit_limits(struct wg_session *wgs) 3368 { 3369 3370 /* 3371 * [W] 6.2: Transport Message Limits 3372 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3373 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3374 * comes first, WireGuard will refuse to send any more transport data 3375 * messages using the current secure session, ..." 3376 */ 3377 KASSERT(wgs->wgs_time_established != 0); 3378 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3379 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3380 return true; 3381 } else if (wg_session_get_send_counter(wgs) > 3382 wg_reject_after_messages) { 3383 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3384 return true; 3385 } 3386 3387 return false; 3388 } 3389 3390 static void 3391 wgintr(void *cookie) 3392 { 3393 struct wg_peer *wgp; 3394 struct wg_session *wgs; 3395 struct mbuf *m; 3396 struct psref psref; 3397 3398 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3399 wgp = M_GETCTX(m, struct wg_peer *); 3400 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3401 WG_TRACE("no stable session"); 3402 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3403 goto next0; 3404 } 3405 if (__predict_false(wg_session_hit_limits(wgs))) { 3406 WG_TRACE("stable session hit limits"); 3407 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3408 goto next1; 3409 } 3410 wg_send_data_msg(wgp, wgs, m); 3411 m = NULL; /* consumed */ 3412 next1: wg_put_session(wgs, &psref); 3413 next0: m_freem(m); 3414 /* XXX Yield to avoid userland starvation? */ 3415 } 3416 } 3417 3418 static void 3419 wg_rekey_timer(void *arg) 3420 { 3421 struct wg_peer *wgp = arg; 3422 3423 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3424 } 3425 3426 static void 3427 wg_purge_pending_packets(struct wg_peer *wgp) 3428 { 3429 struct mbuf *m; 3430 3431 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 3432 m_freem(m); 3433 pktq_barrier(wg_pktq); 3434 } 3435 3436 static void 3437 wg_handshake_timeout_timer(void *arg) 3438 { 3439 struct wg_peer *wgp = arg; 3440 3441 WG_TRACE("enter"); 3442 3443 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3444 } 3445 3446 static struct wg_peer * 3447 wg_alloc_peer(struct wg_softc *wg) 3448 { 3449 struct wg_peer *wgp; 3450 3451 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3452 3453 wgp->wgp_sc = wg; 3454 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3455 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3456 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3457 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3458 wg_handshake_timeout_timer, wgp); 3459 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3460 callout_setfunc(&wgp->wgp_session_dtor_timer, 3461 wg_session_dtor_timer, wgp); 3462 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3463 wgp->wgp_endpoint_changing = false; 3464 wgp->wgp_endpoint_available = false; 3465 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3466 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3467 wgp->wgp_psz = pserialize_create(); 3468 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3469 3470 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3471 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3472 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3473 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3474 3475 struct wg_session *wgs; 3476 wgp->wgp_session_stable = 3477 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3478 wgp->wgp_session_unstable = 3479 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3480 wgs = wgp->wgp_session_stable; 3481 wgs->wgs_peer = wgp; 3482 wgs->wgs_state = WGS_STATE_UNKNOWN; 3483 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3484 #ifndef __HAVE_ATOMIC64_LOADSTORE 3485 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3486 #endif 3487 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3488 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3489 3490 wgs = wgp->wgp_session_unstable; 3491 wgs->wgs_peer = wgp; 3492 wgs->wgs_state = WGS_STATE_UNKNOWN; 3493 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3494 #ifndef __HAVE_ATOMIC64_LOADSTORE 3495 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3496 #endif 3497 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3498 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3499 3500 return wgp; 3501 } 3502 3503 static void 3504 wg_destroy_peer(struct wg_peer *wgp) 3505 { 3506 struct wg_session *wgs; 3507 struct wg_softc *wg = wgp->wgp_sc; 3508 3509 /* Prevent new packets from this peer on any source address. */ 3510 rw_enter(wg->wg_rwlock, RW_WRITER); 3511 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3512 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3513 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3514 struct radix_node *rn; 3515 3516 KASSERT(rnh != NULL); 3517 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3518 &wga->wga_sa_mask, rnh); 3519 if (rn == NULL) { 3520 char addrstr[128]; 3521 sockaddr_format(&wga->wga_sa_addr, addrstr, 3522 sizeof(addrstr)); 3523 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3524 if_name(&wg->wg_if), addrstr); 3525 } 3526 } 3527 rw_exit(wg->wg_rwlock); 3528 3529 /* Purge pending packets. */ 3530 wg_purge_pending_packets(wgp); 3531 3532 /* Halt all packet processing and timeouts. */ 3533 callout_halt(&wgp->wgp_rekey_timer, NULL); 3534 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3535 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3536 3537 /* Wait for any queued work to complete. */ 3538 workqueue_wait(wg_wq, &wgp->wgp_work); 3539 3540 wgs = wgp->wgp_session_unstable; 3541 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3542 mutex_enter(wgp->wgp_lock); 3543 wg_destroy_session(wg, wgs); 3544 mutex_exit(wgp->wgp_lock); 3545 } 3546 mutex_destroy(&wgs->wgs_recvwin->lock); 3547 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3548 #ifndef __HAVE_ATOMIC64_LOADSTORE 3549 mutex_destroy(&wgs->wgs_send_counter_lock); 3550 #endif 3551 kmem_free(wgs, sizeof(*wgs)); 3552 3553 wgs = wgp->wgp_session_stable; 3554 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3555 mutex_enter(wgp->wgp_lock); 3556 wg_destroy_session(wg, wgs); 3557 mutex_exit(wgp->wgp_lock); 3558 } 3559 mutex_destroy(&wgs->wgs_recvwin->lock); 3560 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3561 #ifndef __HAVE_ATOMIC64_LOADSTORE 3562 mutex_destroy(&wgs->wgs_send_counter_lock); 3563 #endif 3564 kmem_free(wgs, sizeof(*wgs)); 3565 3566 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3567 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3568 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3569 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3570 3571 pserialize_destroy(wgp->wgp_psz); 3572 mutex_obj_free(wgp->wgp_intr_lock); 3573 mutex_obj_free(wgp->wgp_lock); 3574 3575 kmem_free(wgp, sizeof(*wgp)); 3576 } 3577 3578 static void 3579 wg_destroy_all_peers(struct wg_softc *wg) 3580 { 3581 struct wg_peer *wgp, *wgp0 __diagused; 3582 void *garbage_byname, *garbage_bypubkey; 3583 3584 restart: 3585 garbage_byname = garbage_bypubkey = NULL; 3586 mutex_enter(wg->wg_lock); 3587 WG_PEER_WRITER_FOREACH(wgp, wg) { 3588 if (wgp->wgp_name[0]) { 3589 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3590 strlen(wgp->wgp_name)); 3591 KASSERT(wgp0 == wgp); 3592 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3593 } 3594 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3595 sizeof(wgp->wgp_pubkey)); 3596 KASSERT(wgp0 == wgp); 3597 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3598 WG_PEER_WRITER_REMOVE(wgp); 3599 wg->wg_npeers--; 3600 mutex_enter(wgp->wgp_lock); 3601 pserialize_perform(wgp->wgp_psz); 3602 mutex_exit(wgp->wgp_lock); 3603 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3604 break; 3605 } 3606 mutex_exit(wg->wg_lock); 3607 3608 if (wgp == NULL) 3609 return; 3610 3611 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3612 3613 wg_destroy_peer(wgp); 3614 thmap_gc(wg->wg_peers_byname, garbage_byname); 3615 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3616 3617 goto restart; 3618 } 3619 3620 static int 3621 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3622 { 3623 struct wg_peer *wgp, *wgp0 __diagused; 3624 void *garbage_byname, *garbage_bypubkey; 3625 3626 mutex_enter(wg->wg_lock); 3627 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3628 if (wgp != NULL) { 3629 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3630 sizeof(wgp->wgp_pubkey)); 3631 KASSERT(wgp0 == wgp); 3632 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3633 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3634 WG_PEER_WRITER_REMOVE(wgp); 3635 wg->wg_npeers--; 3636 if (wg->wg_npeers == 0) 3637 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3638 mutex_enter(wgp->wgp_lock); 3639 pserialize_perform(wgp->wgp_psz); 3640 mutex_exit(wgp->wgp_lock); 3641 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3642 } 3643 mutex_exit(wg->wg_lock); 3644 3645 if (wgp == NULL) 3646 return ENOENT; 3647 3648 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3649 3650 wg_destroy_peer(wgp); 3651 thmap_gc(wg->wg_peers_byname, garbage_byname); 3652 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3653 3654 return 0; 3655 } 3656 3657 static int 3658 wg_if_attach(struct wg_softc *wg) 3659 { 3660 3661 wg->wg_if.if_addrlen = 0; 3662 wg->wg_if.if_mtu = WG_MTU; 3663 wg->wg_if.if_flags = IFF_MULTICAST; 3664 wg->wg_if.if_extflags = IFEF_MPSAFE; 3665 wg->wg_if.if_ioctl = wg_ioctl; 3666 wg->wg_if.if_output = wg_output; 3667 wg->wg_if.if_init = wg_init; 3668 #ifdef ALTQ 3669 wg->wg_if.if_start = wg_start; 3670 #endif 3671 wg->wg_if.if_stop = wg_stop; 3672 wg->wg_if.if_type = IFT_OTHER; 3673 wg->wg_if.if_dlt = DLT_NULL; 3674 wg->wg_if.if_softc = wg; 3675 #ifdef ALTQ 3676 IFQ_SET_READY(&wg->wg_if.if_snd); 3677 #endif 3678 if_initialize(&wg->wg_if); 3679 3680 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3681 if_alloc_sadl(&wg->wg_if); 3682 if_register(&wg->wg_if); 3683 3684 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3685 3686 return 0; 3687 } 3688 3689 static void 3690 wg_if_detach(struct wg_softc *wg) 3691 { 3692 struct ifnet *ifp = &wg->wg_if; 3693 3694 bpf_detach(ifp); 3695 if_detach(ifp); 3696 } 3697 3698 static int 3699 wg_clone_create(struct if_clone *ifc, int unit) 3700 { 3701 struct wg_softc *wg; 3702 int error; 3703 3704 wg_guarantee_initialized(); 3705 3706 error = wg_count_inc(); 3707 if (error) 3708 return error; 3709 3710 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3711 3712 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3713 3714 PSLIST_INIT(&wg->wg_peers); 3715 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3716 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3717 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3718 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3719 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3720 wg->wg_rwlock = rw_obj_alloc(); 3721 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3722 "%s", if_name(&wg->wg_if)); 3723 wg->wg_ops = &wg_ops_rumpkernel; 3724 3725 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3726 if (error) 3727 goto fail0; 3728 3729 #ifdef INET 3730 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3731 if (error) 3732 goto fail1; 3733 rn_inithead((void **)&wg->wg_rtable_ipv4, 3734 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3735 #endif 3736 #ifdef INET6 3737 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3738 if (error) 3739 goto fail2; 3740 rn_inithead((void **)&wg->wg_rtable_ipv6, 3741 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3742 #endif 3743 3744 error = wg_if_attach(wg); 3745 if (error) 3746 goto fail3; 3747 3748 return 0; 3749 3750 fail4: __unused 3751 wg_if_detach(wg); 3752 fail3: wg_destroy_all_peers(wg); 3753 #ifdef INET6 3754 solock(wg->wg_so6); 3755 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3756 sounlock(wg->wg_so6); 3757 #endif 3758 #ifdef INET 3759 solock(wg->wg_so4); 3760 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3761 sounlock(wg->wg_so4); 3762 #endif 3763 mutex_enter(wg->wg_intr_lock); 3764 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3765 mutex_exit(wg->wg_intr_lock); 3766 #ifdef INET6 3767 if (wg->wg_rtable_ipv6 != NULL) 3768 free(wg->wg_rtable_ipv6, M_RTABLE); 3769 soclose(wg->wg_so6); 3770 fail2: 3771 #endif 3772 #ifdef INET 3773 if (wg->wg_rtable_ipv4 != NULL) 3774 free(wg->wg_rtable_ipv4, M_RTABLE); 3775 soclose(wg->wg_so4); 3776 fail1: 3777 #endif 3778 threadpool_put(wg->wg_threadpool, PRI_NONE); 3779 fail0: threadpool_job_destroy(&wg->wg_job); 3780 rw_obj_free(wg->wg_rwlock); 3781 mutex_obj_free(wg->wg_intr_lock); 3782 mutex_obj_free(wg->wg_lock); 3783 thmap_destroy(wg->wg_sessions_byindex); 3784 thmap_destroy(wg->wg_peers_byname); 3785 thmap_destroy(wg->wg_peers_bypubkey); 3786 PSLIST_DESTROY(&wg->wg_peers); 3787 kmem_free(wg, sizeof(*wg)); 3788 wg_count_dec(); 3789 return error; 3790 } 3791 3792 static int 3793 wg_clone_destroy(struct ifnet *ifp) 3794 { 3795 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3796 3797 #ifdef WG_RUMPKERNEL 3798 if (wg_user_mode(wg)) { 3799 rumpuser_wg_destroy(wg->wg_user); 3800 wg->wg_user = NULL; 3801 } 3802 #endif 3803 3804 wg_if_detach(wg); 3805 wg_destroy_all_peers(wg); 3806 #ifdef INET6 3807 solock(wg->wg_so6); 3808 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3809 sounlock(wg->wg_so6); 3810 #endif 3811 #ifdef INET 3812 solock(wg->wg_so4); 3813 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3814 sounlock(wg->wg_so4); 3815 #endif 3816 mutex_enter(wg->wg_intr_lock); 3817 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3818 mutex_exit(wg->wg_intr_lock); 3819 #ifdef INET6 3820 if (wg->wg_rtable_ipv6 != NULL) 3821 free(wg->wg_rtable_ipv6, M_RTABLE); 3822 soclose(wg->wg_so6); 3823 #endif 3824 #ifdef INET 3825 if (wg->wg_rtable_ipv4 != NULL) 3826 free(wg->wg_rtable_ipv4, M_RTABLE); 3827 soclose(wg->wg_so4); 3828 #endif 3829 threadpool_put(wg->wg_threadpool, PRI_NONE); 3830 threadpool_job_destroy(&wg->wg_job); 3831 rw_obj_free(wg->wg_rwlock); 3832 mutex_obj_free(wg->wg_intr_lock); 3833 mutex_obj_free(wg->wg_lock); 3834 thmap_destroy(wg->wg_sessions_byindex); 3835 thmap_destroy(wg->wg_peers_byname); 3836 thmap_destroy(wg->wg_peers_bypubkey); 3837 PSLIST_DESTROY(&wg->wg_peers); 3838 kmem_free(wg, sizeof(*wg)); 3839 wg_count_dec(); 3840 3841 return 0; 3842 } 3843 3844 static struct wg_peer * 3845 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3846 struct psref *psref) 3847 { 3848 struct radix_node_head *rnh; 3849 struct radix_node *rn; 3850 struct wg_peer *wgp = NULL; 3851 struct wg_allowedip *wga; 3852 3853 #ifdef WG_DEBUG_LOG 3854 char addrstr[128]; 3855 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3856 WG_DLOG("sa=%s\n", addrstr); 3857 #endif 3858 3859 rw_enter(wg->wg_rwlock, RW_READER); 3860 3861 rnh = wg_rnh(wg, sa->sa_family); 3862 if (rnh == NULL) 3863 goto out; 3864 3865 rn = rnh->rnh_matchaddr(sa, rnh); 3866 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3867 goto out; 3868 3869 WG_TRACE("success"); 3870 3871 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3872 wgp = wga->wga_peer; 3873 wg_get_peer(wgp, psref); 3874 3875 out: 3876 rw_exit(wg->wg_rwlock); 3877 return wgp; 3878 } 3879 3880 static void 3881 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3882 struct wg_session *wgs, struct wg_msg_data *wgmd) 3883 { 3884 3885 memset(wgmd, 0, sizeof(*wgmd)); 3886 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3887 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3888 /* [W] 5.4.6: msg.counter := Nm^send */ 3889 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3890 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3891 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3892 } 3893 3894 static int 3895 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3896 const struct rtentry *rt) 3897 { 3898 struct wg_softc *wg = ifp->if_softc; 3899 struct wg_peer *wgp = NULL; 3900 struct wg_session *wgs = NULL; 3901 struct psref wgp_psref, wgs_psref; 3902 int bound; 3903 int error; 3904 3905 bound = curlwp_bind(); 3906 3907 /* TODO make the nest limit configurable via sysctl */ 3908 error = if_tunnel_check_nesting(ifp, m, 1); 3909 if (error) { 3910 WGLOG(LOG_ERR, 3911 "%s: tunneling loop detected and packet dropped\n", 3912 if_name(&wg->wg_if)); 3913 goto out0; 3914 } 3915 3916 #ifdef ALTQ 3917 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 3918 & ALTQF_ENABLED; 3919 if (altq) 3920 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3921 #endif 3922 3923 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3924 3925 m->m_flags &= ~(M_BCAST|M_MCAST); 3926 3927 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3928 if (wgp == NULL) { 3929 WG_TRACE("peer not found"); 3930 error = EHOSTUNREACH; 3931 goto out0; 3932 } 3933 3934 /* Clear checksum-offload flags. */ 3935 m->m_pkthdr.csum_flags = 0; 3936 m->m_pkthdr.csum_data = 0; 3937 3938 /* Check whether there's an established session. */ 3939 wgs = wg_get_stable_session(wgp, &wgs_psref); 3940 if (wgs == NULL) { 3941 /* 3942 * No established session. If we're the first to try 3943 * sending data, schedule a handshake and queue the 3944 * packet for when the handshake is done; otherwise 3945 * just drop the packet and let the ongoing handshake 3946 * attempt continue. We could queue more data packets 3947 * but it's not clear that's worthwhile. 3948 */ 3949 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 3950 m = NULL; /* consume */ 3951 WG_TRACE("queued first packet; init handshake"); 3952 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3953 } else { 3954 WG_TRACE("first packet already queued, dropping"); 3955 } 3956 goto out1; 3957 } 3958 3959 /* There's an established session. Toss it in the queue. */ 3960 #ifdef ALTQ 3961 if (altq) { 3962 mutex_enter(ifp->if_snd.ifq_lock); 3963 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 3964 M_SETCTX(m, wgp); 3965 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 3966 m = NULL; /* consume */ 3967 } 3968 mutex_exit(ifp->if_snd.ifq_lock); 3969 if (m == NULL) { 3970 wg_start(ifp); 3971 goto out2; 3972 } 3973 } 3974 #endif 3975 kpreempt_disable(); 3976 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3977 M_SETCTX(m, wgp); 3978 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3979 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3980 if_name(&wg->wg_if)); 3981 error = ENOBUFS; 3982 goto out3; 3983 } 3984 m = NULL; /* consumed */ 3985 error = 0; 3986 out3: kpreempt_enable(); 3987 3988 #ifdef ALTQ 3989 out2: 3990 #endif 3991 wg_put_session(wgs, &wgs_psref); 3992 out1: wg_put_peer(wgp, &wgp_psref); 3993 out0: m_freem(m); 3994 curlwp_bindx(bound); 3995 return error; 3996 } 3997 3998 static int 3999 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 4000 { 4001 struct psref psref; 4002 struct wg_sockaddr *wgsa; 4003 int error; 4004 struct socket *so; 4005 4006 wgsa = wg_get_endpoint_sa(wgp, &psref); 4007 so = wg_get_so_by_peer(wgp, wgsa); 4008 solock(so); 4009 if (wgsatosa(wgsa)->sa_family == AF_INET) { 4010 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 4011 } else { 4012 #ifdef INET6 4013 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 4014 NULL, curlwp); 4015 #else 4016 m_freem(m); 4017 error = EPFNOSUPPORT; 4018 #endif 4019 } 4020 sounlock(so); 4021 wg_put_sa(wgp, wgsa, &psref); 4022 4023 return error; 4024 } 4025 4026 /* Inspired by pppoe_get_mbuf */ 4027 static struct mbuf * 4028 wg_get_mbuf(size_t leading_len, size_t len) 4029 { 4030 struct mbuf *m; 4031 4032 KASSERT(leading_len <= MCLBYTES); 4033 KASSERT(len <= MCLBYTES - leading_len); 4034 4035 m = m_gethdr(M_DONTWAIT, MT_DATA); 4036 if (m == NULL) 4037 return NULL; 4038 if (len + leading_len > MHLEN) { 4039 m_clget(m, M_DONTWAIT); 4040 if ((m->m_flags & M_EXT) == 0) { 4041 m_free(m); 4042 return NULL; 4043 } 4044 } 4045 m->m_data += leading_len; 4046 m->m_pkthdr.len = m->m_len = len; 4047 4048 return m; 4049 } 4050 4051 static int 4052 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 4053 struct mbuf *m) 4054 { 4055 struct wg_softc *wg = wgp->wgp_sc; 4056 int error; 4057 size_t inner_len, padded_len, encrypted_len; 4058 char *padded_buf = NULL; 4059 size_t mlen; 4060 struct wg_msg_data *wgmd; 4061 bool free_padded_buf = false; 4062 struct mbuf *n; 4063 size_t leading_len = max_hdr + sizeof(struct udphdr); 4064 4065 mlen = m_length(m); 4066 inner_len = mlen; 4067 padded_len = roundup(mlen, 16); 4068 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4069 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 4070 inner_len, padded_len, encrypted_len); 4071 if (mlen != 0) { 4072 bool success; 4073 success = m_ensure_contig(&m, padded_len); 4074 if (success) { 4075 padded_buf = mtod(m, char *); 4076 } else { 4077 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4078 if (padded_buf == NULL) { 4079 error = ENOBUFS; 4080 goto end; 4081 } 4082 free_padded_buf = true; 4083 m_copydata(m, 0, mlen, padded_buf); 4084 } 4085 memset(padded_buf + mlen, 0, padded_len - inner_len); 4086 } 4087 4088 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4089 if (n == NULL) { 4090 error = ENOBUFS; 4091 goto end; 4092 } 4093 KASSERT(n->m_len >= sizeof(*wgmd)); 4094 wgmd = mtod(n, struct wg_msg_data *); 4095 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4096 #ifdef WG_DEBUG_PACKET 4097 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4098 char *hex = gethexdump(wgmd, sizeof(*wgmd)); 4099 if (hex != NULL) { 4100 log(LOG_DEBUG, "wgmd=%p, sizeof(*wgmd)=%zu\n%s\n", 4101 wgmd, sizeof(*wgmd), hex); 4102 puthexdump(hex, wgmd, sizeof(*wgmd)); 4103 } 4104 hex = gethexdump(padded_buf, padded_len); 4105 if (hex != NULL) { 4106 log(LOG_DEBUG, "padded_buf=%p, padded_len=%zu\n%s\n", 4107 padded_buf, padded_len, hex); 4108 puthexdump(hex, padded_buf, padded_len); 4109 } 4110 } 4111 #endif 4112 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4113 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4114 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4115 padded_buf, padded_len, 4116 NULL, 0); 4117 4118 error = wg->wg_ops->send_data_msg(wgp, n); 4119 if (error == 0) { 4120 struct ifnet *ifp = &wg->wg_if; 4121 if_statadd(ifp, if_obytes, mlen); 4122 if_statinc(ifp, if_opackets); 4123 if (wgs->wgs_is_initiator && 4124 wgs->wgs_time_last_data_sent == 0) { 4125 /* 4126 * [W] 6.2 Transport Message Limits 4127 * "if a peer is the initiator of a current secure 4128 * session, WireGuard will send a handshake initiation 4129 * message to begin a new secure session if, after 4130 * transmitting a transport data message, the current 4131 * secure session is REKEY-AFTER-TIME seconds old," 4132 */ 4133 wg_schedule_rekey_timer(wgp); 4134 } 4135 wgs->wgs_time_last_data_sent = time_uptime; 4136 if (wg_session_get_send_counter(wgs) >= 4137 wg_rekey_after_messages) { 4138 /* 4139 * [W] 6.2 Transport Message Limits 4140 * "WireGuard will try to create a new session, by 4141 * sending a handshake initiation message (section 4142 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4143 * transport data messages..." 4144 */ 4145 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4146 } 4147 } 4148 end: 4149 m_freem(m); 4150 if (free_padded_buf) 4151 kmem_intr_free(padded_buf, padded_len); 4152 return error; 4153 } 4154 4155 static void 4156 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4157 { 4158 pktqueue_t *pktq; 4159 size_t pktlen; 4160 4161 KASSERT(af == AF_INET || af == AF_INET6); 4162 4163 WG_TRACE(""); 4164 4165 m_set_rcvif(m, ifp); 4166 pktlen = m->m_pkthdr.len; 4167 4168 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4169 4170 switch (af) { 4171 case AF_INET: 4172 pktq = ip_pktq; 4173 break; 4174 #ifdef INET6 4175 case AF_INET6: 4176 pktq = ip6_pktq; 4177 break; 4178 #endif 4179 default: 4180 panic("invalid af=%d", af); 4181 } 4182 4183 kpreempt_disable(); 4184 const u_int h = curcpu()->ci_index; 4185 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4186 if_statadd(ifp, if_ibytes, pktlen); 4187 if_statinc(ifp, if_ipackets); 4188 } else { 4189 m_freem(m); 4190 } 4191 kpreempt_enable(); 4192 } 4193 4194 static void 4195 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4196 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4197 { 4198 4199 crypto_scalarmult_base(pubkey, privkey); 4200 } 4201 4202 static int 4203 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4204 { 4205 struct radix_node_head *rnh; 4206 struct radix_node *rn; 4207 int error = 0; 4208 4209 rw_enter(wg->wg_rwlock, RW_WRITER); 4210 rnh = wg_rnh(wg, wga->wga_family); 4211 KASSERT(rnh != NULL); 4212 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4213 wga->wga_nodes); 4214 rw_exit(wg->wg_rwlock); 4215 4216 if (rn == NULL) 4217 error = EEXIST; 4218 4219 return error; 4220 } 4221 4222 static int 4223 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4224 struct wg_peer **wgpp) 4225 { 4226 int error = 0; 4227 const void *pubkey; 4228 size_t pubkey_len; 4229 const void *psk; 4230 size_t psk_len; 4231 const char *name = NULL; 4232 4233 if (prop_dictionary_get_string(peer, "name", &name)) { 4234 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4235 error = EINVAL; 4236 goto out; 4237 } 4238 } 4239 4240 if (!prop_dictionary_get_data(peer, "public_key", 4241 &pubkey, &pubkey_len)) { 4242 error = EINVAL; 4243 goto out; 4244 } 4245 #ifdef WG_DEBUG_DUMP 4246 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4247 char *hex = gethexdump(pubkey, pubkey_len); 4248 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n%s\n", 4249 pubkey, pubkey_len, hex); 4250 puthexdump(hex, pubkey, pubkey_len); 4251 } 4252 #endif 4253 4254 struct wg_peer *wgp = wg_alloc_peer(wg); 4255 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4256 if (name != NULL) 4257 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4258 4259 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4260 if (psk_len != sizeof(wgp->wgp_psk)) { 4261 error = EINVAL; 4262 goto out; 4263 } 4264 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4265 } 4266 4267 const void *addr; 4268 size_t addr_len; 4269 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4270 4271 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4272 goto skip_endpoint; 4273 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4274 addr_len > sizeof(*wgsatoss(wgsa))) { 4275 error = EINVAL; 4276 goto out; 4277 } 4278 memcpy(wgsatoss(wgsa), addr, addr_len); 4279 switch (wgsa_family(wgsa)) { 4280 case AF_INET: 4281 #ifdef INET6 4282 case AF_INET6: 4283 #endif 4284 break; 4285 default: 4286 error = EPFNOSUPPORT; 4287 goto out; 4288 } 4289 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4290 error = EINVAL; 4291 goto out; 4292 } 4293 { 4294 char addrstr[128]; 4295 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4296 WG_DLOG("addr=%s\n", addrstr); 4297 } 4298 wgp->wgp_endpoint_available = true; 4299 4300 prop_array_t allowedips; 4301 skip_endpoint: 4302 allowedips = prop_dictionary_get(peer, "allowedips"); 4303 if (allowedips == NULL) 4304 goto skip; 4305 4306 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4307 prop_dictionary_t prop_allowedip; 4308 int j = 0; 4309 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4310 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4311 4312 if (!prop_dictionary_get_int(prop_allowedip, "family", 4313 &wga->wga_family)) 4314 continue; 4315 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4316 &addr, &addr_len)) 4317 continue; 4318 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4319 &wga->wga_cidr)) 4320 continue; 4321 4322 switch (wga->wga_family) { 4323 case AF_INET: { 4324 struct sockaddr_in sin; 4325 char addrstr[128]; 4326 struct in_addr mask; 4327 struct sockaddr_in sin_mask; 4328 4329 if (addr_len != sizeof(struct in_addr)) 4330 return EINVAL; 4331 memcpy(&wga->wga_addr4, addr, addr_len); 4332 4333 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4334 0); 4335 sockaddr_copy(&wga->wga_sa_addr, 4336 sizeof(sin), sintosa(&sin)); 4337 4338 sockaddr_format(sintosa(&sin), 4339 addrstr, sizeof(addrstr)); 4340 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4341 4342 in_len2mask(&mask, wga->wga_cidr); 4343 sockaddr_in_init(&sin_mask, &mask, 0); 4344 sockaddr_copy(&wga->wga_sa_mask, 4345 sizeof(sin_mask), sintosa(&sin_mask)); 4346 4347 break; 4348 } 4349 #ifdef INET6 4350 case AF_INET6: { 4351 struct sockaddr_in6 sin6; 4352 char addrstr[128]; 4353 struct in6_addr mask; 4354 struct sockaddr_in6 sin6_mask; 4355 4356 if (addr_len != sizeof(struct in6_addr)) 4357 return EINVAL; 4358 memcpy(&wga->wga_addr6, addr, addr_len); 4359 4360 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4361 0, 0, 0); 4362 sockaddr_copy(&wga->wga_sa_addr, 4363 sizeof(sin6), sin6tosa(&sin6)); 4364 4365 sockaddr_format(sin6tosa(&sin6), 4366 addrstr, sizeof(addrstr)); 4367 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4368 4369 in6_prefixlen2mask(&mask, wga->wga_cidr); 4370 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4371 sockaddr_copy(&wga->wga_sa_mask, 4372 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4373 4374 break; 4375 } 4376 #endif 4377 default: 4378 error = EINVAL; 4379 goto out; 4380 } 4381 wga->wga_peer = wgp; 4382 4383 error = wg_rtable_add_route(wg, wga); 4384 if (error != 0) 4385 goto out; 4386 4387 j++; 4388 } 4389 wgp->wgp_n_allowedips = j; 4390 skip: 4391 *wgpp = wgp; 4392 out: 4393 return error; 4394 } 4395 4396 static int 4397 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4398 { 4399 int error; 4400 char *buf; 4401 4402 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4403 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4404 return E2BIG; 4405 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4406 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4407 if (error != 0) 4408 return error; 4409 buf[ifd->ifd_len] = '\0'; 4410 #ifdef WG_DEBUG_DUMP 4411 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4412 log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len), 4413 (const char *)buf); 4414 } 4415 #endif 4416 *_buf = buf; 4417 return 0; 4418 } 4419 4420 static int 4421 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4422 { 4423 int error; 4424 prop_dictionary_t prop_dict; 4425 char *buf = NULL; 4426 const void *privkey; 4427 size_t privkey_len; 4428 4429 error = wg_alloc_prop_buf(&buf, ifd); 4430 if (error != 0) 4431 return error; 4432 error = EINVAL; 4433 prop_dict = prop_dictionary_internalize(buf); 4434 if (prop_dict == NULL) 4435 goto out; 4436 if (!prop_dictionary_get_data(prop_dict, "private_key", 4437 &privkey, &privkey_len)) 4438 goto out; 4439 #ifdef WG_DEBUG_DUMP 4440 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4441 char *hex = gethexdump(privkey, privkey_len); 4442 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n%s\n", 4443 privkey, privkey_len, hex); 4444 puthexdump(hex, privkey, privkey_len); 4445 } 4446 #endif 4447 if (privkey_len != WG_STATIC_KEY_LEN) 4448 goto out; 4449 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4450 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4451 error = 0; 4452 4453 out: 4454 kmem_free(buf, ifd->ifd_len + 1); 4455 return error; 4456 } 4457 4458 static int 4459 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4460 { 4461 int error; 4462 prop_dictionary_t prop_dict; 4463 char *buf = NULL; 4464 uint16_t port; 4465 4466 error = wg_alloc_prop_buf(&buf, ifd); 4467 if (error != 0) 4468 return error; 4469 error = EINVAL; 4470 prop_dict = prop_dictionary_internalize(buf); 4471 if (prop_dict == NULL) 4472 goto out; 4473 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4474 goto out; 4475 4476 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4477 4478 out: 4479 kmem_free(buf, ifd->ifd_len + 1); 4480 return error; 4481 } 4482 4483 static int 4484 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4485 { 4486 int error; 4487 prop_dictionary_t prop_dict; 4488 char *buf = NULL; 4489 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4490 4491 error = wg_alloc_prop_buf(&buf, ifd); 4492 if (error != 0) 4493 return error; 4494 error = EINVAL; 4495 prop_dict = prop_dictionary_internalize(buf); 4496 if (prop_dict == NULL) 4497 goto out; 4498 4499 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4500 if (error != 0) 4501 goto out; 4502 4503 mutex_enter(wg->wg_lock); 4504 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4505 sizeof(wgp->wgp_pubkey)) != NULL || 4506 (wgp->wgp_name[0] && 4507 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4508 strlen(wgp->wgp_name)) != NULL)) { 4509 mutex_exit(wg->wg_lock); 4510 wg_destroy_peer(wgp); 4511 error = EEXIST; 4512 goto out; 4513 } 4514 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4515 sizeof(wgp->wgp_pubkey), wgp); 4516 KASSERT(wgp0 == wgp); 4517 if (wgp->wgp_name[0]) { 4518 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4519 strlen(wgp->wgp_name), wgp); 4520 KASSERT(wgp0 == wgp); 4521 } 4522 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4523 wg->wg_npeers++; 4524 mutex_exit(wg->wg_lock); 4525 4526 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4527 4528 out: 4529 kmem_free(buf, ifd->ifd_len + 1); 4530 return error; 4531 } 4532 4533 static int 4534 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4535 { 4536 int error; 4537 prop_dictionary_t prop_dict; 4538 char *buf = NULL; 4539 const char *name; 4540 4541 error = wg_alloc_prop_buf(&buf, ifd); 4542 if (error != 0) 4543 return error; 4544 error = EINVAL; 4545 prop_dict = prop_dictionary_internalize(buf); 4546 if (prop_dict == NULL) 4547 goto out; 4548 4549 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4550 goto out; 4551 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4552 goto out; 4553 4554 error = wg_destroy_peer_name(wg, name); 4555 out: 4556 kmem_free(buf, ifd->ifd_len + 1); 4557 return error; 4558 } 4559 4560 static bool 4561 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4562 { 4563 int au = cmd == SIOCGDRVSPEC ? 4564 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4565 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4566 return kauth_authorize_network(kauth_cred_get(), 4567 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4568 (void *)cmd, NULL) == 0; 4569 } 4570 4571 static int 4572 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4573 { 4574 int error = ENOMEM; 4575 prop_dictionary_t prop_dict; 4576 prop_array_t peers = NULL; 4577 char *buf; 4578 struct wg_peer *wgp; 4579 int s, i; 4580 4581 prop_dict = prop_dictionary_create(); 4582 if (prop_dict == NULL) 4583 goto error; 4584 4585 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4586 if (!prop_dictionary_set_data(prop_dict, "private_key", 4587 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4588 goto error; 4589 } 4590 4591 if (wg->wg_listen_port != 0) { 4592 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4593 wg->wg_listen_port)) 4594 goto error; 4595 } 4596 4597 if (wg->wg_npeers == 0) 4598 goto skip_peers; 4599 4600 peers = prop_array_create(); 4601 if (peers == NULL) 4602 goto error; 4603 4604 s = pserialize_read_enter(); 4605 i = 0; 4606 WG_PEER_READER_FOREACH(wgp, wg) { 4607 struct wg_sockaddr *wgsa; 4608 struct psref wgp_psref, wgsa_psref; 4609 prop_dictionary_t prop_peer; 4610 4611 wg_get_peer(wgp, &wgp_psref); 4612 pserialize_read_exit(s); 4613 4614 prop_peer = prop_dictionary_create(); 4615 if (prop_peer == NULL) 4616 goto next; 4617 4618 if (strlen(wgp->wgp_name) > 0) { 4619 if (!prop_dictionary_set_string(prop_peer, "name", 4620 wgp->wgp_name)) 4621 goto next; 4622 } 4623 4624 if (!prop_dictionary_set_data(prop_peer, "public_key", 4625 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4626 goto next; 4627 4628 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4629 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4630 sizeof(wgp->wgp_psk))) { 4631 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4632 if (!prop_dictionary_set_data(prop_peer, 4633 "preshared_key", 4634 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4635 goto next; 4636 } 4637 } 4638 4639 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4640 CTASSERT(AF_UNSPEC == 0); 4641 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4642 !prop_dictionary_set_data(prop_peer, "endpoint", 4643 wgsatoss(wgsa), 4644 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4645 wg_put_sa(wgp, wgsa, &wgsa_psref); 4646 goto next; 4647 } 4648 wg_put_sa(wgp, wgsa, &wgsa_psref); 4649 4650 const struct timespec *t = &wgp->wgp_last_handshake_time; 4651 4652 if (!prop_dictionary_set_uint64(prop_peer, 4653 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4654 goto next; 4655 if (!prop_dictionary_set_uint32(prop_peer, 4656 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4657 goto next; 4658 4659 if (wgp->wgp_n_allowedips == 0) 4660 goto skip_allowedips; 4661 4662 prop_array_t allowedips = prop_array_create(); 4663 if (allowedips == NULL) 4664 goto next; 4665 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4666 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4667 prop_dictionary_t prop_allowedip; 4668 4669 prop_allowedip = prop_dictionary_create(); 4670 if (prop_allowedip == NULL) 4671 break; 4672 4673 if (!prop_dictionary_set_int(prop_allowedip, "family", 4674 wga->wga_family)) 4675 goto _next; 4676 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4677 wga->wga_cidr)) 4678 goto _next; 4679 4680 switch (wga->wga_family) { 4681 case AF_INET: 4682 if (!prop_dictionary_set_data(prop_allowedip, 4683 "ip", &wga->wga_addr4, 4684 sizeof(wga->wga_addr4))) 4685 goto _next; 4686 break; 4687 #ifdef INET6 4688 case AF_INET6: 4689 if (!prop_dictionary_set_data(prop_allowedip, 4690 "ip", &wga->wga_addr6, 4691 sizeof(wga->wga_addr6))) 4692 goto _next; 4693 break; 4694 #endif 4695 default: 4696 break; 4697 } 4698 prop_array_set(allowedips, j, prop_allowedip); 4699 _next: 4700 prop_object_release(prop_allowedip); 4701 } 4702 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4703 prop_object_release(allowedips); 4704 4705 skip_allowedips: 4706 4707 prop_array_set(peers, i, prop_peer); 4708 next: 4709 if (prop_peer) 4710 prop_object_release(prop_peer); 4711 i++; 4712 4713 s = pserialize_read_enter(); 4714 wg_put_peer(wgp, &wgp_psref); 4715 } 4716 pserialize_read_exit(s); 4717 4718 prop_dictionary_set(prop_dict, "peers", peers); 4719 prop_object_release(peers); 4720 peers = NULL; 4721 4722 skip_peers: 4723 buf = prop_dictionary_externalize(prop_dict); 4724 if (buf == NULL) 4725 goto error; 4726 if (ifd->ifd_len < (strlen(buf) + 1)) { 4727 error = EINVAL; 4728 goto error; 4729 } 4730 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4731 4732 free(buf, 0); 4733 error: 4734 if (peers != NULL) 4735 prop_object_release(peers); 4736 if (prop_dict != NULL) 4737 prop_object_release(prop_dict); 4738 4739 return error; 4740 } 4741 4742 static int 4743 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4744 { 4745 struct wg_softc *wg = ifp->if_softc; 4746 struct ifreq *ifr = data; 4747 struct ifaddr *ifa = data; 4748 struct ifdrv *ifd = data; 4749 int error = 0; 4750 4751 switch (cmd) { 4752 case SIOCINITIFADDR: 4753 if (ifa->ifa_addr->sa_family != AF_LINK && 4754 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4755 (IFF_UP | IFF_RUNNING)) { 4756 ifp->if_flags |= IFF_UP; 4757 error = if_init(ifp); 4758 } 4759 return error; 4760 case SIOCADDMULTI: 4761 case SIOCDELMULTI: 4762 switch (ifr->ifr_addr.sa_family) { 4763 case AF_INET: /* IP supports Multicast */ 4764 break; 4765 #ifdef INET6 4766 case AF_INET6: /* IP6 supports Multicast */ 4767 break; 4768 #endif 4769 default: /* Other protocols doesn't support Multicast */ 4770 error = EAFNOSUPPORT; 4771 break; 4772 } 4773 return error; 4774 case SIOCSDRVSPEC: 4775 if (!wg_is_authorized(wg, cmd)) { 4776 return EPERM; 4777 } 4778 switch (ifd->ifd_cmd) { 4779 case WG_IOCTL_SET_PRIVATE_KEY: 4780 error = wg_ioctl_set_private_key(wg, ifd); 4781 break; 4782 case WG_IOCTL_SET_LISTEN_PORT: 4783 error = wg_ioctl_set_listen_port(wg, ifd); 4784 break; 4785 case WG_IOCTL_ADD_PEER: 4786 error = wg_ioctl_add_peer(wg, ifd); 4787 break; 4788 case WG_IOCTL_DELETE_PEER: 4789 error = wg_ioctl_delete_peer(wg, ifd); 4790 break; 4791 default: 4792 error = EINVAL; 4793 break; 4794 } 4795 return error; 4796 case SIOCGDRVSPEC: 4797 return wg_ioctl_get(wg, ifd); 4798 case SIOCSIFFLAGS: 4799 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4800 break; 4801 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4802 case IFF_RUNNING: 4803 /* 4804 * If interface is marked down and it is running, 4805 * then stop and disable it. 4806 */ 4807 if_stop(ifp, 1); 4808 break; 4809 case IFF_UP: 4810 /* 4811 * If interface is marked up and it is stopped, then 4812 * start it. 4813 */ 4814 error = if_init(ifp); 4815 break; 4816 default: 4817 break; 4818 } 4819 return error; 4820 #ifdef WG_RUMPKERNEL 4821 case SIOCSLINKSTR: 4822 error = wg_ioctl_linkstr(wg, ifd); 4823 if (error == 0) 4824 wg->wg_ops = &wg_ops_rumpuser; 4825 return error; 4826 #endif 4827 default: 4828 break; 4829 } 4830 4831 error = ifioctl_common(ifp, cmd, data); 4832 4833 #ifdef WG_RUMPKERNEL 4834 if (!wg_user_mode(wg)) 4835 return error; 4836 4837 /* Do the same to the corresponding tun device on the host */ 4838 /* 4839 * XXX Actually the command has not been handled yet. It 4840 * will be handled via pr_ioctl form doifioctl later. 4841 */ 4842 switch (cmd) { 4843 case SIOCAIFADDR: 4844 case SIOCDIFADDR: { 4845 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4846 struct in_aliasreq *ifra = &_ifra; 4847 KASSERT(error == ENOTTY); 4848 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4849 IFNAMSIZ); 4850 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4851 if (error == 0) 4852 error = ENOTTY; 4853 break; 4854 } 4855 #ifdef INET6 4856 case SIOCAIFADDR_IN6: 4857 case SIOCDIFADDR_IN6: { 4858 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4859 struct in6_aliasreq *ifra = &_ifra; 4860 KASSERT(error == ENOTTY); 4861 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4862 IFNAMSIZ); 4863 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4864 if (error == 0) 4865 error = ENOTTY; 4866 break; 4867 } 4868 #endif 4869 } 4870 #endif /* WG_RUMPKERNEL */ 4871 4872 return error; 4873 } 4874 4875 static int 4876 wg_init(struct ifnet *ifp) 4877 { 4878 4879 ifp->if_flags |= IFF_RUNNING; 4880 4881 /* TODO flush pending packets. */ 4882 return 0; 4883 } 4884 4885 #ifdef ALTQ 4886 static void 4887 wg_start(struct ifnet *ifp) 4888 { 4889 struct mbuf *m; 4890 4891 for (;;) { 4892 IFQ_DEQUEUE(&ifp->if_snd, m); 4893 if (m == NULL) 4894 break; 4895 4896 kpreempt_disable(); 4897 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4898 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4899 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4900 if_name(ifp)); 4901 m_freem(m); 4902 } 4903 kpreempt_enable(); 4904 } 4905 } 4906 #endif 4907 4908 static void 4909 wg_stop(struct ifnet *ifp, int disable) 4910 { 4911 4912 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4913 ifp->if_flags &= ~IFF_RUNNING; 4914 4915 /* Need to do something? */ 4916 } 4917 4918 #ifdef WG_DEBUG_PARAMS 4919 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4920 { 4921 const struct sysctlnode *node = NULL; 4922 4923 sysctl_createv(clog, 0, NULL, &node, 4924 CTLFLAG_PERMANENT, 4925 CTLTYPE_NODE, "wg", 4926 SYSCTL_DESCR("wg(4)"), 4927 NULL, 0, NULL, 0, 4928 CTL_NET, CTL_CREATE, CTL_EOL); 4929 sysctl_createv(clog, 0, &node, NULL, 4930 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4931 CTLTYPE_QUAD, "rekey_after_messages", 4932 SYSCTL_DESCR("session liftime by messages"), 4933 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4934 sysctl_createv(clog, 0, &node, NULL, 4935 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4936 CTLTYPE_INT, "rekey_after_time", 4937 SYSCTL_DESCR("session liftime"), 4938 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4939 sysctl_createv(clog, 0, &node, NULL, 4940 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4941 CTLTYPE_INT, "rekey_timeout", 4942 SYSCTL_DESCR("session handshake retry time"), 4943 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4944 sysctl_createv(clog, 0, &node, NULL, 4945 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4946 CTLTYPE_INT, "rekey_attempt_time", 4947 SYSCTL_DESCR("session handshake timeout"), 4948 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4949 sysctl_createv(clog, 0, &node, NULL, 4950 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4951 CTLTYPE_INT, "keepalive_timeout", 4952 SYSCTL_DESCR("keepalive timeout"), 4953 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4954 sysctl_createv(clog, 0, &node, NULL, 4955 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4956 CTLTYPE_BOOL, "force_underload", 4957 SYSCTL_DESCR("force to detemine under load"), 4958 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4959 sysctl_createv(clog, 0, &node, NULL, 4960 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4961 CTLTYPE_INT, "debug", 4962 SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"), 4963 NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL); 4964 } 4965 #endif 4966 4967 #ifdef WG_RUMPKERNEL 4968 static bool 4969 wg_user_mode(struct wg_softc *wg) 4970 { 4971 4972 return wg->wg_user != NULL; 4973 } 4974 4975 static int 4976 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4977 { 4978 struct ifnet *ifp = &wg->wg_if; 4979 int error; 4980 4981 if (ifp->if_flags & IFF_UP) 4982 return EBUSY; 4983 4984 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4985 /* XXX do nothing */ 4986 return 0; 4987 } else if (ifd->ifd_cmd != 0) { 4988 return EINVAL; 4989 } else if (wg->wg_user != NULL) { 4990 return EBUSY; 4991 } 4992 4993 /* Assume \0 included */ 4994 if (ifd->ifd_len > IFNAMSIZ) { 4995 return E2BIG; 4996 } else if (ifd->ifd_len < 1) { 4997 return EINVAL; 4998 } 4999 5000 char tun_name[IFNAMSIZ]; 5001 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 5002 if (error != 0) 5003 return error; 5004 5005 if (strncmp(tun_name, "tun", 3) != 0) 5006 return EINVAL; 5007 5008 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 5009 5010 return error; 5011 } 5012 5013 static int 5014 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 5015 { 5016 int error; 5017 struct psref psref; 5018 struct wg_sockaddr *wgsa; 5019 struct wg_softc *wg = wgp->wgp_sc; 5020 struct iovec iov[1]; 5021 5022 wgsa = wg_get_endpoint_sa(wgp, &psref); 5023 5024 iov[0].iov_base = mtod(m, void *); 5025 iov[0].iov_len = m->m_len; 5026 5027 /* Send messages to a peer via an ordinary socket. */ 5028 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 5029 5030 wg_put_sa(wgp, wgsa, &psref); 5031 5032 m_freem(m); 5033 5034 return error; 5035 } 5036 5037 static void 5038 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 5039 { 5040 struct wg_softc *wg = ifp->if_softc; 5041 struct iovec iov[2]; 5042 struct sockaddr_storage ss; 5043 5044 KASSERT(af == AF_INET || af == AF_INET6); 5045 5046 WG_TRACE(""); 5047 5048 if (af == AF_INET) { 5049 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 5050 struct ip *ip; 5051 5052 KASSERT(m->m_len >= sizeof(struct ip)); 5053 ip = mtod(m, struct ip *); 5054 sockaddr_in_init(sin, &ip->ip_dst, 0); 5055 } else { 5056 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 5057 struct ip6_hdr *ip6; 5058 5059 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 5060 ip6 = mtod(m, struct ip6_hdr *); 5061 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 5062 } 5063 5064 iov[0].iov_base = &ss; 5065 iov[0].iov_len = ss.ss_len; 5066 iov[1].iov_base = mtod(m, void *); 5067 iov[1].iov_len = m->m_len; 5068 5069 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5070 5071 /* Send decrypted packets to users via a tun. */ 5072 rumpuser_wg_send_user(wg->wg_user, iov, 2); 5073 5074 m_freem(m); 5075 } 5076 5077 static int 5078 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 5079 { 5080 int error; 5081 uint16_t old_port = wg->wg_listen_port; 5082 5083 if (port != 0 && old_port == port) 5084 return 0; 5085 5086 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5087 if (error == 0) 5088 wg->wg_listen_port = port; 5089 return error; 5090 } 5091 5092 /* 5093 * Receive user packets. 5094 */ 5095 void 5096 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5097 { 5098 struct ifnet *ifp = &wg->wg_if; 5099 struct mbuf *m; 5100 const struct sockaddr *dst; 5101 5102 WG_TRACE(""); 5103 5104 dst = iov[0].iov_base; 5105 5106 m = m_gethdr(M_DONTWAIT, MT_DATA); 5107 if (m == NULL) 5108 return; 5109 m->m_len = m->m_pkthdr.len = 0; 5110 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5111 5112 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5113 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5114 5115 (void)wg_output(ifp, m, dst, NULL); 5116 } 5117 5118 /* 5119 * Receive packets from a peer. 5120 */ 5121 void 5122 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5123 { 5124 struct mbuf *m; 5125 const struct sockaddr *src; 5126 int bound; 5127 5128 WG_TRACE(""); 5129 5130 src = iov[0].iov_base; 5131 5132 m = m_gethdr(M_DONTWAIT, MT_DATA); 5133 if (m == NULL) 5134 return; 5135 m->m_len = m->m_pkthdr.len = 0; 5136 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5137 5138 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5139 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5140 5141 bound = curlwp_bind(); 5142 wg_handle_packet(wg, m, src); 5143 curlwp_bindx(bound); 5144 } 5145 #endif /* WG_RUMPKERNEL */ 5146 5147 /* 5148 * Module infrastructure 5149 */ 5150 #include "if_module.h" 5151 5152 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5153
Indexes created Thu Sep 25 16:09:42 GMT 2025