if_wg.c revision 1.91
1 /* $NetBSD: if_wg.c,v 1.91 2024/07/25 16:45:36 christos Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.91 2024/07/25 16:45:36 christos Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <lib/libkern/libkern.h> 87 88 #include <net/bpf.h> 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/if_wg.h> 92 #include <net/pktqueue.h> 93 #include <net/route.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip.h> 99 #include <netinet/ip_var.h> 100 #include <netinet/udp.h> 101 #include <netinet/udp_var.h> 102 103 #ifdef INET6 104 #include <netinet/ip6.h> 105 #include <netinet6/in6_pcb.h> 106 #include <netinet6/in6_var.h> 107 #include <netinet6/ip6_var.h> 108 #include <netinet6/udp6_var.h> 109 #endif /* INET6 */ 110 111 #include <prop/proplib.h> 112 113 #include <crypto/blake2/blake2s.h> 114 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 115 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 116 #include <crypto/sodium/crypto_scalarmult.h> 117 118 #include "ioconf.h" 119 120 #ifdef WG_RUMPKERNEL 121 #include "wg_user.h" 122 #endif 123 124 /* 125 * Data structures 126 * - struct wg_softc is an instance of wg interfaces 127 * - It has a list of peers (struct wg_peer) 128 * - It has a threadpool job that sends/receives handshake messages and 129 * runs event handlers 130 * - It has its own two routing tables: one is for IPv4 and the other IPv6 131 * - struct wg_peer is a representative of a peer 132 * - It has a struct work to handle handshakes and timer tasks 133 * - It has a pair of session instances (struct wg_session) 134 * - It has a pair of endpoint instances (struct wg_sockaddr) 135 * - Normally one endpoint is used and the second one is used only on 136 * a peer migration (a change of peer's IP address) 137 * - It has a list of IP addresses and sub networks called allowedips 138 * (struct wg_allowedip) 139 * - A packets sent over a session is allowed if its destination matches 140 * any IP addresses or sub networks of the list 141 * - struct wg_session represents a session of a secure tunnel with a peer 142 * - Two instances of sessions belong to a peer; a stable session and a 143 * unstable session 144 * - A handshake process of a session always starts with a unstable instance 145 * - Once a session is established, its instance becomes stable and the 146 * other becomes unstable instead 147 * - Data messages are always sent via a stable session 148 * 149 * Locking notes: 150 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 151 * - Changes to the peer list are serialized by wg_lock 152 * - The peer list may be read with pserialize(9) and psref(9) 153 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 154 * => XXX replace by pserialize when routing table is psz-safe 155 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 156 * only in thread context and serializes: 157 * - the stable and unstable session pointers 158 * - all unstable session state 159 * - Packet processing may be done in softint context: 160 * - The stable session can be read under pserialize(9) or psref(9) 161 * - The stable session is always ESTABLISHED 162 * - On a session swap, we must wait for all readers to release a 163 * reference to a stable session before changing wgs_state and 164 * session states 165 * - Lock order: wg_lock -> wgp_lock 166 */ 167 168 169 #define WGLOG(level, fmt, args...) \ 170 log(level, "%s: " fmt, __func__, ##args) 171 172 #define WG_DEBUG 173 174 /* Debug options */ 175 #ifdef WG_DEBUG 176 /* Output debug logs */ 177 #ifndef WG_DEBUG_LOG 178 #define WG_DEBUG_LOG 179 #endif 180 /* Output trace logs */ 181 #ifndef WG_DEBUG_TRACE 182 #define WG_DEBUG_TRACE 183 #endif 184 /* Output hash values, etc. */ 185 #ifndef WG_DEBUG_DUMP 186 #define WG_DEBUG_DUMP 187 #endif 188 /* debug packets */ 189 #ifndef WG_DEBUG_PACKET 190 #define WG_DEBUG_PACKET 191 #endif 192 /* Make some internal parameters configurable for testing and debugging */ 193 #ifndef WG_DEBUG_PARAMS 194 #define WG_DEBUG_PARAMS 195 #endif 196 #endif /* WG_DEBUG */ 197 198 #ifndef WG_DEBUG 199 # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \ 200 defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS) || \ 201 defined(WG_DEBUG_PACKET) 202 # define WG_DEBUG 203 # endif 204 #endif 205 206 #ifdef WG_DEBUG 207 int wg_debug; 208 #define WG_DEBUG_FLAGS_LOG 1 209 #define WG_DEBUG_FLAGS_TRACE 2 210 #define WG_DEBUG_FLAGS_DUMP 4 211 #define WG_DEBUG_FLAGS_PACKET 8 212 #endif 213 214 215 #ifdef WG_DEBUG_TRACE 216 #define WG_TRACE(msg) do { \ 217 if (wg_debug & WG_DEBUG_FLAGS_TRACE) \ 218 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \ 219 } while (0) 220 #else 221 #define WG_TRACE(msg) __nothing 222 #endif 223 224 #ifdef WG_DEBUG_LOG 225 #define WG_DLOG(fmt, args...) do { \ 226 if (wg_debug & WG_DEBUG_FLAGS_LOG) \ 227 log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \ 228 } while (0) 229 #else 230 #define WG_DLOG(fmt, args...) __nothing 231 #endif 232 233 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 234 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 235 &(wgprc)->wgprc_curpps, 1)) { \ 236 log(level, fmt, ##args); \ 237 } \ 238 } while (0) 239 240 #ifdef WG_DEBUG_PARAMS 241 static bool wg_force_underload = false; 242 #endif 243 244 #ifdef WG_DEBUG_DUMP 245 246 static char enomem[10] = "[enomem]"; 247 248 static char * 249 gethexdump(const void *vp, size_t n) 250 { 251 char *buf; 252 const uint8_t *p = vp; 253 size_t i; 254 255 if (n > (SIZE_MAX - 1) / 3) 256 return enomem; 257 buf = kmem_alloc(3 * n + 1, KM_NOSLEEP); 258 if (buf == NULL) 259 return enomem; 260 for (i = 0; i < n; i++) 261 snprintf(buf + 3 * i, 3 + 1, " %02hhx", p[i]); 262 return buf; 263 } 264 265 static void 266 puthexdump(char *buf, const void *p, size_t n) 267 { 268 269 if (buf == NULL || buf == enomem) 270 return; 271 kmem_free(buf, 3*n + 1); 272 } 273 274 #ifdef WG_RUMPKERNEL 275 static void 276 wg_dump_buf(const char *func, const char *buf, const size_t size) 277 { 278 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 279 return; 280 281 char *hex = gethexdump(buf, size); 282 283 log(LOG_DEBUG, "%s: %s\n", func, hex); 284 puthexdump(hex, buf, size); 285 } 286 #endif 287 288 static void 289 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 290 const size_t size) 291 { 292 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 293 return; 294 295 char *hex = gethexdump(hash, size); 296 297 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex); 298 puthexdump(hex, hash, size); 299 } 300 301 #define WG_DUMP_HASH(name, hash) \ 302 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 303 #define WG_DUMP_HASH48(name, hash) \ 304 wg_dump_hash(__func__, name, hash, 48) 305 #define WG_DUMP_BUF(buf, size) \ 306 wg_dump_buf(__func__, buf, size) 307 #else 308 #define WG_DUMP_HASH(name, hash) __nothing 309 #define WG_DUMP_HASH48(name, hash) __nothing 310 #define WG_DUMP_BUF(buf, size) __nothing 311 #endif /* WG_DEBUG_DUMP */ 312 313 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 314 #define WG_MAX_PROPLEN 32766 315 316 #define WG_MTU 1420 317 #define WG_ALLOWEDIPS 16 318 319 #define CURVE25519_KEY_LEN 32 320 #define TAI64N_LEN sizeof(uint32_t) * 3 321 #define POLY1305_AUTHTAG_LEN 16 322 #define HMAC_BLOCK_LEN 64 323 324 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 325 /* [N] 4.3: Hash functions */ 326 #define NOISE_DHLEN 32 327 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 328 #define NOISE_HASHLEN 32 329 #define NOISE_BLOCKLEN 64 330 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 331 /* [N] 5.1: "k" */ 332 #define NOISE_CIPHER_KEY_LEN 32 333 /* 334 * [N] 9.2: "psk" 335 * "... psk is a 32-byte secret value provided by the application." 336 */ 337 #define NOISE_PRESHARED_KEY_LEN 32 338 339 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 340 #define WG_TIMESTAMP_LEN TAI64N_LEN 341 342 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 343 344 #define WG_COOKIE_LEN 16 345 #define WG_MAC_LEN 16 346 #define WG_RANDVAL_LEN 24 347 348 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 349 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 350 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 351 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 352 #define WG_HASH_LEN NOISE_HASHLEN 353 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 354 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 355 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 356 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 357 #define WG_DATA_KEY_LEN 32 358 #define WG_SALT_LEN 24 359 360 /* 361 * The protocol messages 362 */ 363 struct wg_msg { 364 uint32_t wgm_type; 365 } __packed; 366 367 /* [W] 5.4.2 First Message: Initiator to Responder */ 368 struct wg_msg_init { 369 uint32_t wgmi_type; 370 uint32_t wgmi_sender; 371 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 372 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 373 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 374 uint8_t wgmi_mac1[WG_MAC_LEN]; 375 uint8_t wgmi_mac2[WG_MAC_LEN]; 376 } __packed; 377 378 /* [W] 5.4.3 Second Message: Responder to Initiator */ 379 struct wg_msg_resp { 380 uint32_t wgmr_type; 381 uint32_t wgmr_sender; 382 uint32_t wgmr_receiver; 383 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 384 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 385 uint8_t wgmr_mac1[WG_MAC_LEN]; 386 uint8_t wgmr_mac2[WG_MAC_LEN]; 387 } __packed; 388 389 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 390 struct wg_msg_data { 391 uint32_t wgmd_type; 392 uint32_t wgmd_receiver; 393 uint64_t wgmd_counter; 394 uint32_t wgmd_packet[0]; 395 } __packed; 396 397 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 398 struct wg_msg_cookie { 399 uint32_t wgmc_type; 400 uint32_t wgmc_receiver; 401 uint8_t wgmc_salt[WG_SALT_LEN]; 402 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 403 } __packed; 404 405 #define WG_MSG_TYPE_INIT 1 406 #define WG_MSG_TYPE_RESP 2 407 #define WG_MSG_TYPE_COOKIE 3 408 #define WG_MSG_TYPE_DATA 4 409 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 410 411 /* Sliding windows */ 412 413 #define SLIWIN_BITS 2048u 414 #define SLIWIN_TYPE uint32_t 415 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 416 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 417 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 418 419 struct sliwin { 420 SLIWIN_TYPE B[SLIWIN_WORDS]; 421 uint64_t T; 422 }; 423 424 static void 425 sliwin_reset(struct sliwin *W) 426 { 427 428 memset(W, 0, sizeof(*W)); 429 } 430 431 static int 432 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 433 { 434 435 /* 436 * If it's more than one window older than the highest sequence 437 * number we've seen, reject. 438 */ 439 #ifdef __HAVE_ATOMIC64_LOADSTORE 440 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 441 return EAUTH; 442 #endif 443 444 /* 445 * Otherwise, we need to take the lock to decide, so don't 446 * reject just yet. Caller must serialize a call to 447 * sliwin_update in this case. 448 */ 449 return 0; 450 } 451 452 static int 453 sliwin_update(struct sliwin *W, uint64_t S) 454 { 455 unsigned word, bit; 456 457 /* 458 * If it's more than one window older than the highest sequence 459 * number we've seen, reject. 460 */ 461 if (S + SLIWIN_NPKT < W->T) 462 return EAUTH; 463 464 /* 465 * If it's higher than the highest sequence number we've seen, 466 * advance the window. 467 */ 468 if (S > W->T) { 469 uint64_t i = W->T / SLIWIN_BPW; 470 uint64_t j = S / SLIWIN_BPW; 471 unsigned k; 472 473 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 474 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 475 #ifdef __HAVE_ATOMIC64_LOADSTORE 476 atomic_store_relaxed(&W->T, S); 477 #else 478 W->T = S; 479 #endif 480 } 481 482 /* Test and set the bit -- if already set, reject. */ 483 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 484 bit = S % SLIWIN_BPW; 485 if (W->B[word] & (1UL << bit)) 486 return EAUTH; 487 W->B[word] |= 1U << bit; 488 489 /* Accept! */ 490 return 0; 491 } 492 493 struct wg_session { 494 struct wg_peer *wgs_peer; 495 struct psref_target 496 wgs_psref; 497 498 int wgs_state; 499 #define WGS_STATE_UNKNOWN 0 500 #define WGS_STATE_INIT_ACTIVE 1 501 #define WGS_STATE_INIT_PASSIVE 2 502 #define WGS_STATE_ESTABLISHED 3 503 #define WGS_STATE_DESTROYING 4 504 505 time_t wgs_time_established; 506 time_t wgs_time_last_data_sent; 507 bool wgs_is_initiator; 508 509 uint32_t wgs_local_index; 510 uint32_t wgs_remote_index; 511 #ifdef __HAVE_ATOMIC64_LOADSTORE 512 volatile uint64_t 513 wgs_send_counter; 514 #else 515 kmutex_t wgs_send_counter_lock; 516 uint64_t wgs_send_counter; 517 #endif 518 519 struct { 520 kmutex_t lock; 521 struct sliwin window; 522 } *wgs_recvwin; 523 524 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 525 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 526 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 527 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 528 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 529 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 530 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 531 }; 532 533 struct wg_sockaddr { 534 union { 535 struct sockaddr_storage _ss; 536 struct sockaddr _sa; 537 struct sockaddr_in _sin; 538 struct sockaddr_in6 _sin6; 539 }; 540 struct psref_target wgsa_psref; 541 }; 542 543 #define wgsatoss(wgsa) (&(wgsa)->_ss) 544 #define wgsatosa(wgsa) (&(wgsa)->_sa) 545 #define wgsatosin(wgsa) (&(wgsa)->_sin) 546 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 547 548 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 549 550 struct wg_peer; 551 struct wg_allowedip { 552 struct radix_node wga_nodes[2]; 553 struct wg_sockaddr _wga_sa_addr; 554 struct wg_sockaddr _wga_sa_mask; 555 #define wga_sa_addr _wga_sa_addr._sa 556 #define wga_sa_mask _wga_sa_mask._sa 557 558 int wga_family; 559 uint8_t wga_cidr; 560 union { 561 struct in_addr _ip4; 562 struct in6_addr _ip6; 563 } wga_addr; 564 #define wga_addr4 wga_addr._ip4 565 #define wga_addr6 wga_addr._ip6 566 567 struct wg_peer *wga_peer; 568 }; 569 570 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 571 572 struct wg_ppsratecheck { 573 struct timeval wgprc_lasttime; 574 int wgprc_curpps; 575 }; 576 577 struct wg_softc; 578 struct wg_peer { 579 struct wg_softc *wgp_sc; 580 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 581 struct pslist_entry wgp_peerlist_entry; 582 pserialize_t wgp_psz; 583 struct psref_target wgp_psref; 584 kmutex_t *wgp_lock; 585 kmutex_t *wgp_intr_lock; 586 587 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 588 struct wg_sockaddr *wgp_endpoint; 589 struct wg_sockaddr *wgp_endpoint0; 590 volatile unsigned wgp_endpoint_changing; 591 bool wgp_endpoint_available; 592 593 /* The preshared key (optional) */ 594 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 595 596 struct wg_session *wgp_session_stable; 597 struct wg_session *wgp_session_unstable; 598 599 /* first outgoing packet awaiting session initiation */ 600 struct mbuf *wgp_pending; 601 602 /* timestamp in big-endian */ 603 wg_timestamp_t wgp_timestamp_latest_init; 604 605 struct timespec wgp_last_handshake_time; 606 607 callout_t wgp_rekey_timer; 608 callout_t wgp_handshake_timeout_timer; 609 callout_t wgp_session_dtor_timer; 610 611 time_t wgp_handshake_start_time; 612 613 int wgp_n_allowedips; 614 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 615 616 time_t wgp_latest_cookie_time; 617 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 618 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 619 bool wgp_last_sent_mac1_valid; 620 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 621 bool wgp_last_sent_cookie_valid; 622 623 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 624 625 time_t wgp_last_genrandval_time; 626 uint32_t wgp_randval; 627 628 struct wg_ppsratecheck wgp_ppsratecheck; 629 630 struct work wgp_work; 631 unsigned int wgp_tasks; 632 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 633 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 634 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 635 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 636 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 637 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 638 }; 639 640 struct wg_ops; 641 642 struct wg_softc { 643 struct ifnet wg_if; 644 LIST_ENTRY(wg_softc) wg_list; 645 kmutex_t *wg_lock; 646 kmutex_t *wg_intr_lock; 647 krwlock_t *wg_rwlock; 648 649 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 650 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 651 652 int wg_npeers; 653 struct pslist_head wg_peers; 654 struct thmap *wg_peers_bypubkey; 655 struct thmap *wg_peers_byname; 656 struct thmap *wg_sessions_byindex; 657 uint16_t wg_listen_port; 658 659 struct threadpool *wg_threadpool; 660 661 struct threadpool_job wg_job; 662 int wg_upcalls; 663 #define WG_UPCALL_INET __BIT(0) 664 #define WG_UPCALL_INET6 __BIT(1) 665 666 #ifdef INET 667 struct socket *wg_so4; 668 struct radix_node_head *wg_rtable_ipv4; 669 #endif 670 #ifdef INET6 671 struct socket *wg_so6; 672 struct radix_node_head *wg_rtable_ipv6; 673 #endif 674 675 struct wg_ppsratecheck wg_ppsratecheck; 676 677 struct wg_ops *wg_ops; 678 679 #ifdef WG_RUMPKERNEL 680 struct wg_user *wg_user; 681 #endif 682 }; 683 684 /* [W] 6.1 Preliminaries */ 685 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 686 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 687 #define WG_REKEY_AFTER_TIME 120 688 #define WG_REJECT_AFTER_TIME 180 689 #define WG_REKEY_ATTEMPT_TIME 90 690 #define WG_REKEY_TIMEOUT 5 691 #define WG_KEEPALIVE_TIMEOUT 10 692 693 #define WG_COOKIE_TIME 120 694 #define WG_RANDVAL_TIME (2 * 60) 695 696 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 697 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 698 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 699 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 700 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 701 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 702 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 703 704 static struct mbuf * 705 wg_get_mbuf(size_t, size_t); 706 707 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 708 struct mbuf *); 709 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 710 const uint32_t, const uint8_t [WG_MAC_LEN], 711 const struct sockaddr *); 712 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 713 struct wg_session *, const struct wg_msg_init *); 714 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 715 716 static struct wg_peer * 717 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 718 struct psref *); 719 static struct wg_peer * 720 wg_lookup_peer_by_pubkey(struct wg_softc *, 721 const uint8_t [WG_STATIC_KEY_LEN], struct psref *); 722 723 static struct wg_session * 724 wg_lookup_session_by_index(struct wg_softc *, 725 const uint32_t, struct psref *); 726 727 static void wg_update_endpoint_if_necessary(struct wg_peer *, 728 const struct sockaddr *); 729 730 static void wg_schedule_rekey_timer(struct wg_peer *); 731 static void wg_schedule_session_dtor_timer(struct wg_peer *); 732 733 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 734 static void wg_calculate_keys(struct wg_session *, const bool); 735 736 static void wg_clear_states(struct wg_session *); 737 738 static void wg_get_peer(struct wg_peer *, struct psref *); 739 static void wg_put_peer(struct wg_peer *, struct psref *); 740 741 static int wg_send_so(struct wg_peer *, struct mbuf *); 742 static int wg_send_udp(struct wg_peer *, struct mbuf *); 743 static int wg_output(struct ifnet *, struct mbuf *, 744 const struct sockaddr *, const struct rtentry *); 745 static void wg_input(struct ifnet *, struct mbuf *, const int); 746 static int wg_ioctl(struct ifnet *, u_long, void *); 747 static int wg_bind_port(struct wg_softc *, const uint16_t); 748 static int wg_init(struct ifnet *); 749 #ifdef ALTQ 750 static void wg_start(struct ifnet *); 751 #endif 752 static void wg_stop(struct ifnet *, int); 753 754 static void wg_peer_work(struct work *, void *); 755 static void wg_job(struct threadpool_job *); 756 static void wgintr(void *); 757 static void wg_purge_pending_packets(struct wg_peer *); 758 759 static int wg_clone_create(struct if_clone *, int); 760 static int wg_clone_destroy(struct ifnet *); 761 762 struct wg_ops { 763 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 764 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 765 void (*input)(struct ifnet *, struct mbuf *, const int); 766 int (*bind_port)(struct wg_softc *, const uint16_t); 767 }; 768 769 struct wg_ops wg_ops_rumpkernel = { 770 .send_hs_msg = wg_send_so, 771 .send_data_msg = wg_send_udp, 772 .input = wg_input, 773 .bind_port = wg_bind_port, 774 }; 775 776 #ifdef WG_RUMPKERNEL 777 static bool wg_user_mode(struct wg_softc *); 778 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 779 780 static int wg_send_user(struct wg_peer *, struct mbuf *); 781 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 782 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 783 784 struct wg_ops wg_ops_rumpuser = { 785 .send_hs_msg = wg_send_user, 786 .send_data_msg = wg_send_user, 787 .input = wg_input_user, 788 .bind_port = wg_bind_port_user, 789 }; 790 #endif 791 792 #define WG_PEER_READER_FOREACH(wgp, wg) \ 793 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 794 wgp_peerlist_entry) 795 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 796 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 797 wgp_peerlist_entry) 798 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 799 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 800 #define WG_PEER_WRITER_REMOVE(wgp) \ 801 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 802 803 struct wg_route { 804 struct radix_node wgr_nodes[2]; 805 struct wg_peer *wgr_peer; 806 }; 807 808 static struct radix_node_head * 809 wg_rnh(struct wg_softc *wg, const int family) 810 { 811 812 switch (family) { 813 case AF_INET: 814 return wg->wg_rtable_ipv4; 815 #ifdef INET6 816 case AF_INET6: 817 return wg->wg_rtable_ipv6; 818 #endif 819 default: 820 return NULL; 821 } 822 } 823 824 825 /* 826 * Global variables 827 */ 828 static volatile unsigned wg_count __cacheline_aligned; 829 830 struct psref_class *wg_psref_class __read_mostly; 831 832 static struct if_clone wg_cloner = 833 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 834 835 static struct pktqueue *wg_pktq __read_mostly; 836 static struct workqueue *wg_wq __read_mostly; 837 838 void wgattach(int); 839 /* ARGSUSED */ 840 void 841 wgattach(int count) 842 { 843 /* 844 * Nothing to do here, initialization is handled by the 845 * module initialization code in wginit() below). 846 */ 847 } 848 849 static void 850 wginit(void) 851 { 852 853 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 854 855 if_clone_attach(&wg_cloner); 856 } 857 858 /* 859 * XXX Kludge: This should just happen in wginit, but workqueue_create 860 * cannot be run until after CPUs have been detected, and wginit runs 861 * before configure. 862 */ 863 static int 864 wginitqueues(void) 865 { 866 int error __diagused; 867 868 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 869 KASSERT(wg_pktq != NULL); 870 871 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 872 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 873 KASSERT(error == 0); 874 875 return 0; 876 } 877 878 static void 879 wg_guarantee_initialized(void) 880 { 881 static ONCE_DECL(init); 882 int error __diagused; 883 884 error = RUN_ONCE(&init, wginitqueues); 885 KASSERT(error == 0); 886 } 887 888 static int 889 wg_count_inc(void) 890 { 891 unsigned o, n; 892 893 do { 894 o = atomic_load_relaxed(&wg_count); 895 if (o == UINT_MAX) 896 return ENFILE; 897 n = o + 1; 898 } while (atomic_cas_uint(&wg_count, o, n) != o); 899 900 return 0; 901 } 902 903 static void 904 wg_count_dec(void) 905 { 906 unsigned c __diagused; 907 908 c = atomic_dec_uint_nv(&wg_count); 909 KASSERT(c != UINT_MAX); 910 } 911 912 static int 913 wgdetach(void) 914 { 915 916 /* Prevent new interface creation. */ 917 if_clone_detach(&wg_cloner); 918 919 /* Check whether there are any existing interfaces. */ 920 if (atomic_load_relaxed(&wg_count)) { 921 /* Back out -- reattach the cloner. */ 922 if_clone_attach(&wg_cloner); 923 return EBUSY; 924 } 925 926 /* No interfaces left. Nuke it. */ 927 workqueue_destroy(wg_wq); 928 pktq_destroy(wg_pktq); 929 psref_class_destroy(wg_psref_class); 930 931 return 0; 932 } 933 934 static void 935 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 936 uint8_t hash[WG_HASH_LEN]) 937 { 938 /* [W] 5.4: CONSTRUCTION */ 939 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 940 /* [W] 5.4: IDENTIFIER */ 941 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 942 struct blake2s state; 943 944 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 945 signature, strlen(signature)); 946 947 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 948 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 949 950 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 951 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 952 blake2s_update(&state, id, strlen(id)); 953 blake2s_final(&state, hash); 954 955 WG_DUMP_HASH("ckey", ckey); 956 WG_DUMP_HASH("hash", hash); 957 } 958 959 static void 960 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 961 const size_t inputsize) 962 { 963 struct blake2s state; 964 965 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 966 blake2s_update(&state, hash, WG_HASH_LEN); 967 blake2s_update(&state, input, inputsize); 968 blake2s_final(&state, hash); 969 } 970 971 static void 972 wg_algo_mac(uint8_t out[], const size_t outsize, 973 const uint8_t key[], const size_t keylen, 974 const uint8_t input1[], const size_t input1len, 975 const uint8_t input2[], const size_t input2len) 976 { 977 struct blake2s state; 978 979 blake2s_init(&state, outsize, key, keylen); 980 981 blake2s_update(&state, input1, input1len); 982 if (input2 != NULL) 983 blake2s_update(&state, input2, input2len); 984 blake2s_final(&state, out); 985 } 986 987 static void 988 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 989 const uint8_t input1[], const size_t input1len, 990 const uint8_t input2[], const size_t input2len) 991 { 992 struct blake2s state; 993 /* [W] 5.4: LABEL-MAC1 */ 994 const char *label = "mac1----"; 995 uint8_t key[WG_HASH_LEN]; 996 997 blake2s_init(&state, sizeof(key), NULL, 0); 998 blake2s_update(&state, label, strlen(label)); 999 blake2s_update(&state, input1, input1len); 1000 blake2s_final(&state, key); 1001 1002 blake2s_init(&state, outsize, key, sizeof(key)); 1003 if (input2 != NULL) 1004 blake2s_update(&state, input2, input2len); 1005 blake2s_final(&state, out); 1006 } 1007 1008 static void 1009 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 1010 const uint8_t input1[], const size_t input1len) 1011 { 1012 struct blake2s state; 1013 /* [W] 5.4: LABEL-COOKIE */ 1014 const char *label = "cookie--"; 1015 1016 blake2s_init(&state, outsize, NULL, 0); 1017 blake2s_update(&state, label, strlen(label)); 1018 blake2s_update(&state, input1, input1len); 1019 blake2s_final(&state, out); 1020 } 1021 1022 static void 1023 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 1024 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 1025 { 1026 1027 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1028 1029 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 1030 crypto_scalarmult_base(pubkey, privkey); 1031 } 1032 1033 static void 1034 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 1035 const uint8_t privkey[WG_STATIC_KEY_LEN], 1036 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 1037 { 1038 1039 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1040 1041 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1042 KASSERT(ret == 0); 1043 } 1044 1045 static void 1046 wg_algo_hmac(uint8_t out[], const size_t outlen, 1047 const uint8_t key[], const size_t keylen, 1048 const uint8_t in[], const size_t inlen) 1049 { 1050 #define IPAD 0x36 1051 #define OPAD 0x5c 1052 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1053 uint8_t ipad[HMAC_BLOCK_LEN]; 1054 uint8_t opad[HMAC_BLOCK_LEN]; 1055 size_t i; 1056 struct blake2s state; 1057 1058 KASSERT(outlen == WG_HASH_LEN); 1059 KASSERT(keylen <= HMAC_BLOCK_LEN); 1060 1061 memcpy(hmackey, key, keylen); 1062 1063 for (i = 0; i < sizeof(hmackey); i++) { 1064 ipad[i] = hmackey[i] ^ IPAD; 1065 opad[i] = hmackey[i] ^ OPAD; 1066 } 1067 1068 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1069 blake2s_update(&state, ipad, sizeof(ipad)); 1070 blake2s_update(&state, in, inlen); 1071 blake2s_final(&state, out); 1072 1073 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1074 blake2s_update(&state, opad, sizeof(opad)); 1075 blake2s_update(&state, out, WG_HASH_LEN); 1076 blake2s_final(&state, out); 1077 #undef IPAD 1078 #undef OPAD 1079 } 1080 1081 static void 1082 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1083 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1084 const uint8_t input[], const size_t inputlen) 1085 { 1086 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1087 uint8_t one[1]; 1088 1089 /* 1090 * [N] 4.3: "an input_key_material byte sequence with length 1091 * either zero bytes, 32 bytes, or DHLEN bytes." 1092 */ 1093 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1094 1095 WG_DUMP_HASH("ckey", ckey); 1096 if (input != NULL) 1097 WG_DUMP_HASH("input", input); 1098 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1099 input, inputlen); 1100 WG_DUMP_HASH("tmp1", tmp1); 1101 one[0] = 1; 1102 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1103 one, sizeof(one)); 1104 WG_DUMP_HASH("out1", out1); 1105 if (out2 == NULL) 1106 return; 1107 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1108 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1109 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1110 tmp2, sizeof(tmp2)); 1111 WG_DUMP_HASH("out2", out2); 1112 if (out3 == NULL) 1113 return; 1114 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1115 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1116 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1117 tmp2, sizeof(tmp2)); 1118 WG_DUMP_HASH("out3", out3); 1119 } 1120 1121 static void __noinline 1122 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1123 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1124 const uint8_t local_key[WG_STATIC_KEY_LEN], 1125 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1126 { 1127 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1128 1129 wg_algo_dh(dhout, local_key, remote_key); 1130 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1131 1132 WG_DUMP_HASH("dhout", dhout); 1133 WG_DUMP_HASH("ckey", ckey); 1134 if (cipher_key != NULL) 1135 WG_DUMP_HASH("cipher_key", cipher_key); 1136 } 1137 1138 static void 1139 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1140 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1141 const uint8_t auth[], size_t authlen) 1142 { 1143 uint8_t nonce[(32 + 64) / 8] = {0}; 1144 long long unsigned int outsize; 1145 int error __diagused; 1146 1147 le64enc(&nonce[4], counter); 1148 1149 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1150 plainsize, auth, authlen, NULL, nonce, key); 1151 KASSERT(error == 0); 1152 KASSERT(outsize == expected_outsize); 1153 } 1154 1155 static int 1156 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1157 const uint64_t counter, const uint8_t encrypted[], 1158 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1159 { 1160 uint8_t nonce[(32 + 64) / 8] = {0}; 1161 long long unsigned int outsize; 1162 int error; 1163 1164 le64enc(&nonce[4], counter); 1165 1166 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1167 encrypted, encryptedsize, auth, authlen, nonce, key); 1168 if (error == 0) 1169 KASSERT(outsize == expected_outsize); 1170 return error; 1171 } 1172 1173 static void 1174 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1175 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1176 const uint8_t auth[], size_t authlen, 1177 const uint8_t nonce[WG_SALT_LEN]) 1178 { 1179 long long unsigned int outsize; 1180 int error __diagused; 1181 1182 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1183 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1184 plain, plainsize, auth, authlen, NULL, nonce, key); 1185 KASSERT(error == 0); 1186 KASSERT(outsize == expected_outsize); 1187 } 1188 1189 static int 1190 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1191 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1192 const uint8_t auth[], size_t authlen, 1193 const uint8_t nonce[WG_SALT_LEN]) 1194 { 1195 long long unsigned int outsize; 1196 int error; 1197 1198 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1199 encrypted, encryptedsize, auth, authlen, nonce, key); 1200 if (error == 0) 1201 KASSERT(outsize == expected_outsize); 1202 return error; 1203 } 1204 1205 static void 1206 wg_algo_tai64n(wg_timestamp_t timestamp) 1207 { 1208 struct timespec ts; 1209 1210 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1211 getnanotime(&ts); 1212 /* TAI64 label in external TAI64 format */ 1213 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1214 /* second beginning from 1970 TAI */ 1215 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1216 /* nanosecond in big-endian format */ 1217 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1218 } 1219 1220 /* 1221 * wg_get_stable_session(wgp, psref) 1222 * 1223 * Get a passive reference to the current stable session, or 1224 * return NULL if there is no current stable session. 1225 * 1226 * The pointer is always there but the session is not necessarily 1227 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1228 * the session may transition from ESTABLISHED to DESTROYING while 1229 * holding the passive reference. 1230 */ 1231 static struct wg_session * 1232 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1233 { 1234 int s; 1235 struct wg_session *wgs; 1236 1237 s = pserialize_read_enter(); 1238 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1239 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1240 wgs = NULL; 1241 else 1242 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1243 pserialize_read_exit(s); 1244 1245 return wgs; 1246 } 1247 1248 static void 1249 wg_put_session(struct wg_session *wgs, struct psref *psref) 1250 { 1251 1252 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1253 } 1254 1255 static void 1256 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1257 { 1258 struct wg_peer *wgp = wgs->wgs_peer; 1259 struct wg_session *wgs0 __diagused; 1260 void *garbage; 1261 1262 KASSERT(mutex_owned(wgp->wgp_lock)); 1263 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1264 1265 /* Remove the session from the table. */ 1266 wgs0 = thmap_del(wg->wg_sessions_byindex, 1267 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1268 KASSERT(wgs0 == wgs); 1269 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1270 1271 /* Wait for passive references to drain. */ 1272 pserialize_perform(wgp->wgp_psz); 1273 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1274 1275 /* Free memory, zero state, and transition to UNKNOWN. */ 1276 thmap_gc(wg->wg_sessions_byindex, garbage); 1277 wg_clear_states(wgs); 1278 wgs->wgs_state = WGS_STATE_UNKNOWN; 1279 } 1280 1281 /* 1282 * wg_get_session_index(wg, wgs) 1283 * 1284 * Choose a session index for wgs->wgs_local_index, and store it 1285 * in wg's table of sessions by index. 1286 * 1287 * wgs must be the unstable session of its peer, and must be 1288 * transitioning out of the UNKNOWN state. 1289 */ 1290 static void 1291 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1292 { 1293 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1294 struct wg_session *wgs0; 1295 uint32_t index; 1296 1297 KASSERT(mutex_owned(wgp->wgp_lock)); 1298 KASSERT(wgs == wgp->wgp_session_unstable); 1299 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1300 1301 do { 1302 /* Pick a uniform random index. */ 1303 index = cprng_strong32(); 1304 1305 /* Try to take it. */ 1306 wgs->wgs_local_index = index; 1307 wgs0 = thmap_put(wg->wg_sessions_byindex, 1308 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1309 1310 /* If someone else beat us, start over. */ 1311 } while (__predict_false(wgs0 != wgs)); 1312 } 1313 1314 /* 1315 * wg_put_session_index(wg, wgs) 1316 * 1317 * Remove wgs from the table of sessions by index, wait for any 1318 * passive references to drain, and transition the session to the 1319 * UNKNOWN state. 1320 * 1321 * wgs must be the unstable session of its peer, and must not be 1322 * UNKNOWN or ESTABLISHED. 1323 */ 1324 static void 1325 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1326 { 1327 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1328 1329 KASSERT(mutex_owned(wgp->wgp_lock)); 1330 KASSERT(wgs == wgp->wgp_session_unstable); 1331 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1332 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1333 1334 wg_destroy_session(wg, wgs); 1335 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1336 } 1337 1338 /* 1339 * Handshake patterns 1340 * 1341 * [W] 5: "These messages use the "IK" pattern from Noise" 1342 * [N] 7.5. Interactive handshake patterns (fundamental) 1343 * "The first character refers to the initiators static key:" 1344 * "I = Static key for initiator Immediately transmitted to responder, 1345 * despite reduced or absent identity hiding" 1346 * "The second character refers to the responders static key:" 1347 * "K = Static key for responder Known to initiator" 1348 * "IK: 1349 * <- s 1350 * ... 1351 * -> e, es, s, ss 1352 * <- e, ee, se" 1353 * [N] 9.4. Pattern modifiers 1354 * "IKpsk2: 1355 * <- s 1356 * ... 1357 * -> e, es, s, ss 1358 * <- e, ee, se, psk" 1359 */ 1360 static void 1361 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1362 struct wg_session *wgs, struct wg_msg_init *wgmi) 1363 { 1364 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1365 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1366 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1367 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1368 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1369 1370 KASSERT(mutex_owned(wgp->wgp_lock)); 1371 KASSERT(wgs == wgp->wgp_session_unstable); 1372 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1373 1374 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1375 wgmi->wgmi_sender = wgs->wgs_local_index; 1376 1377 /* [W] 5.4.2: First Message: Initiator to Responder */ 1378 1379 /* Ci := HASH(CONSTRUCTION) */ 1380 /* Hi := HASH(Ci || IDENTIFIER) */ 1381 wg_init_key_and_hash(ckey, hash); 1382 /* Hi := HASH(Hi || Sr^pub) */ 1383 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1384 1385 WG_DUMP_HASH("hash", hash); 1386 1387 /* [N] 2.2: "e" */ 1388 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1389 wg_algo_generate_keypair(pubkey, privkey); 1390 /* Ci := KDF1(Ci, Ei^pub) */ 1391 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1392 /* msg.ephemeral := Ei^pub */ 1393 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1394 /* Hi := HASH(Hi || msg.ephemeral) */ 1395 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1396 1397 WG_DUMP_HASH("ckey", ckey); 1398 WG_DUMP_HASH("hash", hash); 1399 1400 /* [N] 2.2: "es" */ 1401 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1402 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1403 1404 /* [N] 2.2: "s" */ 1405 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1406 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1407 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1408 hash, sizeof(hash)); 1409 /* Hi := HASH(Hi || msg.static) */ 1410 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1411 1412 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1413 1414 /* [N] 2.2: "ss" */ 1415 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1416 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1417 1418 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1419 wg_timestamp_t timestamp; 1420 wg_algo_tai64n(timestamp); 1421 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1422 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1423 /* Hi := HASH(Hi || msg.timestamp) */ 1424 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1425 1426 /* [W] 5.4.4 Cookie MACs */ 1427 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1428 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1429 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1430 /* Need mac1 to decrypt a cookie from a cookie message */ 1431 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1432 sizeof(wgp->wgp_last_sent_mac1)); 1433 wgp->wgp_last_sent_mac1_valid = true; 1434 1435 if (wgp->wgp_latest_cookie_time == 0 || 1436 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1437 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1438 else { 1439 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1440 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1441 (const uint8_t *)wgmi, 1442 offsetof(struct wg_msg_init, wgmi_mac2), 1443 NULL, 0); 1444 } 1445 1446 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1447 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1448 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1449 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1450 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1451 } 1452 1453 static void __noinline 1454 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1455 const struct sockaddr *src) 1456 { 1457 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1458 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1459 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1460 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1461 struct wg_peer *wgp; 1462 struct wg_session *wgs; 1463 int error, ret; 1464 struct psref psref_peer; 1465 uint8_t mac1[WG_MAC_LEN]; 1466 1467 WG_TRACE("init msg received"); 1468 1469 wg_algo_mac_mac1(mac1, sizeof(mac1), 1470 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1471 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1472 1473 /* 1474 * [W] 5.3: Denial of Service Mitigation & Cookies 1475 * "the responder, ..., must always reject messages with an invalid 1476 * msg.mac1" 1477 */ 1478 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1479 WG_DLOG("mac1 is invalid\n"); 1480 return; 1481 } 1482 1483 /* 1484 * [W] 5.4.2: First Message: Initiator to Responder 1485 * "When the responder receives this message, it does the same 1486 * operations so that its final state variables are identical, 1487 * replacing the operands of the DH function to produce equivalent 1488 * values." 1489 * Note that the following comments of operations are just copies of 1490 * the initiator's ones. 1491 */ 1492 1493 /* Ci := HASH(CONSTRUCTION) */ 1494 /* Hi := HASH(Ci || IDENTIFIER) */ 1495 wg_init_key_and_hash(ckey, hash); 1496 /* Hi := HASH(Hi || Sr^pub) */ 1497 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1498 1499 /* [N] 2.2: "e" */ 1500 /* Ci := KDF1(Ci, Ei^pub) */ 1501 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1502 sizeof(wgmi->wgmi_ephemeral)); 1503 /* Hi := HASH(Hi || msg.ephemeral) */ 1504 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1505 1506 WG_DUMP_HASH("ckey", ckey); 1507 1508 /* [N] 2.2: "es" */ 1509 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1510 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1511 1512 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1513 1514 /* [N] 2.2: "s" */ 1515 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1516 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1517 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1518 if (error != 0) { 1519 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1520 "%s: wg_algo_aead_dec for secret key failed\n", 1521 if_name(&wg->wg_if)); 1522 return; 1523 } 1524 /* Hi := HASH(Hi || msg.static) */ 1525 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1526 1527 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1528 if (wgp == NULL) { 1529 WG_DLOG("peer not found\n"); 1530 return; 1531 } 1532 1533 /* 1534 * Lock the peer to serialize access to cookie state. 1535 * 1536 * XXX Can we safely avoid holding the lock across DH? Take it 1537 * just to verify mac2 and then unlock/DH/lock? 1538 */ 1539 mutex_enter(wgp->wgp_lock); 1540 1541 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1542 WG_TRACE("under load"); 1543 /* 1544 * [W] 5.3: Denial of Service Mitigation & Cookies 1545 * "the responder, ..., and when under load may reject messages 1546 * with an invalid msg.mac2. If the responder receives a 1547 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1548 * and is under load, it may respond with a cookie reply 1549 * message" 1550 */ 1551 uint8_t zero[WG_MAC_LEN] = {0}; 1552 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1553 WG_TRACE("sending a cookie message: no cookie included"); 1554 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1555 wgmi->wgmi_mac1, src); 1556 goto out; 1557 } 1558 if (!wgp->wgp_last_sent_cookie_valid) { 1559 WG_TRACE("sending a cookie message: no cookie sent ever"); 1560 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1561 wgmi->wgmi_mac1, src); 1562 goto out; 1563 } 1564 uint8_t mac2[WG_MAC_LEN]; 1565 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1566 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1567 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1568 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1569 WG_DLOG("mac2 is invalid\n"); 1570 goto out; 1571 } 1572 WG_TRACE("under load, but continue to sending"); 1573 } 1574 1575 /* [N] 2.2: "ss" */ 1576 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1577 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1578 1579 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1580 wg_timestamp_t timestamp; 1581 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1582 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1583 hash, sizeof(hash)); 1584 if (error != 0) { 1585 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1586 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1587 if_name(&wg->wg_if), wgp->wgp_name); 1588 goto out; 1589 } 1590 /* Hi := HASH(Hi || msg.timestamp) */ 1591 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1592 1593 /* 1594 * [W] 5.1 "The responder keeps track of the greatest timestamp 1595 * received per peer and discards packets containing 1596 * timestamps less than or equal to it." 1597 */ 1598 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1599 sizeof(timestamp)); 1600 if (ret <= 0) { 1601 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1602 "%s: peer %s: invalid init msg: timestamp is old\n", 1603 if_name(&wg->wg_if), wgp->wgp_name); 1604 goto out; 1605 } 1606 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1607 1608 /* 1609 * Message is good -- we're committing to handle it now, unless 1610 * we were already initiating a session. 1611 */ 1612 wgs = wgp->wgp_session_unstable; 1613 switch (wgs->wgs_state) { 1614 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1615 wg_get_session_index(wg, wgs); 1616 break; 1617 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1618 WG_TRACE("Session already initializing, ignoring the message"); 1619 goto out; 1620 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1621 WG_TRACE("Session already initializing, destroying old states"); 1622 wg_clear_states(wgs); 1623 /* keep session index */ 1624 break; 1625 case WGS_STATE_ESTABLISHED: /* can't happen */ 1626 panic("unstable session can't be established"); 1627 break; 1628 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1629 WG_TRACE("Session destroying, but force to clear"); 1630 callout_stop(&wgp->wgp_session_dtor_timer); 1631 wg_clear_states(wgs); 1632 /* keep session index */ 1633 break; 1634 default: 1635 panic("invalid session state: %d", wgs->wgs_state); 1636 } 1637 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1638 1639 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1640 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1641 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1642 sizeof(wgmi->wgmi_ephemeral)); 1643 1644 wg_update_endpoint_if_necessary(wgp, src); 1645 1646 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1647 1648 wg_calculate_keys(wgs, false); 1649 wg_clear_states(wgs); 1650 1651 out: 1652 mutex_exit(wgp->wgp_lock); 1653 wg_put_peer(wgp, &psref_peer); 1654 } 1655 1656 static struct socket * 1657 wg_get_so_by_af(struct wg_softc *wg, const int af) 1658 { 1659 1660 switch (af) { 1661 #ifdef INET 1662 case AF_INET: 1663 return wg->wg_so4; 1664 #endif 1665 #ifdef INET6 1666 case AF_INET6: 1667 return wg->wg_so6; 1668 #endif 1669 default: 1670 panic("wg: no such af: %d", af); 1671 } 1672 } 1673 1674 static struct socket * 1675 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1676 { 1677 1678 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1679 } 1680 1681 static struct wg_sockaddr * 1682 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1683 { 1684 struct wg_sockaddr *wgsa; 1685 int s; 1686 1687 s = pserialize_read_enter(); 1688 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1689 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1690 pserialize_read_exit(s); 1691 1692 return wgsa; 1693 } 1694 1695 static void 1696 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1697 { 1698 1699 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1700 } 1701 1702 static int 1703 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1704 { 1705 int error; 1706 struct socket *so; 1707 struct psref psref; 1708 struct wg_sockaddr *wgsa; 1709 1710 wgsa = wg_get_endpoint_sa(wgp, &psref); 1711 so = wg_get_so_by_peer(wgp, wgsa); 1712 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1713 wg_put_sa(wgp, wgsa, &psref); 1714 1715 return error; 1716 } 1717 1718 static int 1719 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1720 { 1721 int error; 1722 struct mbuf *m; 1723 struct wg_msg_init *wgmi; 1724 struct wg_session *wgs; 1725 1726 KASSERT(mutex_owned(wgp->wgp_lock)); 1727 1728 wgs = wgp->wgp_session_unstable; 1729 /* XXX pull dispatch out into wg_task_send_init_message */ 1730 switch (wgs->wgs_state) { 1731 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1732 wg_get_session_index(wg, wgs); 1733 break; 1734 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1735 WG_TRACE("Session already initializing, skip starting new one"); 1736 return EBUSY; 1737 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1738 WG_TRACE("Session already initializing, destroying old states"); 1739 wg_clear_states(wgs); 1740 /* keep session index */ 1741 break; 1742 case WGS_STATE_ESTABLISHED: /* can't happen */ 1743 panic("unstable session can't be established"); 1744 break; 1745 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1746 WG_TRACE("Session destroying"); 1747 /* XXX should wait? */ 1748 return EBUSY; 1749 } 1750 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1751 1752 m = m_gethdr(M_WAIT, MT_DATA); 1753 if (sizeof(*wgmi) > MHLEN) { 1754 m_clget(m, M_WAIT); 1755 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1756 } 1757 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1758 wgmi = mtod(m, struct wg_msg_init *); 1759 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1760 1761 error = wg->wg_ops->send_hs_msg(wgp, m); 1762 if (error == 0) { 1763 WG_TRACE("init msg sent"); 1764 1765 if (wgp->wgp_handshake_start_time == 0) 1766 wgp->wgp_handshake_start_time = time_uptime; 1767 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1768 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1769 } else { 1770 wg_put_session_index(wg, wgs); 1771 /* Initiation failed; toss packet waiting for it if any. */ 1772 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 1773 m_freem(m); 1774 } 1775 1776 return error; 1777 } 1778 1779 static void 1780 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1781 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1782 const struct wg_msg_init *wgmi) 1783 { 1784 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1785 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1786 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1787 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1788 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1789 1790 KASSERT(mutex_owned(wgp->wgp_lock)); 1791 KASSERT(wgs == wgp->wgp_session_unstable); 1792 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1793 1794 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1795 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1796 1797 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1798 wgmr->wgmr_sender = wgs->wgs_local_index; 1799 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1800 1801 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1802 1803 /* [N] 2.2: "e" */ 1804 /* Er^priv, Er^pub := DH-GENERATE() */ 1805 wg_algo_generate_keypair(pubkey, privkey); 1806 /* Cr := KDF1(Cr, Er^pub) */ 1807 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1808 /* msg.ephemeral := Er^pub */ 1809 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1810 /* Hr := HASH(Hr || msg.ephemeral) */ 1811 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1812 1813 WG_DUMP_HASH("ckey", ckey); 1814 WG_DUMP_HASH("hash", hash); 1815 1816 /* [N] 2.2: "ee" */ 1817 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1818 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1819 1820 /* [N] 2.2: "se" */ 1821 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1822 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1823 1824 /* [N] 9.2: "psk" */ 1825 { 1826 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1827 /* Cr, r, k := KDF3(Cr, Q) */ 1828 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1829 sizeof(wgp->wgp_psk)); 1830 /* Hr := HASH(Hr || r) */ 1831 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1832 } 1833 1834 /* msg.empty := AEAD(k, 0, e, Hr) */ 1835 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1836 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1837 /* Hr := HASH(Hr || msg.empty) */ 1838 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1839 1840 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1841 1842 /* [W] 5.4.4: Cookie MACs */ 1843 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1844 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1845 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1846 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1847 /* Need mac1 to decrypt a cookie from a cookie message */ 1848 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1849 sizeof(wgp->wgp_last_sent_mac1)); 1850 wgp->wgp_last_sent_mac1_valid = true; 1851 1852 if (wgp->wgp_latest_cookie_time == 0 || 1853 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1854 /* msg.mac2 := 0^16 */ 1855 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1856 else { 1857 /* msg.mac2 := MAC(Lm, msg_b) */ 1858 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1859 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1860 (const uint8_t *)wgmr, 1861 offsetof(struct wg_msg_resp, wgmr_mac2), 1862 NULL, 0); 1863 } 1864 1865 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1866 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1867 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1868 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1869 wgs->wgs_remote_index = wgmi->wgmi_sender; 1870 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1871 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1872 } 1873 1874 static void 1875 wg_swap_sessions(struct wg_peer *wgp) 1876 { 1877 struct wg_session *wgs, *wgs_prev; 1878 1879 KASSERT(mutex_owned(wgp->wgp_lock)); 1880 1881 wgs = wgp->wgp_session_unstable; 1882 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1883 1884 wgs_prev = wgp->wgp_session_stable; 1885 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1886 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1887 atomic_store_release(&wgp->wgp_session_stable, wgs); 1888 wgp->wgp_session_unstable = wgs_prev; 1889 } 1890 1891 static void __noinline 1892 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1893 const struct sockaddr *src) 1894 { 1895 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1896 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1897 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1898 struct wg_peer *wgp; 1899 struct wg_session *wgs; 1900 struct psref psref; 1901 int error; 1902 uint8_t mac1[WG_MAC_LEN]; 1903 struct wg_session *wgs_prev; 1904 struct mbuf *m; 1905 1906 wg_algo_mac_mac1(mac1, sizeof(mac1), 1907 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1908 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1909 1910 /* 1911 * [W] 5.3: Denial of Service Mitigation & Cookies 1912 * "the responder, ..., must always reject messages with an invalid 1913 * msg.mac1" 1914 */ 1915 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1916 WG_DLOG("mac1 is invalid\n"); 1917 return; 1918 } 1919 1920 WG_TRACE("resp msg received"); 1921 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1922 if (wgs == NULL) { 1923 WG_TRACE("No session found"); 1924 return; 1925 } 1926 1927 wgp = wgs->wgs_peer; 1928 1929 mutex_enter(wgp->wgp_lock); 1930 1931 /* If we weren't waiting for a handshake response, drop it. */ 1932 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1933 WG_TRACE("peer sent spurious handshake response, ignoring"); 1934 goto out; 1935 } 1936 1937 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1938 WG_TRACE("under load"); 1939 /* 1940 * [W] 5.3: Denial of Service Mitigation & Cookies 1941 * "the responder, ..., and when under load may reject messages 1942 * with an invalid msg.mac2. If the responder receives a 1943 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1944 * and is under load, it may respond with a cookie reply 1945 * message" 1946 */ 1947 uint8_t zero[WG_MAC_LEN] = {0}; 1948 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1949 WG_TRACE("sending a cookie message: no cookie included"); 1950 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1951 wgmr->wgmr_mac1, src); 1952 goto out; 1953 } 1954 if (!wgp->wgp_last_sent_cookie_valid) { 1955 WG_TRACE("sending a cookie message: no cookie sent ever"); 1956 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1957 wgmr->wgmr_mac1, src); 1958 goto out; 1959 } 1960 uint8_t mac2[WG_MAC_LEN]; 1961 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1962 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1963 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1964 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1965 WG_DLOG("mac2 is invalid\n"); 1966 goto out; 1967 } 1968 WG_TRACE("under load, but continue to sending"); 1969 } 1970 1971 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1972 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1973 1974 /* 1975 * [W] 5.4.3 Second Message: Responder to Initiator 1976 * "When the initiator receives this message, it does the same 1977 * operations so that its final state variables are identical, 1978 * replacing the operands of the DH function to produce equivalent 1979 * values." 1980 * Note that the following comments of operations are just copies of 1981 * the initiator's ones. 1982 */ 1983 1984 /* [N] 2.2: "e" */ 1985 /* Cr := KDF1(Cr, Er^pub) */ 1986 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1987 sizeof(wgmr->wgmr_ephemeral)); 1988 /* Hr := HASH(Hr || msg.ephemeral) */ 1989 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1990 1991 WG_DUMP_HASH("ckey", ckey); 1992 WG_DUMP_HASH("hash", hash); 1993 1994 /* [N] 2.2: "ee" */ 1995 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1996 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1997 wgmr->wgmr_ephemeral); 1998 1999 /* [N] 2.2: "se" */ 2000 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 2001 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 2002 2003 /* [N] 9.2: "psk" */ 2004 { 2005 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 2006 /* Cr, r, k := KDF3(Cr, Q) */ 2007 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 2008 sizeof(wgp->wgp_psk)); 2009 /* Hr := HASH(Hr || r) */ 2010 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 2011 } 2012 2013 { 2014 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 2015 /* msg.empty := AEAD(k, 0, e, Hr) */ 2016 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 2017 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 2018 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 2019 if (error != 0) { 2020 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2021 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 2022 if_name(&wg->wg_if), wgp->wgp_name); 2023 goto out; 2024 } 2025 /* Hr := HASH(Hr || msg.empty) */ 2026 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 2027 } 2028 2029 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 2030 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 2031 wgs->wgs_remote_index = wgmr->wgmr_sender; 2032 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 2033 2034 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 2035 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2036 wgs->wgs_time_established = time_uptime; 2037 wgs->wgs_time_last_data_sent = 0; 2038 wgs->wgs_is_initiator = true; 2039 wg_calculate_keys(wgs, true); 2040 wg_clear_states(wgs); 2041 WG_TRACE("WGS_STATE_ESTABLISHED"); 2042 2043 callout_stop(&wgp->wgp_handshake_timeout_timer); 2044 2045 wg_swap_sessions(wgp); 2046 KASSERT(wgs == wgp->wgp_session_stable); 2047 wgs_prev = wgp->wgp_session_unstable; 2048 getnanotime(&wgp->wgp_last_handshake_time); 2049 wgp->wgp_handshake_start_time = 0; 2050 wgp->wgp_last_sent_mac1_valid = false; 2051 wgp->wgp_last_sent_cookie_valid = false; 2052 2053 wg_schedule_rekey_timer(wgp); 2054 2055 wg_update_endpoint_if_necessary(wgp, src); 2056 2057 /* 2058 * If we had a data packet queued up, send it; otherwise send a 2059 * keepalive message -- either way we have to send something 2060 * immediately or else the responder will never answer. 2061 */ 2062 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2063 kpreempt_disable(); 2064 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2065 M_SETCTX(m, wgp); 2066 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2067 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2068 if_name(&wg->wg_if)); 2069 m_freem(m); 2070 } 2071 kpreempt_enable(); 2072 } else { 2073 wg_send_keepalive_msg(wgp, wgs); 2074 } 2075 2076 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2077 /* Wait for wg_get_stable_session to drain. */ 2078 pserialize_perform(wgp->wgp_psz); 2079 2080 /* Transition ESTABLISHED->DESTROYING. */ 2081 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2082 2083 /* We can't destroy the old session immediately */ 2084 wg_schedule_session_dtor_timer(wgp); 2085 } else { 2086 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2087 "state=%d", wgs_prev->wgs_state); 2088 } 2089 2090 out: 2091 mutex_exit(wgp->wgp_lock); 2092 wg_put_session(wgs, &psref); 2093 } 2094 2095 static int 2096 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2097 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2098 { 2099 int error; 2100 struct mbuf *m; 2101 struct wg_msg_resp *wgmr; 2102 2103 KASSERT(mutex_owned(wgp->wgp_lock)); 2104 KASSERT(wgs == wgp->wgp_session_unstable); 2105 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 2106 2107 m = m_gethdr(M_WAIT, MT_DATA); 2108 if (sizeof(*wgmr) > MHLEN) { 2109 m_clget(m, M_WAIT); 2110 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2111 } 2112 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2113 wgmr = mtod(m, struct wg_msg_resp *); 2114 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2115 2116 error = wg->wg_ops->send_hs_msg(wgp, m); 2117 if (error == 0) 2118 WG_TRACE("resp msg sent"); 2119 return error; 2120 } 2121 2122 static struct wg_peer * 2123 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2124 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2125 { 2126 struct wg_peer *wgp; 2127 2128 int s = pserialize_read_enter(); 2129 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2130 if (wgp != NULL) 2131 wg_get_peer(wgp, psref); 2132 pserialize_read_exit(s); 2133 2134 return wgp; 2135 } 2136 2137 static void 2138 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2139 struct wg_msg_cookie *wgmc, const uint32_t sender, 2140 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2141 { 2142 uint8_t cookie[WG_COOKIE_LEN]; 2143 uint8_t key[WG_HASH_LEN]; 2144 uint8_t addr[sizeof(struct in6_addr)]; 2145 size_t addrlen; 2146 uint16_t uh_sport; /* be */ 2147 2148 KASSERT(mutex_owned(wgp->wgp_lock)); 2149 2150 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2151 wgmc->wgmc_receiver = sender; 2152 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2153 2154 /* 2155 * [W] 5.4.7: Under Load: Cookie Reply Message 2156 * "The secret variable, Rm, changes every two minutes to a 2157 * random value" 2158 */ 2159 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2160 wgp->wgp_randval = cprng_strong32(); 2161 wgp->wgp_last_genrandval_time = time_uptime; 2162 } 2163 2164 switch (src->sa_family) { 2165 case AF_INET: { 2166 const struct sockaddr_in *sin = satocsin(src); 2167 addrlen = sizeof(sin->sin_addr); 2168 memcpy(addr, &sin->sin_addr, addrlen); 2169 uh_sport = sin->sin_port; 2170 break; 2171 } 2172 #ifdef INET6 2173 case AF_INET6: { 2174 const struct sockaddr_in6 *sin6 = satocsin6(src); 2175 addrlen = sizeof(sin6->sin6_addr); 2176 memcpy(addr, &sin6->sin6_addr, addrlen); 2177 uh_sport = sin6->sin6_port; 2178 break; 2179 } 2180 #endif 2181 default: 2182 panic("invalid af=%d", src->sa_family); 2183 } 2184 2185 wg_algo_mac(cookie, sizeof(cookie), 2186 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2187 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2188 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2189 sizeof(wg->wg_pubkey)); 2190 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2191 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2192 2193 /* Need to store to calculate mac2 */ 2194 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2195 wgp->wgp_last_sent_cookie_valid = true; 2196 } 2197 2198 static int 2199 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2200 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2201 const struct sockaddr *src) 2202 { 2203 int error; 2204 struct mbuf *m; 2205 struct wg_msg_cookie *wgmc; 2206 2207 KASSERT(mutex_owned(wgp->wgp_lock)); 2208 2209 m = m_gethdr(M_WAIT, MT_DATA); 2210 if (sizeof(*wgmc) > MHLEN) { 2211 m_clget(m, M_WAIT); 2212 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2213 } 2214 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2215 wgmc = mtod(m, struct wg_msg_cookie *); 2216 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2217 2218 error = wg->wg_ops->send_hs_msg(wgp, m); 2219 if (error == 0) 2220 WG_TRACE("cookie msg sent"); 2221 return error; 2222 } 2223 2224 static bool 2225 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2226 { 2227 #ifdef WG_DEBUG_PARAMS 2228 if (wg_force_underload) 2229 return true; 2230 #endif 2231 2232 /* 2233 * XXX we don't have a means of a load estimation. The purpose of 2234 * the mechanism is a DoS mitigation, so we consider frequent handshake 2235 * messages as (a kind of) load; if a message of the same type comes 2236 * to a peer within 1 second, we consider we are under load. 2237 */ 2238 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2239 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2240 return (time_uptime - last) == 0; 2241 } 2242 2243 static void 2244 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2245 { 2246 2247 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2248 2249 /* 2250 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2251 */ 2252 if (initiator) { 2253 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2254 wgs->wgs_chaining_key, NULL, 0); 2255 } else { 2256 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2257 wgs->wgs_chaining_key, NULL, 0); 2258 } 2259 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2260 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2261 } 2262 2263 static uint64_t 2264 wg_session_get_send_counter(struct wg_session *wgs) 2265 { 2266 #ifdef __HAVE_ATOMIC64_LOADSTORE 2267 return atomic_load_relaxed(&wgs->wgs_send_counter); 2268 #else 2269 uint64_t send_counter; 2270 2271 mutex_enter(&wgs->wgs_send_counter_lock); 2272 send_counter = wgs->wgs_send_counter; 2273 mutex_exit(&wgs->wgs_send_counter_lock); 2274 2275 return send_counter; 2276 #endif 2277 } 2278 2279 static uint64_t 2280 wg_session_inc_send_counter(struct wg_session *wgs) 2281 { 2282 #ifdef __HAVE_ATOMIC64_LOADSTORE 2283 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2284 #else 2285 uint64_t send_counter; 2286 2287 mutex_enter(&wgs->wgs_send_counter_lock); 2288 send_counter = wgs->wgs_send_counter++; 2289 mutex_exit(&wgs->wgs_send_counter_lock); 2290 2291 return send_counter; 2292 #endif 2293 } 2294 2295 static void 2296 wg_clear_states(struct wg_session *wgs) 2297 { 2298 2299 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2300 2301 wgs->wgs_send_counter = 0; 2302 sliwin_reset(&wgs->wgs_recvwin->window); 2303 2304 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2305 wgs_clear(handshake_hash); 2306 wgs_clear(chaining_key); 2307 wgs_clear(ephemeral_key_pub); 2308 wgs_clear(ephemeral_key_priv); 2309 wgs_clear(ephemeral_key_peer); 2310 #undef wgs_clear 2311 } 2312 2313 static struct wg_session * 2314 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2315 struct psref *psref) 2316 { 2317 struct wg_session *wgs; 2318 2319 int s = pserialize_read_enter(); 2320 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2321 if (wgs != NULL) { 2322 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2323 WGS_STATE_UNKNOWN); 2324 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2325 } 2326 pserialize_read_exit(s); 2327 2328 return wgs; 2329 } 2330 2331 static void 2332 wg_schedule_rekey_timer(struct wg_peer *wgp) 2333 { 2334 int timeout = MIN(wg_rekey_after_time, (unsigned)(INT_MAX / hz)); 2335 2336 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2337 } 2338 2339 static void 2340 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2341 { 2342 struct mbuf *m; 2343 2344 /* 2345 * [W] 6.5 Passive Keepalive 2346 * "A keepalive message is simply a transport data message with 2347 * a zero-length encapsulated encrypted inner-packet." 2348 */ 2349 WG_TRACE(""); 2350 m = m_gethdr(M_WAIT, MT_DATA); 2351 wg_send_data_msg(wgp, wgs, m); 2352 } 2353 2354 static bool 2355 wg_need_to_send_init_message(struct wg_session *wgs) 2356 { 2357 /* 2358 * [W] 6.2 Transport Message Limits 2359 * "if a peer is the initiator of a current secure session, 2360 * WireGuard will send a handshake initiation message to begin 2361 * a new secure session ... if after receiving a transport data 2362 * message, the current secure session is (REJECT-AFTER-TIME 2363 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2364 * not yet acted upon this event." 2365 */ 2366 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2367 (time_uptime - wgs->wgs_time_established) >= 2368 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2369 } 2370 2371 static void 2372 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2373 { 2374 2375 mutex_enter(wgp->wgp_intr_lock); 2376 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2377 if (wgp->wgp_tasks == 0) 2378 /* 2379 * XXX If the current CPU is already loaded -- e.g., if 2380 * there's already a bunch of handshakes queued up -- 2381 * consider tossing this over to another CPU to 2382 * distribute the load. 2383 */ 2384 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2385 wgp->wgp_tasks |= task; 2386 mutex_exit(wgp->wgp_intr_lock); 2387 } 2388 2389 static void 2390 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2391 { 2392 struct wg_sockaddr *wgsa_prev; 2393 2394 WG_TRACE("Changing endpoint"); 2395 2396 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2397 wgsa_prev = wgp->wgp_endpoint; 2398 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2399 wgp->wgp_endpoint0 = wgsa_prev; 2400 atomic_store_release(&wgp->wgp_endpoint_available, true); 2401 2402 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2403 } 2404 2405 static bool 2406 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2407 { 2408 uint16_t packet_len; 2409 const struct ip *ip; 2410 2411 if (__predict_false(decrypted_len < sizeof(*ip))) { 2412 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2413 sizeof(*ip)); 2414 return false; 2415 } 2416 2417 ip = (const struct ip *)packet; 2418 if (ip->ip_v == 4) 2419 *af = AF_INET; 2420 else if (ip->ip_v == 6) 2421 *af = AF_INET6; 2422 else { 2423 WG_DLOG("ip_v=%d\n", ip->ip_v); 2424 return false; 2425 } 2426 2427 WG_DLOG("af=%d\n", *af); 2428 2429 switch (*af) { 2430 #ifdef INET 2431 case AF_INET: 2432 packet_len = ntohs(ip->ip_len); 2433 break; 2434 #endif 2435 #ifdef INET6 2436 case AF_INET6: { 2437 const struct ip6_hdr *ip6; 2438 2439 if (__predict_false(decrypted_len < sizeof(*ip6))) { 2440 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2441 sizeof(*ip6)); 2442 return false; 2443 } 2444 2445 ip6 = (const struct ip6_hdr *)packet; 2446 packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); 2447 break; 2448 } 2449 #endif 2450 default: 2451 return false; 2452 } 2453 2454 if (packet_len > decrypted_len) { 2455 WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len, 2456 decrypted_len); 2457 return false; 2458 } 2459 2460 return true; 2461 } 2462 2463 static bool 2464 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2465 int af, char *packet) 2466 { 2467 struct sockaddr_storage ss; 2468 struct sockaddr *sa; 2469 struct psref psref; 2470 struct wg_peer *wgp; 2471 bool ok; 2472 2473 /* 2474 * II CRYPTOKEY ROUTING 2475 * "it will only accept it if its source IP resolves in the 2476 * table to the public key used in the secure session for 2477 * decrypting it." 2478 */ 2479 2480 if (af == AF_INET) { 2481 const struct ip *ip = (const struct ip *)packet; 2482 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2483 sockaddr_in_init(sin, &ip->ip_src, 0); 2484 sa = sintosa(sin); 2485 #ifdef INET6 2486 } else { 2487 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2488 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2489 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2490 sa = sin6tosa(sin6); 2491 #endif 2492 } 2493 2494 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2495 ok = (wgp == wgp_expected); 2496 if (wgp != NULL) 2497 wg_put_peer(wgp, &psref); 2498 2499 return ok; 2500 } 2501 2502 static void 2503 wg_session_dtor_timer(void *arg) 2504 { 2505 struct wg_peer *wgp = arg; 2506 2507 WG_TRACE("enter"); 2508 2509 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2510 } 2511 2512 static void 2513 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2514 { 2515 2516 /* 1 second grace period */ 2517 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2518 } 2519 2520 static bool 2521 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2522 { 2523 if (sa1->sa_family != sa2->sa_family) 2524 return false; 2525 2526 switch (sa1->sa_family) { 2527 #ifdef INET 2528 case AF_INET: 2529 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2530 #endif 2531 #ifdef INET6 2532 case AF_INET6: 2533 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2534 #endif 2535 default: 2536 return false; 2537 } 2538 } 2539 2540 static void 2541 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2542 const struct sockaddr *src) 2543 { 2544 struct wg_sockaddr *wgsa; 2545 struct psref psref; 2546 2547 wgsa = wg_get_endpoint_sa(wgp, &psref); 2548 2549 #ifdef WG_DEBUG_LOG 2550 char oldaddr[128], newaddr[128]; 2551 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2552 sockaddr_format(src, newaddr, sizeof(newaddr)); 2553 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2554 #endif 2555 2556 /* 2557 * III: "Since the packet has authenticated correctly, the source IP of 2558 * the outer UDP/IP packet is used to update the endpoint for peer..." 2559 */ 2560 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2561 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2562 /* XXX We can't change the endpoint twice in a short period */ 2563 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2564 wg_change_endpoint(wgp, src); 2565 } 2566 } 2567 2568 wg_put_sa(wgp, wgsa, &psref); 2569 } 2570 2571 static void __noinline 2572 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2573 const struct sockaddr *src) 2574 { 2575 struct wg_msg_data *wgmd; 2576 char *encrypted_buf = NULL, *decrypted_buf; 2577 size_t encrypted_len, decrypted_len; 2578 struct wg_session *wgs; 2579 struct wg_peer *wgp; 2580 int state; 2581 size_t mlen; 2582 struct psref psref; 2583 int error, af; 2584 bool success, free_encrypted_buf = false, ok; 2585 struct mbuf *n; 2586 2587 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2588 wgmd = mtod(m, struct wg_msg_data *); 2589 2590 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2591 WG_TRACE("data"); 2592 2593 /* Find the putative session, or drop. */ 2594 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2595 if (wgs == NULL) { 2596 WG_TRACE("No session found"); 2597 m_freem(m); 2598 return; 2599 } 2600 2601 /* 2602 * We are only ready to handle data when in INIT_PASSIVE, 2603 * ESTABLISHED, or DESTROYING. All transitions out of that 2604 * state dissociate the session index and drain psrefs. 2605 */ 2606 state = atomic_load_relaxed(&wgs->wgs_state); 2607 switch (state) { 2608 case WGS_STATE_UNKNOWN: 2609 panic("wg session %p in unknown state has session index %u", 2610 wgs, wgmd->wgmd_receiver); 2611 case WGS_STATE_INIT_ACTIVE: 2612 WG_TRACE("not yet ready for data"); 2613 goto out; 2614 case WGS_STATE_INIT_PASSIVE: 2615 case WGS_STATE_ESTABLISHED: 2616 case WGS_STATE_DESTROYING: 2617 break; 2618 } 2619 2620 /* 2621 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2622 * to update the endpoint if authentication succeeds. 2623 */ 2624 wgp = wgs->wgs_peer; 2625 2626 /* 2627 * Reject outrageously wrong sequence numbers before doing any 2628 * crypto work or taking any locks. 2629 */ 2630 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2631 le64toh(wgmd->wgmd_counter)); 2632 if (error) { 2633 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2634 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2635 if_name(&wg->wg_if), wgp->wgp_name, 2636 le64toh(wgmd->wgmd_counter)); 2637 goto out; 2638 } 2639 2640 /* Ensure the payload and authenticator are contiguous. */ 2641 mlen = m_length(m); 2642 encrypted_len = mlen - sizeof(*wgmd); 2643 if (encrypted_len < WG_AUTHTAG_LEN) { 2644 WG_DLOG("Short encrypted_len: %zu\n", encrypted_len); 2645 goto out; 2646 } 2647 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2648 if (success) { 2649 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2650 } else { 2651 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2652 if (encrypted_buf == NULL) { 2653 WG_DLOG("failed to allocate encrypted_buf\n"); 2654 goto out; 2655 } 2656 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2657 free_encrypted_buf = true; 2658 } 2659 /* m_ensure_contig may change m regardless of its result */ 2660 KASSERT(m->m_len >= sizeof(*wgmd)); 2661 wgmd = mtod(m, struct wg_msg_data *); 2662 2663 #ifdef WG_DEBUG_PACKET 2664 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2665 hexdump(printf, "incoming packet", encrypted_buf, 2666 encrypted_len); 2667 } 2668 #endif 2669 /* 2670 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2671 * a zero-length buffer (XXX). Drop if plaintext is longer 2672 * than MCLBYTES (XXX). 2673 */ 2674 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2675 if (decrypted_len > MCLBYTES) { 2676 /* FIXME handle larger data than MCLBYTES */ 2677 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2678 goto out; 2679 } 2680 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2681 if (n == NULL) { 2682 WG_DLOG("wg_get_mbuf failed\n"); 2683 goto out; 2684 } 2685 decrypted_buf = mtod(n, char *); 2686 2687 /* Decrypt and verify the packet. */ 2688 WG_DLOG("mlen=%zu, encrypted_len=%zu\n", mlen, encrypted_len); 2689 error = wg_algo_aead_dec(decrypted_buf, 2690 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2691 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2692 encrypted_len, NULL, 0); 2693 if (error != 0) { 2694 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2695 "%s: peer %s: failed to wg_algo_aead_dec\n", 2696 if_name(&wg->wg_if), wgp->wgp_name); 2697 m_freem(n); 2698 goto out; 2699 } 2700 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2701 2702 /* Packet is genuine. Reject it if a replay or just too old. */ 2703 mutex_enter(&wgs->wgs_recvwin->lock); 2704 error = sliwin_update(&wgs->wgs_recvwin->window, 2705 le64toh(wgmd->wgmd_counter)); 2706 mutex_exit(&wgs->wgs_recvwin->lock); 2707 if (error) { 2708 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2709 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2710 if_name(&wg->wg_if), wgp->wgp_name, 2711 le64toh(wgmd->wgmd_counter)); 2712 m_freem(n); 2713 goto out; 2714 } 2715 2716 #ifdef WG_DEBUG_PACKET 2717 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2718 hexdump(printf, "tkey_recv", wgs->wgs_tkey_recv, 2719 sizeof(wgs->wgs_tkey_recv)); 2720 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 2721 hexdump(printf, "decrypted_buf", decrypted_buf, 2722 decrypted_len); 2723 } 2724 #endif 2725 /* We're done with m now; free it and chuck the pointers. */ 2726 m_freem(m); 2727 m = NULL; 2728 wgmd = NULL; 2729 2730 /* 2731 * Validate the encapsulated packet header and get the address 2732 * family, or drop. 2733 */ 2734 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2735 if (!ok) { 2736 m_freem(n); 2737 goto out; 2738 } 2739 2740 /* 2741 * The packet is genuine. Update the peer's endpoint if the 2742 * source address changed. 2743 * 2744 * XXX How to prevent DoS by replaying genuine packets from the 2745 * wrong source address? 2746 */ 2747 wg_update_endpoint_if_necessary(wgp, src); 2748 2749 /* Submit it into our network stack if routable. */ 2750 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2751 if (ok) { 2752 wg->wg_ops->input(&wg->wg_if, n, af); 2753 } else { 2754 char addrstr[INET6_ADDRSTRLEN]; 2755 memset(addrstr, 0, sizeof(addrstr)); 2756 if (af == AF_INET) { 2757 const struct ip *ip = (const struct ip *)decrypted_buf; 2758 IN_PRINT(addrstr, &ip->ip_src); 2759 #ifdef INET6 2760 } else if (af == AF_INET6) { 2761 const struct ip6_hdr *ip6 = 2762 (const struct ip6_hdr *)decrypted_buf; 2763 IN6_PRINT(addrstr, &ip6->ip6_src); 2764 #endif 2765 } 2766 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2767 "%s: peer %s: invalid source address (%s)\n", 2768 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2769 m_freem(n); 2770 /* 2771 * The inner address is invalid however the session is valid 2772 * so continue the session processing below. 2773 */ 2774 } 2775 n = NULL; 2776 2777 /* Update the state machine if necessary. */ 2778 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2779 /* 2780 * We were waiting for the initiator to send their 2781 * first data transport message, and that has happened. 2782 * Schedule a task to establish this session. 2783 */ 2784 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2785 } else { 2786 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2787 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2788 } 2789 /* 2790 * [W] 6.5 Passive Keepalive 2791 * "If a peer has received a validly-authenticated transport 2792 * data message (section 5.4.6), but does not have any packets 2793 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2794 * a keepalive message." 2795 */ 2796 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2797 (uintmax_t)time_uptime, 2798 (uintmax_t)wgs->wgs_time_last_data_sent); 2799 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2800 wg_keepalive_timeout) { 2801 WG_TRACE("Schedule sending keepalive message"); 2802 /* 2803 * We can't send a keepalive message here to avoid 2804 * a deadlock; we already hold the solock of a socket 2805 * that is used to send the message. 2806 */ 2807 wg_schedule_peer_task(wgp, 2808 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2809 } 2810 } 2811 out: 2812 wg_put_session(wgs, &psref); 2813 m_freem(m); 2814 if (free_encrypted_buf) 2815 kmem_intr_free(encrypted_buf, encrypted_len); 2816 } 2817 2818 static void __noinline 2819 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2820 { 2821 struct wg_session *wgs; 2822 struct wg_peer *wgp; 2823 struct psref psref; 2824 int error; 2825 uint8_t key[WG_HASH_LEN]; 2826 uint8_t cookie[WG_COOKIE_LEN]; 2827 2828 WG_TRACE("cookie msg received"); 2829 2830 /* Find the putative session. */ 2831 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2832 if (wgs == NULL) { 2833 WG_TRACE("No session found"); 2834 return; 2835 } 2836 2837 /* Lock the peer so we can update the cookie state. */ 2838 wgp = wgs->wgs_peer; 2839 mutex_enter(wgp->wgp_lock); 2840 2841 if (!wgp->wgp_last_sent_mac1_valid) { 2842 WG_TRACE("No valid mac1 sent (or expired)"); 2843 goto out; 2844 } 2845 2846 /* Decrypt the cookie and store it for later handshake retry. */ 2847 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2848 sizeof(wgp->wgp_pubkey)); 2849 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2850 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2851 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2852 wgmc->wgmc_salt); 2853 if (error != 0) { 2854 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2855 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 2856 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 2857 goto out; 2858 } 2859 /* 2860 * [W] 6.6: Interaction with Cookie Reply System 2861 * "it should simply store the decrypted cookie value from the cookie 2862 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2863 * timer for retrying a handshake initiation message." 2864 */ 2865 wgp->wgp_latest_cookie_time = time_uptime; 2866 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2867 out: 2868 mutex_exit(wgp->wgp_lock); 2869 wg_put_session(wgs, &psref); 2870 } 2871 2872 static struct mbuf * 2873 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2874 { 2875 struct wg_msg wgm; 2876 size_t mbuflen; 2877 size_t msglen; 2878 2879 /* 2880 * Get the mbuf chain length. It is already guaranteed, by 2881 * wg_overudp_cb, to be large enough for a struct wg_msg. 2882 */ 2883 mbuflen = m_length(m); 2884 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2885 2886 /* 2887 * Copy the message header (32-bit message type) out -- we'll 2888 * worry about contiguity and alignment later. 2889 */ 2890 m_copydata(m, 0, sizeof(wgm), &wgm); 2891 switch (le32toh(wgm.wgm_type)) { 2892 case WG_MSG_TYPE_INIT: 2893 msglen = sizeof(struct wg_msg_init); 2894 break; 2895 case WG_MSG_TYPE_RESP: 2896 msglen = sizeof(struct wg_msg_resp); 2897 break; 2898 case WG_MSG_TYPE_COOKIE: 2899 msglen = sizeof(struct wg_msg_cookie); 2900 break; 2901 case WG_MSG_TYPE_DATA: 2902 msglen = sizeof(struct wg_msg_data); 2903 break; 2904 default: 2905 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2906 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 2907 le32toh(wgm.wgm_type)); 2908 goto error; 2909 } 2910 2911 /* Verify the mbuf chain is long enough for this type of message. */ 2912 if (__predict_false(mbuflen < msglen)) { 2913 WG_DLOG("Invalid msg size: mbuflen=%zu type=%u\n", mbuflen, 2914 le32toh(wgm.wgm_type)); 2915 goto error; 2916 } 2917 2918 /* Make the message header contiguous if necessary. */ 2919 if (__predict_false(m->m_len < msglen)) { 2920 m = m_pullup(m, msglen); 2921 if (m == NULL) 2922 return NULL; 2923 } 2924 2925 return m; 2926 2927 error: 2928 m_freem(m); 2929 return NULL; 2930 } 2931 2932 static void 2933 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2934 const struct sockaddr *src) 2935 { 2936 struct wg_msg *wgm; 2937 2938 KASSERT(curlwp->l_pflag & LP_BOUND); 2939 2940 m = wg_validate_msg_header(wg, m); 2941 if (__predict_false(m == NULL)) 2942 return; 2943 2944 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2945 wgm = mtod(m, struct wg_msg *); 2946 switch (le32toh(wgm->wgm_type)) { 2947 case WG_MSG_TYPE_INIT: 2948 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2949 break; 2950 case WG_MSG_TYPE_RESP: 2951 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2952 break; 2953 case WG_MSG_TYPE_COOKIE: 2954 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2955 break; 2956 case WG_MSG_TYPE_DATA: 2957 wg_handle_msg_data(wg, m, src); 2958 /* wg_handle_msg_data frees m for us */ 2959 return; 2960 default: 2961 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2962 } 2963 2964 m_freem(m); 2965 } 2966 2967 static void 2968 wg_receive_packets(struct wg_softc *wg, const int af) 2969 { 2970 2971 for (;;) { 2972 int error, flags; 2973 struct socket *so; 2974 struct mbuf *m = NULL; 2975 struct uio dummy_uio; 2976 struct mbuf *paddr = NULL; 2977 struct sockaddr *src; 2978 2979 so = wg_get_so_by_af(wg, af); 2980 flags = MSG_DONTWAIT; 2981 dummy_uio.uio_resid = 1000000000; 2982 2983 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2984 &flags); 2985 if (error || m == NULL) { 2986 //if (error == EWOULDBLOCK) 2987 return; 2988 } 2989 2990 KASSERT(paddr != NULL); 2991 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2992 src = mtod(paddr, struct sockaddr *); 2993 2994 wg_handle_packet(wg, m, src); 2995 } 2996 } 2997 2998 static void 2999 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 3000 { 3001 3002 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 3003 } 3004 3005 static void 3006 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 3007 { 3008 3009 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 3010 } 3011 3012 static void 3013 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 3014 { 3015 struct wg_session *wgs; 3016 3017 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 3018 3019 KASSERT(mutex_owned(wgp->wgp_lock)); 3020 3021 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 3022 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 3023 if_name(&wg->wg_if)); 3024 /* XXX should do something? */ 3025 return; 3026 } 3027 3028 wgs = wgp->wgp_session_stable; 3029 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 3030 /* XXX What if the unstable session is already INIT_ACTIVE? */ 3031 wg_send_handshake_msg_init(wg, wgp); 3032 } else { 3033 /* rekey */ 3034 wgs = wgp->wgp_session_unstable; 3035 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3036 wg_send_handshake_msg_init(wg, wgp); 3037 } 3038 } 3039 3040 static void 3041 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 3042 { 3043 struct wg_session *wgs; 3044 3045 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 3046 3047 KASSERT(mutex_owned(wgp->wgp_lock)); 3048 KASSERT(wgp->wgp_handshake_start_time != 0); 3049 3050 wgs = wgp->wgp_session_unstable; 3051 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3052 return; 3053 3054 /* 3055 * XXX no real need to assign a new index here, but we do need 3056 * to transition to UNKNOWN temporarily 3057 */ 3058 wg_put_session_index(wg, wgs); 3059 3060 /* [W] 6.4 Handshake Initiation Retransmission */ 3061 if ((time_uptime - wgp->wgp_handshake_start_time) > 3062 wg_rekey_attempt_time) { 3063 /* Give up handshaking */ 3064 wgp->wgp_handshake_start_time = 0; 3065 WG_TRACE("give up"); 3066 3067 /* 3068 * If a new data packet comes, handshaking will be retried 3069 * and a new session would be established at that time, 3070 * however we don't want to send pending packets then. 3071 */ 3072 wg_purge_pending_packets(wgp); 3073 return; 3074 } 3075 3076 wg_task_send_init_message(wg, wgp); 3077 } 3078 3079 static void 3080 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3081 { 3082 struct wg_session *wgs, *wgs_prev; 3083 struct mbuf *m; 3084 3085 KASSERT(mutex_owned(wgp->wgp_lock)); 3086 3087 wgs = wgp->wgp_session_unstable; 3088 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3089 /* XXX Can this happen? */ 3090 return; 3091 3092 wgs->wgs_state = WGS_STATE_ESTABLISHED; 3093 wgs->wgs_time_established = time_uptime; 3094 wgs->wgs_time_last_data_sent = 0; 3095 wgs->wgs_is_initiator = false; 3096 WG_TRACE("WGS_STATE_ESTABLISHED"); 3097 3098 wg_swap_sessions(wgp); 3099 KASSERT(wgs == wgp->wgp_session_stable); 3100 wgs_prev = wgp->wgp_session_unstable; 3101 getnanotime(&wgp->wgp_last_handshake_time); 3102 wgp->wgp_handshake_start_time = 0; 3103 wgp->wgp_last_sent_mac1_valid = false; 3104 wgp->wgp_last_sent_cookie_valid = false; 3105 3106 /* If we had a data packet queued up, send it. */ 3107 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3108 kpreempt_disable(); 3109 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3110 M_SETCTX(m, wgp); 3111 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3112 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3113 if_name(&wg->wg_if)); 3114 m_freem(m); 3115 } 3116 kpreempt_enable(); 3117 } 3118 3119 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3120 /* Wait for wg_get_stable_session to drain. */ 3121 pserialize_perform(wgp->wgp_psz); 3122 3123 /* Transition ESTABLISHED->DESTROYING. */ 3124 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 3125 3126 /* We can't destroy the old session immediately */ 3127 wg_schedule_session_dtor_timer(wgp); 3128 } else { 3129 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3130 "state=%d", wgs_prev->wgs_state); 3131 wg_clear_states(wgs_prev); 3132 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3133 } 3134 } 3135 3136 static void 3137 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3138 { 3139 3140 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3141 3142 KASSERT(mutex_owned(wgp->wgp_lock)); 3143 3144 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3145 pserialize_perform(wgp->wgp_psz); 3146 mutex_exit(wgp->wgp_lock); 3147 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3148 wg_psref_class); 3149 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3150 wg_psref_class); 3151 mutex_enter(wgp->wgp_lock); 3152 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3153 } 3154 } 3155 3156 static void 3157 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3158 { 3159 struct wg_session *wgs; 3160 3161 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3162 3163 KASSERT(mutex_owned(wgp->wgp_lock)); 3164 3165 wgs = wgp->wgp_session_stable; 3166 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3167 return; 3168 3169 wg_send_keepalive_msg(wgp, wgs); 3170 } 3171 3172 static void 3173 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3174 { 3175 struct wg_session *wgs; 3176 3177 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3178 3179 KASSERT(mutex_owned(wgp->wgp_lock)); 3180 3181 wgs = wgp->wgp_session_unstable; 3182 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 3183 wg_put_session_index(wg, wgs); 3184 } 3185 } 3186 3187 static void 3188 wg_peer_work(struct work *wk, void *cookie) 3189 { 3190 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3191 struct wg_softc *wg = wgp->wgp_sc; 3192 unsigned int tasks; 3193 3194 mutex_enter(wgp->wgp_intr_lock); 3195 while ((tasks = wgp->wgp_tasks) != 0) { 3196 wgp->wgp_tasks = 0; 3197 mutex_exit(wgp->wgp_intr_lock); 3198 3199 mutex_enter(wgp->wgp_lock); 3200 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3201 wg_task_send_init_message(wg, wgp); 3202 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3203 wg_task_retry_handshake(wg, wgp); 3204 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3205 wg_task_establish_session(wg, wgp); 3206 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3207 wg_task_endpoint_changed(wg, wgp); 3208 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3209 wg_task_send_keepalive_message(wg, wgp); 3210 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3211 wg_task_destroy_prev_session(wg, wgp); 3212 mutex_exit(wgp->wgp_lock); 3213 3214 mutex_enter(wgp->wgp_intr_lock); 3215 } 3216 mutex_exit(wgp->wgp_intr_lock); 3217 } 3218 3219 static void 3220 wg_job(struct threadpool_job *job) 3221 { 3222 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3223 int bound, upcalls; 3224 3225 mutex_enter(wg->wg_intr_lock); 3226 while ((upcalls = wg->wg_upcalls) != 0) { 3227 wg->wg_upcalls = 0; 3228 mutex_exit(wg->wg_intr_lock); 3229 bound = curlwp_bind(); 3230 if (ISSET(upcalls, WG_UPCALL_INET)) 3231 wg_receive_packets(wg, AF_INET); 3232 if (ISSET(upcalls, WG_UPCALL_INET6)) 3233 wg_receive_packets(wg, AF_INET6); 3234 curlwp_bindx(bound); 3235 mutex_enter(wg->wg_intr_lock); 3236 } 3237 threadpool_job_done(job); 3238 mutex_exit(wg->wg_intr_lock); 3239 } 3240 3241 static int 3242 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3243 { 3244 int error; 3245 uint16_t old_port = wg->wg_listen_port; 3246 3247 if (port != 0 && old_port == port) 3248 return 0; 3249 3250 struct sockaddr_in _sin, *sin = &_sin; 3251 sin->sin_len = sizeof(*sin); 3252 sin->sin_family = AF_INET; 3253 sin->sin_addr.s_addr = INADDR_ANY; 3254 sin->sin_port = htons(port); 3255 3256 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3257 if (error != 0) 3258 return error; 3259 3260 #ifdef INET6 3261 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3262 sin6->sin6_len = sizeof(*sin6); 3263 sin6->sin6_family = AF_INET6; 3264 sin6->sin6_addr = in6addr_any; 3265 sin6->sin6_port = htons(port); 3266 3267 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3268 if (error != 0) 3269 return error; 3270 #endif 3271 3272 wg->wg_listen_port = port; 3273 3274 return 0; 3275 } 3276 3277 static void 3278 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3279 { 3280 struct wg_softc *wg = cookie; 3281 int reason; 3282 3283 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3284 WG_UPCALL_INET : 3285 WG_UPCALL_INET6; 3286 3287 mutex_enter(wg->wg_intr_lock); 3288 wg->wg_upcalls |= reason; 3289 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3290 mutex_exit(wg->wg_intr_lock); 3291 } 3292 3293 static int 3294 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3295 struct sockaddr *src, void *arg) 3296 { 3297 struct wg_softc *wg = arg; 3298 struct wg_msg wgm; 3299 struct mbuf *m = *mp; 3300 3301 WG_TRACE("enter"); 3302 3303 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3304 KASSERT(offset <= m_length(m)); 3305 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3306 /* drop on the floor */ 3307 m_freem(m); 3308 return -1; 3309 } 3310 3311 /* 3312 * Copy the message header (32-bit message type) out -- we'll 3313 * worry about contiguity and alignment later. 3314 */ 3315 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3316 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3317 3318 /* 3319 * Handle DATA packets promptly as they arrive. Other packets 3320 * may require expensive public-key crypto and are not as 3321 * sensitive to latency, so defer them to the worker thread. 3322 */ 3323 switch (le32toh(wgm.wgm_type)) { 3324 case WG_MSG_TYPE_DATA: 3325 /* handle immediately */ 3326 m_adj(m, offset); 3327 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3328 m = m_pullup(m, sizeof(struct wg_msg_data)); 3329 if (m == NULL) 3330 return -1; 3331 } 3332 wg_handle_msg_data(wg, m, src); 3333 *mp = NULL; 3334 return 1; 3335 case WG_MSG_TYPE_INIT: 3336 case WG_MSG_TYPE_RESP: 3337 case WG_MSG_TYPE_COOKIE: 3338 /* pass through to so_receive in wg_receive_packets */ 3339 return 0; 3340 default: 3341 /* drop on the floor */ 3342 m_freem(m); 3343 return -1; 3344 } 3345 } 3346 3347 static int 3348 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3349 { 3350 int error; 3351 struct socket *so; 3352 3353 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3354 if (error != 0) 3355 return error; 3356 3357 solock(so); 3358 so->so_upcallarg = wg; 3359 so->so_upcall = wg_so_upcall; 3360 so->so_rcv.sb_flags |= SB_UPCALL; 3361 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3362 sounlock(so); 3363 3364 *sop = so; 3365 3366 return 0; 3367 } 3368 3369 static bool 3370 wg_session_hit_limits(struct wg_session *wgs) 3371 { 3372 3373 /* 3374 * [W] 6.2: Transport Message Limits 3375 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3376 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3377 * comes first, WireGuard will refuse to send any more transport data 3378 * messages using the current secure session, ..." 3379 */ 3380 KASSERT(wgs->wgs_time_established != 0); 3381 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3382 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3383 return true; 3384 } else if (wg_session_get_send_counter(wgs) > 3385 wg_reject_after_messages) { 3386 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3387 return true; 3388 } 3389 3390 return false; 3391 } 3392 3393 static void 3394 wgintr(void *cookie) 3395 { 3396 struct wg_peer *wgp; 3397 struct wg_session *wgs; 3398 struct mbuf *m; 3399 struct psref psref; 3400 3401 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3402 wgp = M_GETCTX(m, struct wg_peer *); 3403 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3404 WG_TRACE("no stable session"); 3405 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3406 goto next0; 3407 } 3408 if (__predict_false(wg_session_hit_limits(wgs))) { 3409 WG_TRACE("stable session hit limits"); 3410 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3411 goto next1; 3412 } 3413 wg_send_data_msg(wgp, wgs, m); 3414 m = NULL; /* consumed */ 3415 next1: wg_put_session(wgs, &psref); 3416 next0: m_freem(m); 3417 /* XXX Yield to avoid userland starvation? */ 3418 } 3419 } 3420 3421 static void 3422 wg_rekey_timer(void *arg) 3423 { 3424 struct wg_peer *wgp = arg; 3425 3426 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3427 } 3428 3429 static void 3430 wg_purge_pending_packets(struct wg_peer *wgp) 3431 { 3432 struct mbuf *m; 3433 3434 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 3435 m_freem(m); 3436 pktq_barrier(wg_pktq); 3437 } 3438 3439 static void 3440 wg_handshake_timeout_timer(void *arg) 3441 { 3442 struct wg_peer *wgp = arg; 3443 3444 WG_TRACE("enter"); 3445 3446 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3447 } 3448 3449 static struct wg_peer * 3450 wg_alloc_peer(struct wg_softc *wg) 3451 { 3452 struct wg_peer *wgp; 3453 3454 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3455 3456 wgp->wgp_sc = wg; 3457 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3458 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3459 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3460 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3461 wg_handshake_timeout_timer, wgp); 3462 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3463 callout_setfunc(&wgp->wgp_session_dtor_timer, 3464 wg_session_dtor_timer, wgp); 3465 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3466 wgp->wgp_endpoint_changing = false; 3467 wgp->wgp_endpoint_available = false; 3468 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3469 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3470 wgp->wgp_psz = pserialize_create(); 3471 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3472 3473 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3474 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3475 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3476 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3477 3478 struct wg_session *wgs; 3479 wgp->wgp_session_stable = 3480 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3481 wgp->wgp_session_unstable = 3482 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3483 wgs = wgp->wgp_session_stable; 3484 wgs->wgs_peer = wgp; 3485 wgs->wgs_state = WGS_STATE_UNKNOWN; 3486 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3487 #ifndef __HAVE_ATOMIC64_LOADSTORE 3488 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3489 #endif 3490 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3491 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3492 3493 wgs = wgp->wgp_session_unstable; 3494 wgs->wgs_peer = wgp; 3495 wgs->wgs_state = WGS_STATE_UNKNOWN; 3496 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3497 #ifndef __HAVE_ATOMIC64_LOADSTORE 3498 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3499 #endif 3500 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3501 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3502 3503 return wgp; 3504 } 3505 3506 static void 3507 wg_destroy_peer(struct wg_peer *wgp) 3508 { 3509 struct wg_session *wgs; 3510 struct wg_softc *wg = wgp->wgp_sc; 3511 3512 /* Prevent new packets from this peer on any source address. */ 3513 rw_enter(wg->wg_rwlock, RW_WRITER); 3514 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3515 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3516 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3517 struct radix_node *rn; 3518 3519 KASSERT(rnh != NULL); 3520 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3521 &wga->wga_sa_mask, rnh); 3522 if (rn == NULL) { 3523 char addrstr[128]; 3524 sockaddr_format(&wga->wga_sa_addr, addrstr, 3525 sizeof(addrstr)); 3526 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3527 if_name(&wg->wg_if), addrstr); 3528 } 3529 } 3530 rw_exit(wg->wg_rwlock); 3531 3532 /* Purge pending packets. */ 3533 wg_purge_pending_packets(wgp); 3534 3535 /* Halt all packet processing and timeouts. */ 3536 callout_halt(&wgp->wgp_rekey_timer, NULL); 3537 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3538 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3539 3540 /* Wait for any queued work to complete. */ 3541 workqueue_wait(wg_wq, &wgp->wgp_work); 3542 3543 wgs = wgp->wgp_session_unstable; 3544 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3545 mutex_enter(wgp->wgp_lock); 3546 wg_destroy_session(wg, wgs); 3547 mutex_exit(wgp->wgp_lock); 3548 } 3549 mutex_destroy(&wgs->wgs_recvwin->lock); 3550 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3551 #ifndef __HAVE_ATOMIC64_LOADSTORE 3552 mutex_destroy(&wgs->wgs_send_counter_lock); 3553 #endif 3554 kmem_free(wgs, sizeof(*wgs)); 3555 3556 wgs = wgp->wgp_session_stable; 3557 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3558 mutex_enter(wgp->wgp_lock); 3559 wg_destroy_session(wg, wgs); 3560 mutex_exit(wgp->wgp_lock); 3561 } 3562 mutex_destroy(&wgs->wgs_recvwin->lock); 3563 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3564 #ifndef __HAVE_ATOMIC64_LOADSTORE 3565 mutex_destroy(&wgs->wgs_send_counter_lock); 3566 #endif 3567 kmem_free(wgs, sizeof(*wgs)); 3568 3569 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3570 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3571 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3572 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3573 3574 pserialize_destroy(wgp->wgp_psz); 3575 mutex_obj_free(wgp->wgp_intr_lock); 3576 mutex_obj_free(wgp->wgp_lock); 3577 3578 kmem_free(wgp, sizeof(*wgp)); 3579 } 3580 3581 static void 3582 wg_destroy_all_peers(struct wg_softc *wg) 3583 { 3584 struct wg_peer *wgp, *wgp0 __diagused; 3585 void *garbage_byname, *garbage_bypubkey; 3586 3587 restart: 3588 garbage_byname = garbage_bypubkey = NULL; 3589 mutex_enter(wg->wg_lock); 3590 WG_PEER_WRITER_FOREACH(wgp, wg) { 3591 if (wgp->wgp_name[0]) { 3592 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3593 strlen(wgp->wgp_name)); 3594 KASSERT(wgp0 == wgp); 3595 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3596 } 3597 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3598 sizeof(wgp->wgp_pubkey)); 3599 KASSERT(wgp0 == wgp); 3600 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3601 WG_PEER_WRITER_REMOVE(wgp); 3602 wg->wg_npeers--; 3603 mutex_enter(wgp->wgp_lock); 3604 pserialize_perform(wgp->wgp_psz); 3605 mutex_exit(wgp->wgp_lock); 3606 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3607 break; 3608 } 3609 mutex_exit(wg->wg_lock); 3610 3611 if (wgp == NULL) 3612 return; 3613 3614 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3615 3616 wg_destroy_peer(wgp); 3617 thmap_gc(wg->wg_peers_byname, garbage_byname); 3618 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3619 3620 goto restart; 3621 } 3622 3623 static int 3624 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3625 { 3626 struct wg_peer *wgp, *wgp0 __diagused; 3627 void *garbage_byname, *garbage_bypubkey; 3628 3629 mutex_enter(wg->wg_lock); 3630 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3631 if (wgp != NULL) { 3632 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3633 sizeof(wgp->wgp_pubkey)); 3634 KASSERT(wgp0 == wgp); 3635 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3636 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3637 WG_PEER_WRITER_REMOVE(wgp); 3638 wg->wg_npeers--; 3639 if (wg->wg_npeers == 0) 3640 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3641 mutex_enter(wgp->wgp_lock); 3642 pserialize_perform(wgp->wgp_psz); 3643 mutex_exit(wgp->wgp_lock); 3644 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3645 } 3646 mutex_exit(wg->wg_lock); 3647 3648 if (wgp == NULL) 3649 return ENOENT; 3650 3651 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3652 3653 wg_destroy_peer(wgp); 3654 thmap_gc(wg->wg_peers_byname, garbage_byname); 3655 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3656 3657 return 0; 3658 } 3659 3660 static int 3661 wg_if_attach(struct wg_softc *wg) 3662 { 3663 3664 wg->wg_if.if_addrlen = 0; 3665 wg->wg_if.if_mtu = WG_MTU; 3666 wg->wg_if.if_flags = IFF_MULTICAST; 3667 wg->wg_if.if_extflags = IFEF_MPSAFE; 3668 wg->wg_if.if_ioctl = wg_ioctl; 3669 wg->wg_if.if_output = wg_output; 3670 wg->wg_if.if_init = wg_init; 3671 #ifdef ALTQ 3672 wg->wg_if.if_start = wg_start; 3673 #endif 3674 wg->wg_if.if_stop = wg_stop; 3675 wg->wg_if.if_type = IFT_OTHER; 3676 wg->wg_if.if_dlt = DLT_NULL; 3677 wg->wg_if.if_softc = wg; 3678 #ifdef ALTQ 3679 IFQ_SET_READY(&wg->wg_if.if_snd); 3680 #endif 3681 if_initialize(&wg->wg_if); 3682 3683 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3684 if_alloc_sadl(&wg->wg_if); 3685 if_register(&wg->wg_if); 3686 3687 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3688 3689 return 0; 3690 } 3691 3692 static void 3693 wg_if_detach(struct wg_softc *wg) 3694 { 3695 struct ifnet *ifp = &wg->wg_if; 3696 3697 bpf_detach(ifp); 3698 if_detach(ifp); 3699 } 3700 3701 static int 3702 wg_clone_create(struct if_clone *ifc, int unit) 3703 { 3704 struct wg_softc *wg; 3705 int error; 3706 3707 wg_guarantee_initialized(); 3708 3709 error = wg_count_inc(); 3710 if (error) 3711 return error; 3712 3713 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3714 3715 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3716 3717 PSLIST_INIT(&wg->wg_peers); 3718 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3719 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3720 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3721 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3722 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3723 wg->wg_rwlock = rw_obj_alloc(); 3724 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3725 "%s", if_name(&wg->wg_if)); 3726 wg->wg_ops = &wg_ops_rumpkernel; 3727 3728 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3729 if (error) 3730 goto fail0; 3731 3732 #ifdef INET 3733 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3734 if (error) 3735 goto fail1; 3736 rn_inithead((void **)&wg->wg_rtable_ipv4, 3737 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3738 #endif 3739 #ifdef INET6 3740 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3741 if (error) 3742 goto fail2; 3743 rn_inithead((void **)&wg->wg_rtable_ipv6, 3744 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3745 #endif 3746 3747 error = wg_if_attach(wg); 3748 if (error) 3749 goto fail3; 3750 3751 return 0; 3752 3753 fail4: __unused 3754 wg_if_detach(wg); 3755 fail3: wg_destroy_all_peers(wg); 3756 #ifdef INET6 3757 solock(wg->wg_so6); 3758 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3759 sounlock(wg->wg_so6); 3760 #endif 3761 #ifdef INET 3762 solock(wg->wg_so4); 3763 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3764 sounlock(wg->wg_so4); 3765 #endif 3766 mutex_enter(wg->wg_intr_lock); 3767 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3768 mutex_exit(wg->wg_intr_lock); 3769 #ifdef INET6 3770 if (wg->wg_rtable_ipv6 != NULL) 3771 free(wg->wg_rtable_ipv6, M_RTABLE); 3772 soclose(wg->wg_so6); 3773 fail2: 3774 #endif 3775 #ifdef INET 3776 if (wg->wg_rtable_ipv4 != NULL) 3777 free(wg->wg_rtable_ipv4, M_RTABLE); 3778 soclose(wg->wg_so4); 3779 fail1: 3780 #endif 3781 threadpool_put(wg->wg_threadpool, PRI_NONE); 3782 fail0: threadpool_job_destroy(&wg->wg_job); 3783 rw_obj_free(wg->wg_rwlock); 3784 mutex_obj_free(wg->wg_intr_lock); 3785 mutex_obj_free(wg->wg_lock); 3786 thmap_destroy(wg->wg_sessions_byindex); 3787 thmap_destroy(wg->wg_peers_byname); 3788 thmap_destroy(wg->wg_peers_bypubkey); 3789 PSLIST_DESTROY(&wg->wg_peers); 3790 kmem_free(wg, sizeof(*wg)); 3791 wg_count_dec(); 3792 return error; 3793 } 3794 3795 static int 3796 wg_clone_destroy(struct ifnet *ifp) 3797 { 3798 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3799 3800 #ifdef WG_RUMPKERNEL 3801 if (wg_user_mode(wg)) { 3802 rumpuser_wg_destroy(wg->wg_user); 3803 wg->wg_user = NULL; 3804 } 3805 #endif 3806 3807 wg_if_detach(wg); 3808 wg_destroy_all_peers(wg); 3809 #ifdef INET6 3810 solock(wg->wg_so6); 3811 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3812 sounlock(wg->wg_so6); 3813 #endif 3814 #ifdef INET 3815 solock(wg->wg_so4); 3816 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3817 sounlock(wg->wg_so4); 3818 #endif 3819 mutex_enter(wg->wg_intr_lock); 3820 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3821 mutex_exit(wg->wg_intr_lock); 3822 #ifdef INET6 3823 if (wg->wg_rtable_ipv6 != NULL) 3824 free(wg->wg_rtable_ipv6, M_RTABLE); 3825 soclose(wg->wg_so6); 3826 #endif 3827 #ifdef INET 3828 if (wg->wg_rtable_ipv4 != NULL) 3829 free(wg->wg_rtable_ipv4, M_RTABLE); 3830 soclose(wg->wg_so4); 3831 #endif 3832 threadpool_put(wg->wg_threadpool, PRI_NONE); 3833 threadpool_job_destroy(&wg->wg_job); 3834 rw_obj_free(wg->wg_rwlock); 3835 mutex_obj_free(wg->wg_intr_lock); 3836 mutex_obj_free(wg->wg_lock); 3837 thmap_destroy(wg->wg_sessions_byindex); 3838 thmap_destroy(wg->wg_peers_byname); 3839 thmap_destroy(wg->wg_peers_bypubkey); 3840 PSLIST_DESTROY(&wg->wg_peers); 3841 kmem_free(wg, sizeof(*wg)); 3842 wg_count_dec(); 3843 3844 return 0; 3845 } 3846 3847 static struct wg_peer * 3848 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3849 struct psref *psref) 3850 { 3851 struct radix_node_head *rnh; 3852 struct radix_node *rn; 3853 struct wg_peer *wgp = NULL; 3854 struct wg_allowedip *wga; 3855 3856 #ifdef WG_DEBUG_LOG 3857 char addrstr[128]; 3858 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3859 WG_DLOG("sa=%s\n", addrstr); 3860 #endif 3861 3862 rw_enter(wg->wg_rwlock, RW_READER); 3863 3864 rnh = wg_rnh(wg, sa->sa_family); 3865 if (rnh == NULL) 3866 goto out; 3867 3868 rn = rnh->rnh_matchaddr(sa, rnh); 3869 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3870 goto out; 3871 3872 WG_TRACE("success"); 3873 3874 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3875 wgp = wga->wga_peer; 3876 wg_get_peer(wgp, psref); 3877 3878 out: 3879 rw_exit(wg->wg_rwlock); 3880 return wgp; 3881 } 3882 3883 static void 3884 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3885 struct wg_session *wgs, struct wg_msg_data *wgmd) 3886 { 3887 3888 memset(wgmd, 0, sizeof(*wgmd)); 3889 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3890 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3891 /* [W] 5.4.6: msg.counter := Nm^send */ 3892 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3893 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3894 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3895 } 3896 3897 static int 3898 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3899 const struct rtentry *rt) 3900 { 3901 struct wg_softc *wg = ifp->if_softc; 3902 struct wg_peer *wgp = NULL; 3903 struct wg_session *wgs = NULL; 3904 struct psref wgp_psref, wgs_psref; 3905 int bound; 3906 int error; 3907 3908 bound = curlwp_bind(); 3909 3910 /* TODO make the nest limit configurable via sysctl */ 3911 error = if_tunnel_check_nesting(ifp, m, 1); 3912 if (error) { 3913 WGLOG(LOG_ERR, 3914 "%s: tunneling loop detected and packet dropped\n", 3915 if_name(&wg->wg_if)); 3916 goto out0; 3917 } 3918 3919 #ifdef ALTQ 3920 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 3921 & ALTQF_ENABLED; 3922 if (altq) 3923 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3924 #endif 3925 3926 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3927 3928 m->m_flags &= ~(M_BCAST|M_MCAST); 3929 3930 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3931 if (wgp == NULL) { 3932 WG_TRACE("peer not found"); 3933 error = EHOSTUNREACH; 3934 goto out0; 3935 } 3936 3937 /* Clear checksum-offload flags. */ 3938 m->m_pkthdr.csum_flags = 0; 3939 m->m_pkthdr.csum_data = 0; 3940 3941 /* Check whether there's an established session. */ 3942 wgs = wg_get_stable_session(wgp, &wgs_psref); 3943 if (wgs == NULL) { 3944 /* 3945 * No established session. If we're the first to try 3946 * sending data, schedule a handshake and queue the 3947 * packet for when the handshake is done; otherwise 3948 * just drop the packet and let the ongoing handshake 3949 * attempt continue. We could queue more data packets 3950 * but it's not clear that's worthwhile. 3951 */ 3952 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 3953 m = NULL; /* consume */ 3954 WG_TRACE("queued first packet; init handshake"); 3955 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3956 } else { 3957 WG_TRACE("first packet already queued, dropping"); 3958 } 3959 goto out1; 3960 } 3961 3962 /* There's an established session. Toss it in the queue. */ 3963 #ifdef ALTQ 3964 if (altq) { 3965 mutex_enter(ifp->if_snd.ifq_lock); 3966 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 3967 M_SETCTX(m, wgp); 3968 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 3969 m = NULL; /* consume */ 3970 } 3971 mutex_exit(ifp->if_snd.ifq_lock); 3972 if (m == NULL) { 3973 wg_start(ifp); 3974 goto out2; 3975 } 3976 } 3977 #endif 3978 kpreempt_disable(); 3979 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3980 M_SETCTX(m, wgp); 3981 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3982 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3983 if_name(&wg->wg_if)); 3984 error = ENOBUFS; 3985 goto out3; 3986 } 3987 m = NULL; /* consumed */ 3988 error = 0; 3989 out3: kpreempt_enable(); 3990 3991 #ifdef ALTQ 3992 out2: 3993 #endif 3994 wg_put_session(wgs, &wgs_psref); 3995 out1: wg_put_peer(wgp, &wgp_psref); 3996 out0: m_freem(m); 3997 curlwp_bindx(bound); 3998 return error; 3999 } 4000 4001 static int 4002 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 4003 { 4004 struct psref psref; 4005 struct wg_sockaddr *wgsa; 4006 int error; 4007 struct socket *so; 4008 4009 wgsa = wg_get_endpoint_sa(wgp, &psref); 4010 so = wg_get_so_by_peer(wgp, wgsa); 4011 solock(so); 4012 if (wgsatosa(wgsa)->sa_family == AF_INET) { 4013 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 4014 } else { 4015 #ifdef INET6 4016 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 4017 NULL, curlwp); 4018 #else 4019 m_freem(m); 4020 error = EPFNOSUPPORT; 4021 #endif 4022 } 4023 sounlock(so); 4024 wg_put_sa(wgp, wgsa, &psref); 4025 4026 return error; 4027 } 4028 4029 /* Inspired by pppoe_get_mbuf */ 4030 static struct mbuf * 4031 wg_get_mbuf(size_t leading_len, size_t len) 4032 { 4033 struct mbuf *m; 4034 4035 KASSERT(leading_len <= MCLBYTES); 4036 KASSERT(len <= MCLBYTES - leading_len); 4037 4038 m = m_gethdr(M_DONTWAIT, MT_DATA); 4039 if (m == NULL) 4040 return NULL; 4041 if (len + leading_len > MHLEN) { 4042 m_clget(m, M_DONTWAIT); 4043 if ((m->m_flags & M_EXT) == 0) { 4044 m_free(m); 4045 return NULL; 4046 } 4047 } 4048 m->m_data += leading_len; 4049 m->m_pkthdr.len = m->m_len = len; 4050 4051 return m; 4052 } 4053 4054 static int 4055 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 4056 struct mbuf *m) 4057 { 4058 struct wg_softc *wg = wgp->wgp_sc; 4059 int error; 4060 size_t inner_len, padded_len, encrypted_len; 4061 char *padded_buf = NULL; 4062 size_t mlen; 4063 struct wg_msg_data *wgmd; 4064 bool free_padded_buf = false; 4065 struct mbuf *n; 4066 size_t leading_len = max_hdr + sizeof(struct udphdr); 4067 4068 mlen = m_length(m); 4069 inner_len = mlen; 4070 padded_len = roundup(mlen, 16); 4071 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4072 WG_DLOG("inner=%zu, padded=%zu, encrypted_len=%zu\n", 4073 inner_len, padded_len, encrypted_len); 4074 if (mlen != 0) { 4075 bool success; 4076 success = m_ensure_contig(&m, padded_len); 4077 if (success) { 4078 padded_buf = mtod(m, char *); 4079 } else { 4080 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4081 if (padded_buf == NULL) { 4082 error = ENOBUFS; 4083 goto end; 4084 } 4085 free_padded_buf = true; 4086 m_copydata(m, 0, mlen, padded_buf); 4087 } 4088 memset(padded_buf + mlen, 0, padded_len - inner_len); 4089 } 4090 4091 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4092 if (n == NULL) { 4093 error = ENOBUFS; 4094 goto end; 4095 } 4096 KASSERT(n->m_len >= sizeof(*wgmd)); 4097 wgmd = mtod(n, struct wg_msg_data *); 4098 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4099 #ifdef WG_DEBUG_PACKET 4100 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4101 hexdump(printf, "padded_buf", padded_buf, 4102 padded_len); 4103 } 4104 #endif 4105 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4106 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4107 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4108 padded_buf, padded_len, 4109 NULL, 0); 4110 #ifdef WG_DEBUG_PACKET 4111 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4112 hexdump(printf, "tkey_send", wgs->wgs_tkey_send, 4113 sizeof(wgs->wgs_tkey_send)); 4114 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 4115 hexdump(printf, "outgoing packet", 4116 (char *)wgmd + sizeof(*wgmd), encrypted_len); 4117 size_t decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 4118 char *decrypted_buf = kmem_intr_alloc((decrypted_len + 4119 WG_AUTHTAG_LEN/*XXX*/), KM_NOSLEEP); 4120 if (decrypted_buf != NULL) { 4121 error = wg_algo_aead_dec( 4122 1 + decrypted_buf /* force misalignment */, 4123 encrypted_len - WG_AUTHTAG_LEN /* XXX */, 4124 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4125 (char *)wgmd + sizeof(*wgmd), encrypted_len, 4126 NULL, 0); 4127 if (error) { 4128 WG_DLOG("wg_algo_aead_dec failed: %d\n", 4129 error); 4130 } 4131 if (!consttime_memequal(1 + decrypted_buf, 4132 (char *)wgmd + sizeof(*wgmd), 4133 decrypted_len)) { 4134 WG_DLOG("wg_algo_aead_dec returned garbage\n"); 4135 } 4136 kmem_intr_free(decrypted_buf, (decrypted_len + 4137 WG_AUTHTAG_LEN/*XXX*/)); 4138 } 4139 } 4140 #endif 4141 4142 error = wg->wg_ops->send_data_msg(wgp, n); 4143 if (error == 0) { 4144 struct ifnet *ifp = &wg->wg_if; 4145 if_statadd(ifp, if_obytes, mlen); 4146 if_statinc(ifp, if_opackets); 4147 if (wgs->wgs_is_initiator && 4148 wgs->wgs_time_last_data_sent == 0) { 4149 /* 4150 * [W] 6.2 Transport Message Limits 4151 * "if a peer is the initiator of a current secure 4152 * session, WireGuard will send a handshake initiation 4153 * message to begin a new secure session if, after 4154 * transmitting a transport data message, the current 4155 * secure session is REKEY-AFTER-TIME seconds old," 4156 */ 4157 wg_schedule_rekey_timer(wgp); 4158 } 4159 wgs->wgs_time_last_data_sent = time_uptime; 4160 if (wg_session_get_send_counter(wgs) >= 4161 wg_rekey_after_messages) { 4162 /* 4163 * [W] 6.2 Transport Message Limits 4164 * "WireGuard will try to create a new session, by 4165 * sending a handshake initiation message (section 4166 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4167 * transport data messages..." 4168 */ 4169 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4170 } 4171 } 4172 end: 4173 m_freem(m); 4174 if (free_padded_buf) 4175 kmem_intr_free(padded_buf, padded_len); 4176 return error; 4177 } 4178 4179 static void 4180 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4181 { 4182 pktqueue_t *pktq; 4183 size_t pktlen; 4184 4185 KASSERT(af == AF_INET || af == AF_INET6); 4186 4187 WG_TRACE(""); 4188 4189 m_set_rcvif(m, ifp); 4190 pktlen = m->m_pkthdr.len; 4191 4192 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4193 4194 switch (af) { 4195 case AF_INET: 4196 pktq = ip_pktq; 4197 break; 4198 #ifdef INET6 4199 case AF_INET6: 4200 pktq = ip6_pktq; 4201 break; 4202 #endif 4203 default: 4204 panic("invalid af=%d", af); 4205 } 4206 4207 kpreempt_disable(); 4208 const u_int h = curcpu()->ci_index; 4209 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4210 if_statadd(ifp, if_ibytes, pktlen); 4211 if_statinc(ifp, if_ipackets); 4212 } else { 4213 m_freem(m); 4214 } 4215 kpreempt_enable(); 4216 } 4217 4218 static void 4219 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4220 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4221 { 4222 4223 crypto_scalarmult_base(pubkey, privkey); 4224 } 4225 4226 static int 4227 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4228 { 4229 struct radix_node_head *rnh; 4230 struct radix_node *rn; 4231 int error = 0; 4232 4233 rw_enter(wg->wg_rwlock, RW_WRITER); 4234 rnh = wg_rnh(wg, wga->wga_family); 4235 KASSERT(rnh != NULL); 4236 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4237 wga->wga_nodes); 4238 rw_exit(wg->wg_rwlock); 4239 4240 if (rn == NULL) 4241 error = EEXIST; 4242 4243 return error; 4244 } 4245 4246 static int 4247 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4248 struct wg_peer **wgpp) 4249 { 4250 int error = 0; 4251 const void *pubkey; 4252 size_t pubkey_len; 4253 const void *psk; 4254 size_t psk_len; 4255 const char *name = NULL; 4256 4257 if (prop_dictionary_get_string(peer, "name", &name)) { 4258 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4259 error = EINVAL; 4260 goto out; 4261 } 4262 } 4263 4264 if (!prop_dictionary_get_data(peer, "public_key", 4265 &pubkey, &pubkey_len)) { 4266 error = EINVAL; 4267 goto out; 4268 } 4269 #ifdef WG_DEBUG_DUMP 4270 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4271 char *hex = gethexdump(pubkey, pubkey_len); 4272 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%zu\n%s\n", 4273 pubkey, pubkey_len, hex); 4274 puthexdump(hex, pubkey, pubkey_len); 4275 } 4276 #endif 4277 4278 struct wg_peer *wgp = wg_alloc_peer(wg); 4279 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4280 if (name != NULL) 4281 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4282 4283 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4284 if (psk_len != sizeof(wgp->wgp_psk)) { 4285 error = EINVAL; 4286 goto out; 4287 } 4288 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4289 } 4290 4291 const void *addr; 4292 size_t addr_len; 4293 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4294 4295 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4296 goto skip_endpoint; 4297 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4298 addr_len > sizeof(*wgsatoss(wgsa))) { 4299 error = EINVAL; 4300 goto out; 4301 } 4302 memcpy(wgsatoss(wgsa), addr, addr_len); 4303 switch (wgsa_family(wgsa)) { 4304 case AF_INET: 4305 #ifdef INET6 4306 case AF_INET6: 4307 #endif 4308 break; 4309 default: 4310 error = EPFNOSUPPORT; 4311 goto out; 4312 } 4313 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4314 error = EINVAL; 4315 goto out; 4316 } 4317 { 4318 char addrstr[128]; 4319 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4320 WG_DLOG("addr=%s\n", addrstr); 4321 } 4322 wgp->wgp_endpoint_available = true; 4323 4324 prop_array_t allowedips; 4325 skip_endpoint: 4326 allowedips = prop_dictionary_get(peer, "allowedips"); 4327 if (allowedips == NULL) 4328 goto skip; 4329 4330 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4331 prop_dictionary_t prop_allowedip; 4332 int j = 0; 4333 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4334 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4335 4336 if (!prop_dictionary_get_int(prop_allowedip, "family", 4337 &wga->wga_family)) 4338 continue; 4339 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4340 &addr, &addr_len)) 4341 continue; 4342 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4343 &wga->wga_cidr)) 4344 continue; 4345 4346 switch (wga->wga_family) { 4347 case AF_INET: { 4348 struct sockaddr_in sin; 4349 char addrstr[128]; 4350 struct in_addr mask; 4351 struct sockaddr_in sin_mask; 4352 4353 if (addr_len != sizeof(struct in_addr)) 4354 return EINVAL; 4355 memcpy(&wga->wga_addr4, addr, addr_len); 4356 4357 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4358 0); 4359 sockaddr_copy(&wga->wga_sa_addr, 4360 sizeof(sin), sintosa(&sin)); 4361 4362 sockaddr_format(sintosa(&sin), 4363 addrstr, sizeof(addrstr)); 4364 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4365 4366 in_len2mask(&mask, wga->wga_cidr); 4367 sockaddr_in_init(&sin_mask, &mask, 0); 4368 sockaddr_copy(&wga->wga_sa_mask, 4369 sizeof(sin_mask), sintosa(&sin_mask)); 4370 4371 break; 4372 } 4373 #ifdef INET6 4374 case AF_INET6: { 4375 struct sockaddr_in6 sin6; 4376 char addrstr[128]; 4377 struct in6_addr mask; 4378 struct sockaddr_in6 sin6_mask; 4379 4380 if (addr_len != sizeof(struct in6_addr)) 4381 return EINVAL; 4382 memcpy(&wga->wga_addr6, addr, addr_len); 4383 4384 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4385 0, 0, 0); 4386 sockaddr_copy(&wga->wga_sa_addr, 4387 sizeof(sin6), sin6tosa(&sin6)); 4388 4389 sockaddr_format(sin6tosa(&sin6), 4390 addrstr, sizeof(addrstr)); 4391 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4392 4393 in6_prefixlen2mask(&mask, wga->wga_cidr); 4394 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4395 sockaddr_copy(&wga->wga_sa_mask, 4396 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4397 4398 break; 4399 } 4400 #endif 4401 default: 4402 error = EINVAL; 4403 goto out; 4404 } 4405 wga->wga_peer = wgp; 4406 4407 error = wg_rtable_add_route(wg, wga); 4408 if (error != 0) 4409 goto out; 4410 4411 j++; 4412 } 4413 wgp->wgp_n_allowedips = j; 4414 skip: 4415 *wgpp = wgp; 4416 out: 4417 return error; 4418 } 4419 4420 static int 4421 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4422 { 4423 int error; 4424 char *buf; 4425 4426 WG_DLOG("buf=%p, len=%zu\n", ifd->ifd_data, ifd->ifd_len); 4427 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4428 return E2BIG; 4429 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4430 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4431 if (error != 0) 4432 return error; 4433 buf[ifd->ifd_len] = '\0'; 4434 #ifdef WG_DEBUG_DUMP 4435 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4436 log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len), 4437 (const char *)buf); 4438 } 4439 #endif 4440 *_buf = buf; 4441 return 0; 4442 } 4443 4444 static int 4445 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4446 { 4447 int error; 4448 prop_dictionary_t prop_dict; 4449 char *buf = NULL; 4450 const void *privkey; 4451 size_t privkey_len; 4452 4453 error = wg_alloc_prop_buf(&buf, ifd); 4454 if (error != 0) 4455 return error; 4456 error = EINVAL; 4457 prop_dict = prop_dictionary_internalize(buf); 4458 if (prop_dict == NULL) 4459 goto out; 4460 if (!prop_dictionary_get_data(prop_dict, "private_key", 4461 &privkey, &privkey_len)) 4462 goto out; 4463 #ifdef WG_DEBUG_DUMP 4464 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4465 char *hex = gethexdump(privkey, privkey_len); 4466 log(LOG_DEBUG, "privkey=%p, privkey_len=%zu\n%s\n", 4467 privkey, privkey_len, hex); 4468 puthexdump(hex, privkey, privkey_len); 4469 } 4470 #endif 4471 if (privkey_len != WG_STATIC_KEY_LEN) 4472 goto out; 4473 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4474 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4475 error = 0; 4476 4477 out: 4478 kmem_free(buf, ifd->ifd_len + 1); 4479 return error; 4480 } 4481 4482 static int 4483 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4484 { 4485 int error; 4486 prop_dictionary_t prop_dict; 4487 char *buf = NULL; 4488 uint16_t port; 4489 4490 error = wg_alloc_prop_buf(&buf, ifd); 4491 if (error != 0) 4492 return error; 4493 error = EINVAL; 4494 prop_dict = prop_dictionary_internalize(buf); 4495 if (prop_dict == NULL) 4496 goto out; 4497 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4498 goto out; 4499 4500 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4501 4502 out: 4503 kmem_free(buf, ifd->ifd_len + 1); 4504 return error; 4505 } 4506 4507 static int 4508 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4509 { 4510 int error; 4511 prop_dictionary_t prop_dict; 4512 char *buf = NULL; 4513 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4514 4515 error = wg_alloc_prop_buf(&buf, ifd); 4516 if (error != 0) 4517 return error; 4518 error = EINVAL; 4519 prop_dict = prop_dictionary_internalize(buf); 4520 if (prop_dict == NULL) 4521 goto out; 4522 4523 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4524 if (error != 0) 4525 goto out; 4526 4527 mutex_enter(wg->wg_lock); 4528 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4529 sizeof(wgp->wgp_pubkey)) != NULL || 4530 (wgp->wgp_name[0] && 4531 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4532 strlen(wgp->wgp_name)) != NULL)) { 4533 mutex_exit(wg->wg_lock); 4534 wg_destroy_peer(wgp); 4535 error = EEXIST; 4536 goto out; 4537 } 4538 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4539 sizeof(wgp->wgp_pubkey), wgp); 4540 KASSERT(wgp0 == wgp); 4541 if (wgp->wgp_name[0]) { 4542 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4543 strlen(wgp->wgp_name), wgp); 4544 KASSERT(wgp0 == wgp); 4545 } 4546 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4547 wg->wg_npeers++; 4548 mutex_exit(wg->wg_lock); 4549 4550 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4551 4552 out: 4553 kmem_free(buf, ifd->ifd_len + 1); 4554 return error; 4555 } 4556 4557 static int 4558 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4559 { 4560 int error; 4561 prop_dictionary_t prop_dict; 4562 char *buf = NULL; 4563 const char *name; 4564 4565 error = wg_alloc_prop_buf(&buf, ifd); 4566 if (error != 0) 4567 return error; 4568 error = EINVAL; 4569 prop_dict = prop_dictionary_internalize(buf); 4570 if (prop_dict == NULL) 4571 goto out; 4572 4573 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4574 goto out; 4575 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4576 goto out; 4577 4578 error = wg_destroy_peer_name(wg, name); 4579 out: 4580 kmem_free(buf, ifd->ifd_len + 1); 4581 return error; 4582 } 4583 4584 static bool 4585 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4586 { 4587 int au = cmd == SIOCGDRVSPEC ? 4588 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4589 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4590 return kauth_authorize_network(kauth_cred_get(), 4591 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4592 (void *)cmd, NULL) == 0; 4593 } 4594 4595 static int 4596 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4597 { 4598 int error = ENOMEM; 4599 prop_dictionary_t prop_dict; 4600 prop_array_t peers = NULL; 4601 char *buf; 4602 struct wg_peer *wgp; 4603 int s, i; 4604 4605 prop_dict = prop_dictionary_create(); 4606 if (prop_dict == NULL) 4607 goto error; 4608 4609 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4610 if (!prop_dictionary_set_data(prop_dict, "private_key", 4611 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4612 goto error; 4613 } 4614 4615 if (wg->wg_listen_port != 0) { 4616 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4617 wg->wg_listen_port)) 4618 goto error; 4619 } 4620 4621 if (wg->wg_npeers == 0) 4622 goto skip_peers; 4623 4624 peers = prop_array_create(); 4625 if (peers == NULL) 4626 goto error; 4627 4628 s = pserialize_read_enter(); 4629 i = 0; 4630 WG_PEER_READER_FOREACH(wgp, wg) { 4631 struct wg_sockaddr *wgsa; 4632 struct psref wgp_psref, wgsa_psref; 4633 prop_dictionary_t prop_peer; 4634 4635 wg_get_peer(wgp, &wgp_psref); 4636 pserialize_read_exit(s); 4637 4638 prop_peer = prop_dictionary_create(); 4639 if (prop_peer == NULL) 4640 goto next; 4641 4642 if (strlen(wgp->wgp_name) > 0) { 4643 if (!prop_dictionary_set_string(prop_peer, "name", 4644 wgp->wgp_name)) 4645 goto next; 4646 } 4647 4648 if (!prop_dictionary_set_data(prop_peer, "public_key", 4649 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4650 goto next; 4651 4652 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4653 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4654 sizeof(wgp->wgp_psk))) { 4655 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4656 if (!prop_dictionary_set_data(prop_peer, 4657 "preshared_key", 4658 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4659 goto next; 4660 } 4661 } 4662 4663 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4664 CTASSERT(AF_UNSPEC == 0); 4665 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4666 !prop_dictionary_set_data(prop_peer, "endpoint", 4667 wgsatoss(wgsa), 4668 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4669 wg_put_sa(wgp, wgsa, &wgsa_psref); 4670 goto next; 4671 } 4672 wg_put_sa(wgp, wgsa, &wgsa_psref); 4673 4674 const struct timespec *t = &wgp->wgp_last_handshake_time; 4675 4676 if (!prop_dictionary_set_uint64(prop_peer, 4677 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4678 goto next; 4679 if (!prop_dictionary_set_uint32(prop_peer, 4680 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4681 goto next; 4682 4683 if (wgp->wgp_n_allowedips == 0) 4684 goto skip_allowedips; 4685 4686 prop_array_t allowedips = prop_array_create(); 4687 if (allowedips == NULL) 4688 goto next; 4689 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4690 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4691 prop_dictionary_t prop_allowedip; 4692 4693 prop_allowedip = prop_dictionary_create(); 4694 if (prop_allowedip == NULL) 4695 break; 4696 4697 if (!prop_dictionary_set_int(prop_allowedip, "family", 4698 wga->wga_family)) 4699 goto _next; 4700 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4701 wga->wga_cidr)) 4702 goto _next; 4703 4704 switch (wga->wga_family) { 4705 case AF_INET: 4706 if (!prop_dictionary_set_data(prop_allowedip, 4707 "ip", &wga->wga_addr4, 4708 sizeof(wga->wga_addr4))) 4709 goto _next; 4710 break; 4711 #ifdef INET6 4712 case AF_INET6: 4713 if (!prop_dictionary_set_data(prop_allowedip, 4714 "ip", &wga->wga_addr6, 4715 sizeof(wga->wga_addr6))) 4716 goto _next; 4717 break; 4718 #endif 4719 default: 4720 break; 4721 } 4722 prop_array_set(allowedips, j, prop_allowedip); 4723 _next: 4724 prop_object_release(prop_allowedip); 4725 } 4726 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4727 prop_object_release(allowedips); 4728 4729 skip_allowedips: 4730 4731 prop_array_set(peers, i, prop_peer); 4732 next: 4733 if (prop_peer) 4734 prop_object_release(prop_peer); 4735 i++; 4736 4737 s = pserialize_read_enter(); 4738 wg_put_peer(wgp, &wgp_psref); 4739 } 4740 pserialize_read_exit(s); 4741 4742 prop_dictionary_set(prop_dict, "peers", peers); 4743 prop_object_release(peers); 4744 peers = NULL; 4745 4746 skip_peers: 4747 buf = prop_dictionary_externalize(prop_dict); 4748 if (buf == NULL) 4749 goto error; 4750 if (ifd->ifd_len < (strlen(buf) + 1)) { 4751 error = EINVAL; 4752 goto error; 4753 } 4754 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4755 4756 free(buf, 0); 4757 error: 4758 if (peers != NULL) 4759 prop_object_release(peers); 4760 if (prop_dict != NULL) 4761 prop_object_release(prop_dict); 4762 4763 return error; 4764 } 4765 4766 static int 4767 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4768 { 4769 struct wg_softc *wg = ifp->if_softc; 4770 struct ifreq *ifr = data; 4771 struct ifaddr *ifa = data; 4772 struct ifdrv *ifd = data; 4773 int error = 0; 4774 4775 switch (cmd) { 4776 case SIOCINITIFADDR: 4777 if (ifa->ifa_addr->sa_family != AF_LINK && 4778 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4779 (IFF_UP | IFF_RUNNING)) { 4780 ifp->if_flags |= IFF_UP; 4781 error = if_init(ifp); 4782 } 4783 return error; 4784 case SIOCADDMULTI: 4785 case SIOCDELMULTI: 4786 switch (ifr->ifr_addr.sa_family) { 4787 case AF_INET: /* IP supports Multicast */ 4788 break; 4789 #ifdef INET6 4790 case AF_INET6: /* IP6 supports Multicast */ 4791 break; 4792 #endif 4793 default: /* Other protocols doesn't support Multicast */ 4794 error = EAFNOSUPPORT; 4795 break; 4796 } 4797 return error; 4798 case SIOCSDRVSPEC: 4799 if (!wg_is_authorized(wg, cmd)) { 4800 return EPERM; 4801 } 4802 switch (ifd->ifd_cmd) { 4803 case WG_IOCTL_SET_PRIVATE_KEY: 4804 error = wg_ioctl_set_private_key(wg, ifd); 4805 break; 4806 case WG_IOCTL_SET_LISTEN_PORT: 4807 error = wg_ioctl_set_listen_port(wg, ifd); 4808 break; 4809 case WG_IOCTL_ADD_PEER: 4810 error = wg_ioctl_add_peer(wg, ifd); 4811 break; 4812 case WG_IOCTL_DELETE_PEER: 4813 error = wg_ioctl_delete_peer(wg, ifd); 4814 break; 4815 default: 4816 error = EINVAL; 4817 break; 4818 } 4819 return error; 4820 case SIOCGDRVSPEC: 4821 return wg_ioctl_get(wg, ifd); 4822 case SIOCSIFFLAGS: 4823 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4824 break; 4825 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4826 case IFF_RUNNING: 4827 /* 4828 * If interface is marked down and it is running, 4829 * then stop and disable it. 4830 */ 4831 if_stop(ifp, 1); 4832 break; 4833 case IFF_UP: 4834 /* 4835 * If interface is marked up and it is stopped, then 4836 * start it. 4837 */ 4838 error = if_init(ifp); 4839 break; 4840 default: 4841 break; 4842 } 4843 return error; 4844 #ifdef WG_RUMPKERNEL 4845 case SIOCSLINKSTR: 4846 error = wg_ioctl_linkstr(wg, ifd); 4847 if (error == 0) 4848 wg->wg_ops = &wg_ops_rumpuser; 4849 return error; 4850 #endif 4851 default: 4852 break; 4853 } 4854 4855 error = ifioctl_common(ifp, cmd, data); 4856 4857 #ifdef WG_RUMPKERNEL 4858 if (!wg_user_mode(wg)) 4859 return error; 4860 4861 /* Do the same to the corresponding tun device on the host */ 4862 /* 4863 * XXX Actually the command has not been handled yet. It 4864 * will be handled via pr_ioctl form doifioctl later. 4865 */ 4866 switch (cmd) { 4867 case SIOCAIFADDR: 4868 case SIOCDIFADDR: { 4869 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4870 struct in_aliasreq *ifra = &_ifra; 4871 KASSERT(error == ENOTTY); 4872 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4873 IFNAMSIZ); 4874 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4875 if (error == 0) 4876 error = ENOTTY; 4877 break; 4878 } 4879 #ifdef INET6 4880 case SIOCAIFADDR_IN6: 4881 case SIOCDIFADDR_IN6: { 4882 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4883 struct in6_aliasreq *ifra = &_ifra; 4884 KASSERT(error == ENOTTY); 4885 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4886 IFNAMSIZ); 4887 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4888 if (error == 0) 4889 error = ENOTTY; 4890 break; 4891 } 4892 #endif 4893 } 4894 #endif /* WG_RUMPKERNEL */ 4895 4896 return error; 4897 } 4898 4899 static int 4900 wg_init(struct ifnet *ifp) 4901 { 4902 4903 ifp->if_flags |= IFF_RUNNING; 4904 4905 /* TODO flush pending packets. */ 4906 return 0; 4907 } 4908 4909 #ifdef ALTQ 4910 static void 4911 wg_start(struct ifnet *ifp) 4912 { 4913 struct mbuf *m; 4914 4915 for (;;) { 4916 IFQ_DEQUEUE(&ifp->if_snd, m); 4917 if (m == NULL) 4918 break; 4919 4920 kpreempt_disable(); 4921 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4922 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4923 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4924 if_name(ifp)); 4925 m_freem(m); 4926 } 4927 kpreempt_enable(); 4928 } 4929 } 4930 #endif 4931 4932 static void 4933 wg_stop(struct ifnet *ifp, int disable) 4934 { 4935 4936 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4937 ifp->if_flags &= ~IFF_RUNNING; 4938 4939 /* Need to do something? */ 4940 } 4941 4942 #ifdef WG_DEBUG_PARAMS 4943 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4944 { 4945 const struct sysctlnode *node = NULL; 4946 4947 sysctl_createv(clog, 0, NULL, &node, 4948 CTLFLAG_PERMANENT, 4949 CTLTYPE_NODE, "wg", 4950 SYSCTL_DESCR("wg(4)"), 4951 NULL, 0, NULL, 0, 4952 CTL_NET, CTL_CREATE, CTL_EOL); 4953 sysctl_createv(clog, 0, &node, NULL, 4954 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4955 CTLTYPE_QUAD, "rekey_after_messages", 4956 SYSCTL_DESCR("session liftime by messages"), 4957 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4958 sysctl_createv(clog, 0, &node, NULL, 4959 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4960 CTLTYPE_INT, "rekey_after_time", 4961 SYSCTL_DESCR("session liftime"), 4962 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4963 sysctl_createv(clog, 0, &node, NULL, 4964 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4965 CTLTYPE_INT, "rekey_timeout", 4966 SYSCTL_DESCR("session handshake retry time"), 4967 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4968 sysctl_createv(clog, 0, &node, NULL, 4969 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4970 CTLTYPE_INT, "rekey_attempt_time", 4971 SYSCTL_DESCR("session handshake timeout"), 4972 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4973 sysctl_createv(clog, 0, &node, NULL, 4974 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4975 CTLTYPE_INT, "keepalive_timeout", 4976 SYSCTL_DESCR("keepalive timeout"), 4977 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4978 sysctl_createv(clog, 0, &node, NULL, 4979 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4980 CTLTYPE_BOOL, "force_underload", 4981 SYSCTL_DESCR("force to detemine under load"), 4982 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4983 sysctl_createv(clog, 0, &node, NULL, 4984 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4985 CTLTYPE_INT, "debug", 4986 SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"), 4987 NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL); 4988 } 4989 #endif 4990 4991 #ifdef WG_RUMPKERNEL 4992 static bool 4993 wg_user_mode(struct wg_softc *wg) 4994 { 4995 4996 return wg->wg_user != NULL; 4997 } 4998 4999 static int 5000 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 5001 { 5002 struct ifnet *ifp = &wg->wg_if; 5003 int error; 5004 5005 if (ifp->if_flags & IFF_UP) 5006 return EBUSY; 5007 5008 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 5009 /* XXX do nothing */ 5010 return 0; 5011 } else if (ifd->ifd_cmd != 0) { 5012 return EINVAL; 5013 } else if (wg->wg_user != NULL) { 5014 return EBUSY; 5015 } 5016 5017 /* Assume \0 included */ 5018 if (ifd->ifd_len > IFNAMSIZ) { 5019 return E2BIG; 5020 } else if (ifd->ifd_len < 1) { 5021 return EINVAL; 5022 } 5023 5024 char tun_name[IFNAMSIZ]; 5025 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 5026 if (error != 0) 5027 return error; 5028 5029 if (strncmp(tun_name, "tun", 3) != 0) 5030 return EINVAL; 5031 5032 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 5033 5034 return error; 5035 } 5036 5037 static int 5038 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 5039 { 5040 int error; 5041 struct psref psref; 5042 struct wg_sockaddr *wgsa; 5043 struct wg_softc *wg = wgp->wgp_sc; 5044 struct iovec iov[1]; 5045 5046 wgsa = wg_get_endpoint_sa(wgp, &psref); 5047 5048 iov[0].iov_base = mtod(m, void *); 5049 iov[0].iov_len = m->m_len; 5050 5051 /* Send messages to a peer via an ordinary socket. */ 5052 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 5053 5054 wg_put_sa(wgp, wgsa, &psref); 5055 5056 m_freem(m); 5057 5058 return error; 5059 } 5060 5061 static void 5062 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 5063 { 5064 struct wg_softc *wg = ifp->if_softc; 5065 struct iovec iov[2]; 5066 struct sockaddr_storage ss; 5067 5068 KASSERT(af == AF_INET || af == AF_INET6); 5069 5070 WG_TRACE(""); 5071 5072 if (af == AF_INET) { 5073 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 5074 struct ip *ip; 5075 5076 KASSERT(m->m_len >= sizeof(struct ip)); 5077 ip = mtod(m, struct ip *); 5078 sockaddr_in_init(sin, &ip->ip_dst, 0); 5079 } else { 5080 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 5081 struct ip6_hdr *ip6; 5082 5083 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 5084 ip6 = mtod(m, struct ip6_hdr *); 5085 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 5086 } 5087 5088 iov[0].iov_base = &ss; 5089 iov[0].iov_len = ss.ss_len; 5090 iov[1].iov_base = mtod(m, void *); 5091 iov[1].iov_len = m->m_len; 5092 5093 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5094 5095 /* Send decrypted packets to users via a tun. */ 5096 rumpuser_wg_send_user(wg->wg_user, iov, 2); 5097 5098 m_freem(m); 5099 } 5100 5101 static int 5102 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 5103 { 5104 int error; 5105 uint16_t old_port = wg->wg_listen_port; 5106 5107 if (port != 0 && old_port == port) 5108 return 0; 5109 5110 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5111 if (error == 0) 5112 wg->wg_listen_port = port; 5113 return error; 5114 } 5115 5116 /* 5117 * Receive user packets. 5118 */ 5119 void 5120 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5121 { 5122 struct ifnet *ifp = &wg->wg_if; 5123 struct mbuf *m; 5124 const struct sockaddr *dst; 5125 5126 WG_TRACE(""); 5127 5128 dst = iov[0].iov_base; 5129 5130 m = m_gethdr(M_DONTWAIT, MT_DATA); 5131 if (m == NULL) 5132 return; 5133 m->m_len = m->m_pkthdr.len = 0; 5134 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5135 5136 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5137 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5138 5139 (void)wg_output(ifp, m, dst, NULL); 5140 } 5141 5142 /* 5143 * Receive packets from a peer. 5144 */ 5145 void 5146 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5147 { 5148 struct mbuf *m; 5149 const struct sockaddr *src; 5150 int bound; 5151 5152 WG_TRACE(""); 5153 5154 src = iov[0].iov_base; 5155 5156 m = m_gethdr(M_DONTWAIT, MT_DATA); 5157 if (m == NULL) 5158 return; 5159 m->m_len = m->m_pkthdr.len = 0; 5160 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5161 5162 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5163 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5164 5165 bound = curlwp_bind(); 5166 wg_handle_packet(wg, m, src); 5167 curlwp_bindx(bound); 5168 } 5169 #endif /* WG_RUMPKERNEL */ 5170 5171 /* 5172 * Module infrastructure 5173 */ 5174 #include "if_module.h" 5175 5176 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5177
Indexes created Thu Sep 25 14:09:45 GMT 2025