if_wg.c revision 1.92
1 /* $NetBSD: if_wg.c,v 1.92 2024/07/26 18:34:35 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.92 2024/07/26 18:34:35 riastradh Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <lib/libkern/libkern.h> 87 88 #include <net/bpf.h> 89 #include <net/if.h> 90 #include <net/if_types.h> 91 #include <net/if_wg.h> 92 #include <net/pktqueue.h> 93 #include <net/route.h> 94 95 #include <netinet/in.h> 96 #include <netinet/in_pcb.h> 97 #include <netinet/in_var.h> 98 #include <netinet/ip.h> 99 #include <netinet/ip_var.h> 100 #include <netinet/udp.h> 101 #include <netinet/udp_var.h> 102 103 #ifdef INET6 104 #include <netinet/ip6.h> 105 #include <netinet6/in6_pcb.h> 106 #include <netinet6/in6_var.h> 107 #include <netinet6/ip6_var.h> 108 #include <netinet6/udp6_var.h> 109 #endif /* INET6 */ 110 111 #include <prop/proplib.h> 112 113 #include <crypto/blake2/blake2s.h> 114 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 115 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 116 #include <crypto/sodium/crypto_scalarmult.h> 117 118 #include "ioconf.h" 119 120 #ifdef WG_RUMPKERNEL 121 #include "wg_user.h" 122 #endif 123 124 /* 125 * Data structures 126 * - struct wg_softc is an instance of wg interfaces 127 * - It has a list of peers (struct wg_peer) 128 * - It has a threadpool job that sends/receives handshake messages and 129 * runs event handlers 130 * - It has its own two routing tables: one is for IPv4 and the other IPv6 131 * - struct wg_peer is a representative of a peer 132 * - It has a struct work to handle handshakes and timer tasks 133 * - It has a pair of session instances (struct wg_session) 134 * - It has a pair of endpoint instances (struct wg_sockaddr) 135 * - Normally one endpoint is used and the second one is used only on 136 * a peer migration (a change of peer's IP address) 137 * - It has a list of IP addresses and sub networks called allowedips 138 * (struct wg_allowedip) 139 * - A packets sent over a session is allowed if its destination matches 140 * any IP addresses or sub networks of the list 141 * - struct wg_session represents a session of a secure tunnel with a peer 142 * - Two instances of sessions belong to a peer; a stable session and a 143 * unstable session 144 * - A handshake process of a session always starts with a unstable instance 145 * - Once a session is established, its instance becomes stable and the 146 * other becomes unstable instead 147 * - Data messages are always sent via a stable session 148 * 149 * Locking notes: 150 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 151 * - Changes to the peer list are serialized by wg_lock 152 * - The peer list may be read with pserialize(9) and psref(9) 153 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 154 * => XXX replace by pserialize when routing table is psz-safe 155 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 156 * only in thread context and serializes: 157 * - the stable and unstable session pointers 158 * - all unstable session state 159 * - Packet processing may be done in softint context: 160 * - The stable session can be read under pserialize(9) or psref(9) 161 * - The stable session is always ESTABLISHED 162 * - On a session swap, we must wait for all readers to release a 163 * reference to a stable session before changing wgs_state and 164 * session states 165 * - Lock order: wg_lock -> wgp_lock 166 */ 167 168 169 #define WGLOG(level, fmt, args...) \ 170 log(level, "%s: " fmt, __func__, ##args) 171 172 #define WG_DEBUG 173 174 /* Debug options */ 175 #ifdef WG_DEBUG 176 /* Output debug logs */ 177 #ifndef WG_DEBUG_LOG 178 #define WG_DEBUG_LOG 179 #endif 180 /* Output trace logs */ 181 #ifndef WG_DEBUG_TRACE 182 #define WG_DEBUG_TRACE 183 #endif 184 /* Output hash values, etc. */ 185 #ifndef WG_DEBUG_DUMP 186 #define WG_DEBUG_DUMP 187 #endif 188 /* debug packets */ 189 #ifndef WG_DEBUG_PACKET 190 #define WG_DEBUG_PACKET 191 #endif 192 /* Make some internal parameters configurable for testing and debugging */ 193 #ifndef WG_DEBUG_PARAMS 194 #define WG_DEBUG_PARAMS 195 #endif 196 #endif /* WG_DEBUG */ 197 198 #ifndef WG_DEBUG 199 # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \ 200 defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS) || \ 201 defined(WG_DEBUG_PACKET) 202 # define WG_DEBUG 203 # endif 204 #endif 205 206 #ifdef WG_DEBUG 207 int wg_debug; 208 #define WG_DEBUG_FLAGS_LOG 1 209 #define WG_DEBUG_FLAGS_TRACE 2 210 #define WG_DEBUG_FLAGS_DUMP 4 211 #define WG_DEBUG_FLAGS_PACKET 8 212 #endif 213 214 215 #ifdef WG_DEBUG_TRACE 216 #define WG_TRACE(msg) do { \ 217 if (wg_debug & WG_DEBUG_FLAGS_TRACE) \ 218 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \ 219 } while (0) 220 #else 221 #define WG_TRACE(msg) __nothing 222 #endif 223 224 #ifdef WG_DEBUG_LOG 225 #define WG_DLOG(fmt, args...) do { \ 226 if (wg_debug & WG_DEBUG_FLAGS_LOG) \ 227 log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \ 228 } while (0) 229 #else 230 #define WG_DLOG(fmt, args...) __nothing 231 #endif 232 233 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 234 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 235 &(wgprc)->wgprc_curpps, 1)) { \ 236 log(level, fmt, ##args); \ 237 } \ 238 } while (0) 239 240 #ifdef WG_DEBUG_PARAMS 241 static bool wg_force_underload = false; 242 #endif 243 244 #ifdef WG_DEBUG_DUMP 245 246 static char enomem[10] = "[enomem]"; 247 248 static char * 249 gethexdump(const void *vp, size_t n) 250 { 251 char *buf; 252 const uint8_t *p = vp; 253 size_t i; 254 255 if (n > (SIZE_MAX - 1) / 3) 256 return enomem; 257 buf = kmem_alloc(3 * n + 1, KM_NOSLEEP); 258 if (buf == NULL) 259 return enomem; 260 for (i = 0; i < n; i++) 261 snprintf(buf + 3 * i, 3 + 1, " %02hhx", p[i]); 262 return buf; 263 } 264 265 static void 266 puthexdump(char *buf, const void *p, size_t n) 267 { 268 269 if (buf == NULL || buf == enomem) 270 return; 271 kmem_free(buf, 3*n + 1); 272 } 273 274 #ifdef WG_RUMPKERNEL 275 static void 276 wg_dump_buf(const char *func, const char *buf, const size_t size) 277 { 278 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 279 return; 280 281 char *hex = gethexdump(buf, size); 282 283 log(LOG_DEBUG, "%s: %s\n", func, hex); 284 puthexdump(hex, buf, size); 285 } 286 #endif 287 288 static void 289 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 290 const size_t size) 291 { 292 if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0) 293 return; 294 295 char *hex = gethexdump(hash, size); 296 297 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex); 298 puthexdump(hex, hash, size); 299 } 300 301 #define WG_DUMP_HASH(name, hash) \ 302 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 303 #define WG_DUMP_HASH48(name, hash) \ 304 wg_dump_hash(__func__, name, hash, 48) 305 #define WG_DUMP_BUF(buf, size) \ 306 wg_dump_buf(__func__, buf, size) 307 #else 308 #define WG_DUMP_HASH(name, hash) __nothing 309 #define WG_DUMP_HASH48(name, hash) __nothing 310 #define WG_DUMP_BUF(buf, size) __nothing 311 #endif /* WG_DEBUG_DUMP */ 312 313 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */ 314 #define WG_MAX_PROPLEN 32766 315 316 #define WG_MTU 1420 317 #define WG_ALLOWEDIPS 16 318 319 #define CURVE25519_KEY_LEN 32 320 #define TAI64N_LEN sizeof(uint32_t) * 3 321 #define POLY1305_AUTHTAG_LEN 16 322 #define HMAC_BLOCK_LEN 64 323 324 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 325 /* [N] 4.3: Hash functions */ 326 #define NOISE_DHLEN 32 327 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 328 #define NOISE_HASHLEN 32 329 #define NOISE_BLOCKLEN 64 330 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 331 /* [N] 5.1: "k" */ 332 #define NOISE_CIPHER_KEY_LEN 32 333 /* 334 * [N] 9.2: "psk" 335 * "... psk is a 32-byte secret value provided by the application." 336 */ 337 #define NOISE_PRESHARED_KEY_LEN 32 338 339 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 340 #define WG_TIMESTAMP_LEN TAI64N_LEN 341 342 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 343 344 #define WG_COOKIE_LEN 16 345 #define WG_MAC_LEN 16 346 #define WG_RANDVAL_LEN 24 347 348 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 349 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 350 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 351 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 352 #define WG_HASH_LEN NOISE_HASHLEN 353 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 354 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 355 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 356 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 357 #define WG_DATA_KEY_LEN 32 358 #define WG_SALT_LEN 24 359 360 /* 361 * The protocol messages 362 */ 363 struct wg_msg { 364 uint32_t wgm_type; 365 } __packed; 366 367 /* [W] 5.4.2 First Message: Initiator to Responder */ 368 struct wg_msg_init { 369 uint32_t wgmi_type; 370 uint32_t wgmi_sender; 371 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 372 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 373 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 374 uint8_t wgmi_mac1[WG_MAC_LEN]; 375 uint8_t wgmi_mac2[WG_MAC_LEN]; 376 } __packed; 377 378 /* [W] 5.4.3 Second Message: Responder to Initiator */ 379 struct wg_msg_resp { 380 uint32_t wgmr_type; 381 uint32_t wgmr_sender; 382 uint32_t wgmr_receiver; 383 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 384 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 385 uint8_t wgmr_mac1[WG_MAC_LEN]; 386 uint8_t wgmr_mac2[WG_MAC_LEN]; 387 } __packed; 388 389 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 390 struct wg_msg_data { 391 uint32_t wgmd_type; 392 uint32_t wgmd_receiver; 393 uint64_t wgmd_counter; 394 uint32_t wgmd_packet[0]; 395 } __packed; 396 397 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 398 struct wg_msg_cookie { 399 uint32_t wgmc_type; 400 uint32_t wgmc_receiver; 401 uint8_t wgmc_salt[WG_SALT_LEN]; 402 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 403 } __packed; 404 405 #define WG_MSG_TYPE_INIT 1 406 #define WG_MSG_TYPE_RESP 2 407 #define WG_MSG_TYPE_COOKIE 3 408 #define WG_MSG_TYPE_DATA 4 409 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 410 411 /* Sliding windows */ 412 413 #define SLIWIN_BITS 2048u 414 #define SLIWIN_TYPE uint32_t 415 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 416 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 417 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 418 419 struct sliwin { 420 SLIWIN_TYPE B[SLIWIN_WORDS]; 421 uint64_t T; 422 }; 423 424 static void 425 sliwin_reset(struct sliwin *W) 426 { 427 428 memset(W, 0, sizeof(*W)); 429 } 430 431 static int 432 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 433 { 434 435 /* 436 * If it's more than one window older than the highest sequence 437 * number we've seen, reject. 438 */ 439 #ifdef __HAVE_ATOMIC64_LOADSTORE 440 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 441 return EAUTH; 442 #endif 443 444 /* 445 * Otherwise, we need to take the lock to decide, so don't 446 * reject just yet. Caller must serialize a call to 447 * sliwin_update in this case. 448 */ 449 return 0; 450 } 451 452 static int 453 sliwin_update(struct sliwin *W, uint64_t S) 454 { 455 unsigned word, bit; 456 457 /* 458 * If it's more than one window older than the highest sequence 459 * number we've seen, reject. 460 */ 461 if (S + SLIWIN_NPKT < W->T) 462 return EAUTH; 463 464 /* 465 * If it's higher than the highest sequence number we've seen, 466 * advance the window. 467 */ 468 if (S > W->T) { 469 uint64_t i = W->T / SLIWIN_BPW; 470 uint64_t j = S / SLIWIN_BPW; 471 unsigned k; 472 473 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 474 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 475 #ifdef __HAVE_ATOMIC64_LOADSTORE 476 atomic_store_relaxed(&W->T, S); 477 #else 478 W->T = S; 479 #endif 480 } 481 482 /* Test and set the bit -- if already set, reject. */ 483 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 484 bit = S % SLIWIN_BPW; 485 if (W->B[word] & (1UL << bit)) 486 return EAUTH; 487 W->B[word] |= 1U << bit; 488 489 /* Accept! */ 490 return 0; 491 } 492 493 struct wg_session { 494 struct wg_peer *wgs_peer; 495 struct psref_target 496 wgs_psref; 497 498 int wgs_state; 499 #define WGS_STATE_UNKNOWN 0 500 #define WGS_STATE_INIT_ACTIVE 1 501 #define WGS_STATE_INIT_PASSIVE 2 502 #define WGS_STATE_ESTABLISHED 3 503 #define WGS_STATE_DESTROYING 4 504 505 time_t wgs_time_established; 506 time_t wgs_time_last_data_sent; 507 bool wgs_is_initiator; 508 509 uint32_t wgs_local_index; 510 uint32_t wgs_remote_index; 511 #ifdef __HAVE_ATOMIC64_LOADSTORE 512 volatile uint64_t 513 wgs_send_counter; 514 #else 515 kmutex_t wgs_send_counter_lock; 516 uint64_t wgs_send_counter; 517 #endif 518 519 struct { 520 kmutex_t lock; 521 struct sliwin window; 522 } *wgs_recvwin; 523 524 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 525 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 526 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 527 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 528 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 529 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 530 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 531 }; 532 533 struct wg_sockaddr { 534 union { 535 struct sockaddr_storage _ss; 536 struct sockaddr _sa; 537 struct sockaddr_in _sin; 538 struct sockaddr_in6 _sin6; 539 }; 540 struct psref_target wgsa_psref; 541 }; 542 543 #define wgsatoss(wgsa) (&(wgsa)->_ss) 544 #define wgsatosa(wgsa) (&(wgsa)->_sa) 545 #define wgsatosin(wgsa) (&(wgsa)->_sin) 546 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 547 548 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 549 550 struct wg_peer; 551 struct wg_allowedip { 552 struct radix_node wga_nodes[2]; 553 struct wg_sockaddr _wga_sa_addr; 554 struct wg_sockaddr _wga_sa_mask; 555 #define wga_sa_addr _wga_sa_addr._sa 556 #define wga_sa_mask _wga_sa_mask._sa 557 558 int wga_family; 559 uint8_t wga_cidr; 560 union { 561 struct in_addr _ip4; 562 struct in6_addr _ip6; 563 } wga_addr; 564 #define wga_addr4 wga_addr._ip4 565 #define wga_addr6 wga_addr._ip6 566 567 struct wg_peer *wga_peer; 568 }; 569 570 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 571 572 struct wg_ppsratecheck { 573 struct timeval wgprc_lasttime; 574 int wgprc_curpps; 575 }; 576 577 struct wg_softc; 578 struct wg_peer { 579 struct wg_softc *wgp_sc; 580 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 581 struct pslist_entry wgp_peerlist_entry; 582 pserialize_t wgp_psz; 583 struct psref_target wgp_psref; 584 kmutex_t *wgp_lock; 585 kmutex_t *wgp_intr_lock; 586 587 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 588 struct wg_sockaddr *wgp_endpoint; 589 struct wg_sockaddr *wgp_endpoint0; 590 volatile unsigned wgp_endpoint_changing; 591 bool wgp_endpoint_available; 592 593 /* The preshared key (optional) */ 594 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 595 596 struct wg_session *wgp_session_stable; 597 struct wg_session *wgp_session_unstable; 598 599 /* first outgoing packet awaiting session initiation */ 600 struct mbuf *wgp_pending; 601 602 /* timestamp in big-endian */ 603 wg_timestamp_t wgp_timestamp_latest_init; 604 605 struct timespec wgp_last_handshake_time; 606 607 callout_t wgp_rekey_timer; 608 callout_t wgp_handshake_timeout_timer; 609 callout_t wgp_session_dtor_timer; 610 611 time_t wgp_handshake_start_time; 612 613 int wgp_n_allowedips; 614 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 615 616 time_t wgp_latest_cookie_time; 617 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 618 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 619 bool wgp_last_sent_mac1_valid; 620 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 621 bool wgp_last_sent_cookie_valid; 622 623 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 624 625 time_t wgp_last_genrandval_time; 626 uint32_t wgp_randval; 627 628 struct wg_ppsratecheck wgp_ppsratecheck; 629 630 struct work wgp_work; 631 unsigned int wgp_tasks; 632 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 633 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 634 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 635 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 636 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 637 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 638 }; 639 640 struct wg_ops; 641 642 struct wg_softc { 643 struct ifnet wg_if; 644 LIST_ENTRY(wg_softc) wg_list; 645 kmutex_t *wg_lock; 646 kmutex_t *wg_intr_lock; 647 krwlock_t *wg_rwlock; 648 649 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 650 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 651 652 int wg_npeers; 653 struct pslist_head wg_peers; 654 struct thmap *wg_peers_bypubkey; 655 struct thmap *wg_peers_byname; 656 struct thmap *wg_sessions_byindex; 657 uint16_t wg_listen_port; 658 659 struct threadpool *wg_threadpool; 660 661 struct threadpool_job wg_job; 662 int wg_upcalls; 663 #define WG_UPCALL_INET __BIT(0) 664 #define WG_UPCALL_INET6 __BIT(1) 665 666 #ifdef INET 667 struct socket *wg_so4; 668 struct radix_node_head *wg_rtable_ipv4; 669 #endif 670 #ifdef INET6 671 struct socket *wg_so6; 672 struct radix_node_head *wg_rtable_ipv6; 673 #endif 674 675 struct wg_ppsratecheck wg_ppsratecheck; 676 677 struct wg_ops *wg_ops; 678 679 #ifdef WG_RUMPKERNEL 680 struct wg_user *wg_user; 681 #endif 682 }; 683 684 /* [W] 6.1 Preliminaries */ 685 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 686 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 687 #define WG_REKEY_AFTER_TIME 120 688 #define WG_REJECT_AFTER_TIME 180 689 #define WG_REKEY_ATTEMPT_TIME 90 690 #define WG_REKEY_TIMEOUT 5 691 #define WG_KEEPALIVE_TIMEOUT 10 692 693 #define WG_COOKIE_TIME 120 694 #define WG_RANDVAL_TIME (2 * 60) 695 696 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 697 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 698 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 699 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 700 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 701 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 702 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 703 704 static struct mbuf * 705 wg_get_mbuf(size_t, size_t); 706 707 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 708 struct mbuf *); 709 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 710 const uint32_t, const uint8_t [WG_MAC_LEN], 711 const struct sockaddr *); 712 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 713 struct wg_session *, const struct wg_msg_init *); 714 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 715 716 static struct wg_peer * 717 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 718 struct psref *); 719 static struct wg_peer * 720 wg_lookup_peer_by_pubkey(struct wg_softc *, 721 const uint8_t [WG_STATIC_KEY_LEN], struct psref *); 722 723 static struct wg_session * 724 wg_lookup_session_by_index(struct wg_softc *, 725 const uint32_t, struct psref *); 726 727 static void wg_update_endpoint_if_necessary(struct wg_peer *, 728 const struct sockaddr *); 729 730 static void wg_schedule_rekey_timer(struct wg_peer *); 731 static void wg_schedule_session_dtor_timer(struct wg_peer *); 732 733 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 734 static void wg_calculate_keys(struct wg_session *, const bool); 735 736 static void wg_clear_states(struct wg_session *); 737 738 static void wg_get_peer(struct wg_peer *, struct psref *); 739 static void wg_put_peer(struct wg_peer *, struct psref *); 740 741 static int wg_send_so(struct wg_peer *, struct mbuf *); 742 static int wg_send_udp(struct wg_peer *, struct mbuf *); 743 static int wg_output(struct ifnet *, struct mbuf *, 744 const struct sockaddr *, const struct rtentry *); 745 static void wg_input(struct ifnet *, struct mbuf *, const int); 746 static int wg_ioctl(struct ifnet *, u_long, void *); 747 static int wg_bind_port(struct wg_softc *, const uint16_t); 748 static int wg_init(struct ifnet *); 749 #ifdef ALTQ 750 static void wg_start(struct ifnet *); 751 #endif 752 static void wg_stop(struct ifnet *, int); 753 754 static void wg_peer_work(struct work *, void *); 755 static void wg_job(struct threadpool_job *); 756 static void wgintr(void *); 757 static void wg_purge_pending_packets(struct wg_peer *); 758 759 static int wg_clone_create(struct if_clone *, int); 760 static int wg_clone_destroy(struct ifnet *); 761 762 struct wg_ops { 763 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 764 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 765 void (*input)(struct ifnet *, struct mbuf *, const int); 766 int (*bind_port)(struct wg_softc *, const uint16_t); 767 }; 768 769 struct wg_ops wg_ops_rumpkernel = { 770 .send_hs_msg = wg_send_so, 771 .send_data_msg = wg_send_udp, 772 .input = wg_input, 773 .bind_port = wg_bind_port, 774 }; 775 776 #ifdef WG_RUMPKERNEL 777 static bool wg_user_mode(struct wg_softc *); 778 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 779 780 static int wg_send_user(struct wg_peer *, struct mbuf *); 781 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 782 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 783 784 struct wg_ops wg_ops_rumpuser = { 785 .send_hs_msg = wg_send_user, 786 .send_data_msg = wg_send_user, 787 .input = wg_input_user, 788 .bind_port = wg_bind_port_user, 789 }; 790 #endif 791 792 #define WG_PEER_READER_FOREACH(wgp, wg) \ 793 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 794 wgp_peerlist_entry) 795 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 796 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 797 wgp_peerlist_entry) 798 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 799 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 800 #define WG_PEER_WRITER_REMOVE(wgp) \ 801 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 802 803 struct wg_route { 804 struct radix_node wgr_nodes[2]; 805 struct wg_peer *wgr_peer; 806 }; 807 808 static struct radix_node_head * 809 wg_rnh(struct wg_softc *wg, const int family) 810 { 811 812 switch (family) { 813 case AF_INET: 814 return wg->wg_rtable_ipv4; 815 #ifdef INET6 816 case AF_INET6: 817 return wg->wg_rtable_ipv6; 818 #endif 819 default: 820 return NULL; 821 } 822 } 823 824 825 /* 826 * Global variables 827 */ 828 static volatile unsigned wg_count __cacheline_aligned; 829 830 struct psref_class *wg_psref_class __read_mostly; 831 832 static struct if_clone wg_cloner = 833 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 834 835 static struct pktqueue *wg_pktq __read_mostly; 836 static struct workqueue *wg_wq __read_mostly; 837 838 void wgattach(int); 839 /* ARGSUSED */ 840 void 841 wgattach(int count) 842 { 843 /* 844 * Nothing to do here, initialization is handled by the 845 * module initialization code in wginit() below). 846 */ 847 } 848 849 static void 850 wginit(void) 851 { 852 853 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 854 855 if_clone_attach(&wg_cloner); 856 } 857 858 /* 859 * XXX Kludge: This should just happen in wginit, but workqueue_create 860 * cannot be run until after CPUs have been detected, and wginit runs 861 * before configure. 862 */ 863 static int 864 wginitqueues(void) 865 { 866 int error __diagused; 867 868 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 869 KASSERT(wg_pktq != NULL); 870 871 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 872 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 873 KASSERT(error == 0); 874 875 return 0; 876 } 877 878 static void 879 wg_guarantee_initialized(void) 880 { 881 static ONCE_DECL(init); 882 int error __diagused; 883 884 error = RUN_ONCE(&init, wginitqueues); 885 KASSERT(error == 0); 886 } 887 888 static int 889 wg_count_inc(void) 890 { 891 unsigned o, n; 892 893 do { 894 o = atomic_load_relaxed(&wg_count); 895 if (o == UINT_MAX) 896 return ENFILE; 897 n = o + 1; 898 } while (atomic_cas_uint(&wg_count, o, n) != o); 899 900 return 0; 901 } 902 903 static void 904 wg_count_dec(void) 905 { 906 unsigned c __diagused; 907 908 c = atomic_dec_uint_nv(&wg_count); 909 KASSERT(c != UINT_MAX); 910 } 911 912 static int 913 wgdetach(void) 914 { 915 916 /* Prevent new interface creation. */ 917 if_clone_detach(&wg_cloner); 918 919 /* Check whether there are any existing interfaces. */ 920 if (atomic_load_relaxed(&wg_count)) { 921 /* Back out -- reattach the cloner. */ 922 if_clone_attach(&wg_cloner); 923 return EBUSY; 924 } 925 926 /* No interfaces left. Nuke it. */ 927 if (wg_wq) 928 workqueue_destroy(wg_wq); 929 if (wg_pktq) 930 pktq_destroy(wg_pktq); 931 psref_class_destroy(wg_psref_class); 932 933 return 0; 934 } 935 936 static void 937 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 938 uint8_t hash[WG_HASH_LEN]) 939 { 940 /* [W] 5.4: CONSTRUCTION */ 941 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 942 /* [W] 5.4: IDENTIFIER */ 943 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 944 struct blake2s state; 945 946 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 947 signature, strlen(signature)); 948 949 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 950 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 951 952 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 953 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 954 blake2s_update(&state, id, strlen(id)); 955 blake2s_final(&state, hash); 956 957 WG_DUMP_HASH("ckey", ckey); 958 WG_DUMP_HASH("hash", hash); 959 } 960 961 static void 962 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 963 const size_t inputsize) 964 { 965 struct blake2s state; 966 967 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 968 blake2s_update(&state, hash, WG_HASH_LEN); 969 blake2s_update(&state, input, inputsize); 970 blake2s_final(&state, hash); 971 } 972 973 static void 974 wg_algo_mac(uint8_t out[], const size_t outsize, 975 const uint8_t key[], const size_t keylen, 976 const uint8_t input1[], const size_t input1len, 977 const uint8_t input2[], const size_t input2len) 978 { 979 struct blake2s state; 980 981 blake2s_init(&state, outsize, key, keylen); 982 983 blake2s_update(&state, input1, input1len); 984 if (input2 != NULL) 985 blake2s_update(&state, input2, input2len); 986 blake2s_final(&state, out); 987 } 988 989 static void 990 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 991 const uint8_t input1[], const size_t input1len, 992 const uint8_t input2[], const size_t input2len) 993 { 994 struct blake2s state; 995 /* [W] 5.4: LABEL-MAC1 */ 996 const char *label = "mac1----"; 997 uint8_t key[WG_HASH_LEN]; 998 999 blake2s_init(&state, sizeof(key), NULL, 0); 1000 blake2s_update(&state, label, strlen(label)); 1001 blake2s_update(&state, input1, input1len); 1002 blake2s_final(&state, key); 1003 1004 blake2s_init(&state, outsize, key, sizeof(key)); 1005 if (input2 != NULL) 1006 blake2s_update(&state, input2, input2len); 1007 blake2s_final(&state, out); 1008 } 1009 1010 static void 1011 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 1012 const uint8_t input1[], const size_t input1len) 1013 { 1014 struct blake2s state; 1015 /* [W] 5.4: LABEL-COOKIE */ 1016 const char *label = "cookie--"; 1017 1018 blake2s_init(&state, outsize, NULL, 0); 1019 blake2s_update(&state, label, strlen(label)); 1020 blake2s_update(&state, input1, input1len); 1021 blake2s_final(&state, out); 1022 } 1023 1024 static void 1025 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 1026 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 1027 { 1028 1029 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1030 1031 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 1032 crypto_scalarmult_base(pubkey, privkey); 1033 } 1034 1035 static void 1036 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 1037 const uint8_t privkey[WG_STATIC_KEY_LEN], 1038 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 1039 { 1040 1041 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1042 1043 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1044 KASSERT(ret == 0); 1045 } 1046 1047 static void 1048 wg_algo_hmac(uint8_t out[], const size_t outlen, 1049 const uint8_t key[], const size_t keylen, 1050 const uint8_t in[], const size_t inlen) 1051 { 1052 #define IPAD 0x36 1053 #define OPAD 0x5c 1054 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1055 uint8_t ipad[HMAC_BLOCK_LEN]; 1056 uint8_t opad[HMAC_BLOCK_LEN]; 1057 size_t i; 1058 struct blake2s state; 1059 1060 KASSERT(outlen == WG_HASH_LEN); 1061 KASSERT(keylen <= HMAC_BLOCK_LEN); 1062 1063 memcpy(hmackey, key, keylen); 1064 1065 for (i = 0; i < sizeof(hmackey); i++) { 1066 ipad[i] = hmackey[i] ^ IPAD; 1067 opad[i] = hmackey[i] ^ OPAD; 1068 } 1069 1070 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1071 blake2s_update(&state, ipad, sizeof(ipad)); 1072 blake2s_update(&state, in, inlen); 1073 blake2s_final(&state, out); 1074 1075 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1076 blake2s_update(&state, opad, sizeof(opad)); 1077 blake2s_update(&state, out, WG_HASH_LEN); 1078 blake2s_final(&state, out); 1079 #undef IPAD 1080 #undef OPAD 1081 } 1082 1083 static void 1084 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1085 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1086 const uint8_t input[], const size_t inputlen) 1087 { 1088 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1089 uint8_t one[1]; 1090 1091 /* 1092 * [N] 4.3: "an input_key_material byte sequence with length 1093 * either zero bytes, 32 bytes, or DHLEN bytes." 1094 */ 1095 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1096 1097 WG_DUMP_HASH("ckey", ckey); 1098 if (input != NULL) 1099 WG_DUMP_HASH("input", input); 1100 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1101 input, inputlen); 1102 WG_DUMP_HASH("tmp1", tmp1); 1103 one[0] = 1; 1104 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1105 one, sizeof(one)); 1106 WG_DUMP_HASH("out1", out1); 1107 if (out2 == NULL) 1108 return; 1109 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1110 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1111 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1112 tmp2, sizeof(tmp2)); 1113 WG_DUMP_HASH("out2", out2); 1114 if (out3 == NULL) 1115 return; 1116 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1117 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1118 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1119 tmp2, sizeof(tmp2)); 1120 WG_DUMP_HASH("out3", out3); 1121 } 1122 1123 static void __noinline 1124 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1125 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1126 const uint8_t local_key[WG_STATIC_KEY_LEN], 1127 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1128 { 1129 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1130 1131 wg_algo_dh(dhout, local_key, remote_key); 1132 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1133 1134 WG_DUMP_HASH("dhout", dhout); 1135 WG_DUMP_HASH("ckey", ckey); 1136 if (cipher_key != NULL) 1137 WG_DUMP_HASH("cipher_key", cipher_key); 1138 } 1139 1140 static void 1141 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1142 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1143 const uint8_t auth[], size_t authlen) 1144 { 1145 uint8_t nonce[(32 + 64) / 8] = {0}; 1146 long long unsigned int outsize; 1147 int error __diagused; 1148 1149 le64enc(&nonce[4], counter); 1150 1151 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1152 plainsize, auth, authlen, NULL, nonce, key); 1153 KASSERT(error == 0); 1154 KASSERT(outsize == expected_outsize); 1155 } 1156 1157 static int 1158 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1159 const uint64_t counter, const uint8_t encrypted[], 1160 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1161 { 1162 uint8_t nonce[(32 + 64) / 8] = {0}; 1163 long long unsigned int outsize; 1164 int error; 1165 1166 le64enc(&nonce[4], counter); 1167 1168 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1169 encrypted, encryptedsize, auth, authlen, nonce, key); 1170 if (error == 0) 1171 KASSERT(outsize == expected_outsize); 1172 return error; 1173 } 1174 1175 static void 1176 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1177 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1178 const uint8_t auth[], size_t authlen, 1179 const uint8_t nonce[WG_SALT_LEN]) 1180 { 1181 long long unsigned int outsize; 1182 int error __diagused; 1183 1184 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1185 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1186 plain, plainsize, auth, authlen, NULL, nonce, key); 1187 KASSERT(error == 0); 1188 KASSERT(outsize == expected_outsize); 1189 } 1190 1191 static int 1192 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1193 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1194 const uint8_t auth[], size_t authlen, 1195 const uint8_t nonce[WG_SALT_LEN]) 1196 { 1197 long long unsigned int outsize; 1198 int error; 1199 1200 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1201 encrypted, encryptedsize, auth, authlen, nonce, key); 1202 if (error == 0) 1203 KASSERT(outsize == expected_outsize); 1204 return error; 1205 } 1206 1207 static void 1208 wg_algo_tai64n(wg_timestamp_t timestamp) 1209 { 1210 struct timespec ts; 1211 1212 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1213 getnanotime(&ts); 1214 /* TAI64 label in external TAI64 format */ 1215 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1216 /* second beginning from 1970 TAI */ 1217 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1218 /* nanosecond in big-endian format */ 1219 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1220 } 1221 1222 /* 1223 * wg_get_stable_session(wgp, psref) 1224 * 1225 * Get a passive reference to the current stable session, or 1226 * return NULL if there is no current stable session. 1227 * 1228 * The pointer is always there but the session is not necessarily 1229 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1230 * the session may transition from ESTABLISHED to DESTROYING while 1231 * holding the passive reference. 1232 */ 1233 static struct wg_session * 1234 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1235 { 1236 int s; 1237 struct wg_session *wgs; 1238 1239 s = pserialize_read_enter(); 1240 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1241 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1242 wgs = NULL; 1243 else 1244 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1245 pserialize_read_exit(s); 1246 1247 return wgs; 1248 } 1249 1250 static void 1251 wg_put_session(struct wg_session *wgs, struct psref *psref) 1252 { 1253 1254 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1255 } 1256 1257 static void 1258 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1259 { 1260 struct wg_peer *wgp = wgs->wgs_peer; 1261 struct wg_session *wgs0 __diagused; 1262 void *garbage; 1263 1264 KASSERT(mutex_owned(wgp->wgp_lock)); 1265 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1266 1267 /* Remove the session from the table. */ 1268 wgs0 = thmap_del(wg->wg_sessions_byindex, 1269 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1270 KASSERT(wgs0 == wgs); 1271 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1272 1273 /* Wait for passive references to drain. */ 1274 pserialize_perform(wgp->wgp_psz); 1275 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1276 1277 /* Free memory, zero state, and transition to UNKNOWN. */ 1278 thmap_gc(wg->wg_sessions_byindex, garbage); 1279 wg_clear_states(wgs); 1280 wgs->wgs_state = WGS_STATE_UNKNOWN; 1281 } 1282 1283 /* 1284 * wg_get_session_index(wg, wgs) 1285 * 1286 * Choose a session index for wgs->wgs_local_index, and store it 1287 * in wg's table of sessions by index. 1288 * 1289 * wgs must be the unstable session of its peer, and must be 1290 * transitioning out of the UNKNOWN state. 1291 */ 1292 static void 1293 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1294 { 1295 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1296 struct wg_session *wgs0; 1297 uint32_t index; 1298 1299 KASSERT(mutex_owned(wgp->wgp_lock)); 1300 KASSERT(wgs == wgp->wgp_session_unstable); 1301 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1302 1303 do { 1304 /* Pick a uniform random index. */ 1305 index = cprng_strong32(); 1306 1307 /* Try to take it. */ 1308 wgs->wgs_local_index = index; 1309 wgs0 = thmap_put(wg->wg_sessions_byindex, 1310 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1311 1312 /* If someone else beat us, start over. */ 1313 } while (__predict_false(wgs0 != wgs)); 1314 } 1315 1316 /* 1317 * wg_put_session_index(wg, wgs) 1318 * 1319 * Remove wgs from the table of sessions by index, wait for any 1320 * passive references to drain, and transition the session to the 1321 * UNKNOWN state. 1322 * 1323 * wgs must be the unstable session of its peer, and must not be 1324 * UNKNOWN or ESTABLISHED. 1325 */ 1326 static void 1327 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1328 { 1329 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1330 1331 KASSERT(mutex_owned(wgp->wgp_lock)); 1332 KASSERT(wgs == wgp->wgp_session_unstable); 1333 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1334 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1335 1336 wg_destroy_session(wg, wgs); 1337 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1338 } 1339 1340 /* 1341 * Handshake patterns 1342 * 1343 * [W] 5: "These messages use the "IK" pattern from Noise" 1344 * [N] 7.5. Interactive handshake patterns (fundamental) 1345 * "The first character refers to the initiators static key:" 1346 * "I = Static key for initiator Immediately transmitted to responder, 1347 * despite reduced or absent identity hiding" 1348 * "The second character refers to the responders static key:" 1349 * "K = Static key for responder Known to initiator" 1350 * "IK: 1351 * <- s 1352 * ... 1353 * -> e, es, s, ss 1354 * <- e, ee, se" 1355 * [N] 9.4. Pattern modifiers 1356 * "IKpsk2: 1357 * <- s 1358 * ... 1359 * -> e, es, s, ss 1360 * <- e, ee, se, psk" 1361 */ 1362 static void 1363 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1364 struct wg_session *wgs, struct wg_msg_init *wgmi) 1365 { 1366 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1367 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1368 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1369 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1370 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1371 1372 KASSERT(mutex_owned(wgp->wgp_lock)); 1373 KASSERT(wgs == wgp->wgp_session_unstable); 1374 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1375 1376 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1377 wgmi->wgmi_sender = wgs->wgs_local_index; 1378 1379 /* [W] 5.4.2: First Message: Initiator to Responder */ 1380 1381 /* Ci := HASH(CONSTRUCTION) */ 1382 /* Hi := HASH(Ci || IDENTIFIER) */ 1383 wg_init_key_and_hash(ckey, hash); 1384 /* Hi := HASH(Hi || Sr^pub) */ 1385 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1386 1387 WG_DUMP_HASH("hash", hash); 1388 1389 /* [N] 2.2: "e" */ 1390 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1391 wg_algo_generate_keypair(pubkey, privkey); 1392 /* Ci := KDF1(Ci, Ei^pub) */ 1393 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1394 /* msg.ephemeral := Ei^pub */ 1395 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1396 /* Hi := HASH(Hi || msg.ephemeral) */ 1397 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1398 1399 WG_DUMP_HASH("ckey", ckey); 1400 WG_DUMP_HASH("hash", hash); 1401 1402 /* [N] 2.2: "es" */ 1403 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1404 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1405 1406 /* [N] 2.2: "s" */ 1407 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1408 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1409 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1410 hash, sizeof(hash)); 1411 /* Hi := HASH(Hi || msg.static) */ 1412 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1413 1414 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1415 1416 /* [N] 2.2: "ss" */ 1417 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1418 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1419 1420 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1421 wg_timestamp_t timestamp; 1422 wg_algo_tai64n(timestamp); 1423 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1424 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1425 /* Hi := HASH(Hi || msg.timestamp) */ 1426 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1427 1428 /* [W] 5.4.4 Cookie MACs */ 1429 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1430 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1431 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1432 /* Need mac1 to decrypt a cookie from a cookie message */ 1433 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1434 sizeof(wgp->wgp_last_sent_mac1)); 1435 wgp->wgp_last_sent_mac1_valid = true; 1436 1437 if (wgp->wgp_latest_cookie_time == 0 || 1438 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1439 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1440 else { 1441 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1442 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1443 (const uint8_t *)wgmi, 1444 offsetof(struct wg_msg_init, wgmi_mac2), 1445 NULL, 0); 1446 } 1447 1448 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1449 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1450 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1451 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1452 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1453 } 1454 1455 static void __noinline 1456 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1457 const struct sockaddr *src) 1458 { 1459 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1460 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1461 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1462 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1463 struct wg_peer *wgp; 1464 struct wg_session *wgs; 1465 int error, ret; 1466 struct psref psref_peer; 1467 uint8_t mac1[WG_MAC_LEN]; 1468 1469 WG_TRACE("init msg received"); 1470 1471 wg_algo_mac_mac1(mac1, sizeof(mac1), 1472 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1473 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1474 1475 /* 1476 * [W] 5.3: Denial of Service Mitigation & Cookies 1477 * "the responder, ..., must always reject messages with an invalid 1478 * msg.mac1" 1479 */ 1480 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1481 WG_DLOG("mac1 is invalid\n"); 1482 return; 1483 } 1484 1485 /* 1486 * [W] 5.4.2: First Message: Initiator to Responder 1487 * "When the responder receives this message, it does the same 1488 * operations so that its final state variables are identical, 1489 * replacing the operands of the DH function to produce equivalent 1490 * values." 1491 * Note that the following comments of operations are just copies of 1492 * the initiator's ones. 1493 */ 1494 1495 /* Ci := HASH(CONSTRUCTION) */ 1496 /* Hi := HASH(Ci || IDENTIFIER) */ 1497 wg_init_key_and_hash(ckey, hash); 1498 /* Hi := HASH(Hi || Sr^pub) */ 1499 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1500 1501 /* [N] 2.2: "e" */ 1502 /* Ci := KDF1(Ci, Ei^pub) */ 1503 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1504 sizeof(wgmi->wgmi_ephemeral)); 1505 /* Hi := HASH(Hi || msg.ephemeral) */ 1506 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1507 1508 WG_DUMP_HASH("ckey", ckey); 1509 1510 /* [N] 2.2: "es" */ 1511 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1512 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1513 1514 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1515 1516 /* [N] 2.2: "s" */ 1517 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1518 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1519 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1520 if (error != 0) { 1521 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1522 "%s: wg_algo_aead_dec for secret key failed\n", 1523 if_name(&wg->wg_if)); 1524 return; 1525 } 1526 /* Hi := HASH(Hi || msg.static) */ 1527 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1528 1529 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1530 if (wgp == NULL) { 1531 WG_DLOG("peer not found\n"); 1532 return; 1533 } 1534 1535 /* 1536 * Lock the peer to serialize access to cookie state. 1537 * 1538 * XXX Can we safely avoid holding the lock across DH? Take it 1539 * just to verify mac2 and then unlock/DH/lock? 1540 */ 1541 mutex_enter(wgp->wgp_lock); 1542 1543 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1544 WG_TRACE("under load"); 1545 /* 1546 * [W] 5.3: Denial of Service Mitigation & Cookies 1547 * "the responder, ..., and when under load may reject messages 1548 * with an invalid msg.mac2. If the responder receives a 1549 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1550 * and is under load, it may respond with a cookie reply 1551 * message" 1552 */ 1553 uint8_t zero[WG_MAC_LEN] = {0}; 1554 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1555 WG_TRACE("sending a cookie message: no cookie included"); 1556 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1557 wgmi->wgmi_mac1, src); 1558 goto out; 1559 } 1560 if (!wgp->wgp_last_sent_cookie_valid) { 1561 WG_TRACE("sending a cookie message: no cookie sent ever"); 1562 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1563 wgmi->wgmi_mac1, src); 1564 goto out; 1565 } 1566 uint8_t mac2[WG_MAC_LEN]; 1567 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1568 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1569 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1570 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1571 WG_DLOG("mac2 is invalid\n"); 1572 goto out; 1573 } 1574 WG_TRACE("under load, but continue to sending"); 1575 } 1576 1577 /* [N] 2.2: "ss" */ 1578 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1579 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1580 1581 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1582 wg_timestamp_t timestamp; 1583 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1584 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1585 hash, sizeof(hash)); 1586 if (error != 0) { 1587 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1588 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1589 if_name(&wg->wg_if), wgp->wgp_name); 1590 goto out; 1591 } 1592 /* Hi := HASH(Hi || msg.timestamp) */ 1593 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1594 1595 /* 1596 * [W] 5.1 "The responder keeps track of the greatest timestamp 1597 * received per peer and discards packets containing 1598 * timestamps less than or equal to it." 1599 */ 1600 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1601 sizeof(timestamp)); 1602 if (ret <= 0) { 1603 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1604 "%s: peer %s: invalid init msg: timestamp is old\n", 1605 if_name(&wg->wg_if), wgp->wgp_name); 1606 goto out; 1607 } 1608 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1609 1610 /* 1611 * Message is good -- we're committing to handle it now, unless 1612 * we were already initiating a session. 1613 */ 1614 wgs = wgp->wgp_session_unstable; 1615 switch (wgs->wgs_state) { 1616 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1617 wg_get_session_index(wg, wgs); 1618 break; 1619 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1620 WG_TRACE("Session already initializing, ignoring the message"); 1621 goto out; 1622 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1623 WG_TRACE("Session already initializing, destroying old states"); 1624 wg_clear_states(wgs); 1625 /* keep session index */ 1626 break; 1627 case WGS_STATE_ESTABLISHED: /* can't happen */ 1628 panic("unstable session can't be established"); 1629 break; 1630 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1631 WG_TRACE("Session destroying, but force to clear"); 1632 callout_stop(&wgp->wgp_session_dtor_timer); 1633 wg_clear_states(wgs); 1634 /* keep session index */ 1635 break; 1636 default: 1637 panic("invalid session state: %d", wgs->wgs_state); 1638 } 1639 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1640 1641 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1642 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1643 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1644 sizeof(wgmi->wgmi_ephemeral)); 1645 1646 wg_update_endpoint_if_necessary(wgp, src); 1647 1648 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1649 1650 wg_calculate_keys(wgs, false); 1651 wg_clear_states(wgs); 1652 1653 out: 1654 mutex_exit(wgp->wgp_lock); 1655 wg_put_peer(wgp, &psref_peer); 1656 } 1657 1658 static struct socket * 1659 wg_get_so_by_af(struct wg_softc *wg, const int af) 1660 { 1661 1662 switch (af) { 1663 #ifdef INET 1664 case AF_INET: 1665 return wg->wg_so4; 1666 #endif 1667 #ifdef INET6 1668 case AF_INET6: 1669 return wg->wg_so6; 1670 #endif 1671 default: 1672 panic("wg: no such af: %d", af); 1673 } 1674 } 1675 1676 static struct socket * 1677 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1678 { 1679 1680 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1681 } 1682 1683 static struct wg_sockaddr * 1684 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1685 { 1686 struct wg_sockaddr *wgsa; 1687 int s; 1688 1689 s = pserialize_read_enter(); 1690 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1691 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1692 pserialize_read_exit(s); 1693 1694 return wgsa; 1695 } 1696 1697 static void 1698 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1699 { 1700 1701 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1702 } 1703 1704 static int 1705 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1706 { 1707 int error; 1708 struct socket *so; 1709 struct psref psref; 1710 struct wg_sockaddr *wgsa; 1711 1712 wgsa = wg_get_endpoint_sa(wgp, &psref); 1713 so = wg_get_so_by_peer(wgp, wgsa); 1714 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1715 wg_put_sa(wgp, wgsa, &psref); 1716 1717 return error; 1718 } 1719 1720 static int 1721 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1722 { 1723 int error; 1724 struct mbuf *m; 1725 struct wg_msg_init *wgmi; 1726 struct wg_session *wgs; 1727 1728 KASSERT(mutex_owned(wgp->wgp_lock)); 1729 1730 wgs = wgp->wgp_session_unstable; 1731 /* XXX pull dispatch out into wg_task_send_init_message */ 1732 switch (wgs->wgs_state) { 1733 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1734 wg_get_session_index(wg, wgs); 1735 break; 1736 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1737 WG_TRACE("Session already initializing, skip starting new one"); 1738 return EBUSY; 1739 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1740 WG_TRACE("Session already initializing, destroying old states"); 1741 wg_clear_states(wgs); 1742 /* keep session index */ 1743 break; 1744 case WGS_STATE_ESTABLISHED: /* can't happen */ 1745 panic("unstable session can't be established"); 1746 break; 1747 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1748 WG_TRACE("Session destroying"); 1749 /* XXX should wait? */ 1750 return EBUSY; 1751 } 1752 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1753 1754 m = m_gethdr(M_WAIT, MT_DATA); 1755 if (sizeof(*wgmi) > MHLEN) { 1756 m_clget(m, M_WAIT); 1757 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1758 } 1759 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1760 wgmi = mtod(m, struct wg_msg_init *); 1761 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1762 1763 error = wg->wg_ops->send_hs_msg(wgp, m); 1764 if (error == 0) { 1765 WG_TRACE("init msg sent"); 1766 1767 if (wgp->wgp_handshake_start_time == 0) 1768 wgp->wgp_handshake_start_time = time_uptime; 1769 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1770 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1771 } else { 1772 wg_put_session_index(wg, wgs); 1773 /* Initiation failed; toss packet waiting for it if any. */ 1774 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 1775 m_freem(m); 1776 } 1777 1778 return error; 1779 } 1780 1781 static void 1782 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1783 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1784 const struct wg_msg_init *wgmi) 1785 { 1786 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1787 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1788 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1789 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1790 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1791 1792 KASSERT(mutex_owned(wgp->wgp_lock)); 1793 KASSERT(wgs == wgp->wgp_session_unstable); 1794 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1795 1796 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1797 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1798 1799 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1800 wgmr->wgmr_sender = wgs->wgs_local_index; 1801 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1802 1803 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1804 1805 /* [N] 2.2: "e" */ 1806 /* Er^priv, Er^pub := DH-GENERATE() */ 1807 wg_algo_generate_keypair(pubkey, privkey); 1808 /* Cr := KDF1(Cr, Er^pub) */ 1809 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1810 /* msg.ephemeral := Er^pub */ 1811 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1812 /* Hr := HASH(Hr || msg.ephemeral) */ 1813 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1814 1815 WG_DUMP_HASH("ckey", ckey); 1816 WG_DUMP_HASH("hash", hash); 1817 1818 /* [N] 2.2: "ee" */ 1819 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1820 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1821 1822 /* [N] 2.2: "se" */ 1823 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1824 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1825 1826 /* [N] 9.2: "psk" */ 1827 { 1828 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1829 /* Cr, r, k := KDF3(Cr, Q) */ 1830 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1831 sizeof(wgp->wgp_psk)); 1832 /* Hr := HASH(Hr || r) */ 1833 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1834 } 1835 1836 /* msg.empty := AEAD(k, 0, e, Hr) */ 1837 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1838 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1839 /* Hr := HASH(Hr || msg.empty) */ 1840 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1841 1842 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1843 1844 /* [W] 5.4.4: Cookie MACs */ 1845 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1846 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1847 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1848 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1849 /* Need mac1 to decrypt a cookie from a cookie message */ 1850 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1851 sizeof(wgp->wgp_last_sent_mac1)); 1852 wgp->wgp_last_sent_mac1_valid = true; 1853 1854 if (wgp->wgp_latest_cookie_time == 0 || 1855 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1856 /* msg.mac2 := 0^16 */ 1857 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1858 else { 1859 /* msg.mac2 := MAC(Lm, msg_b) */ 1860 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1861 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1862 (const uint8_t *)wgmr, 1863 offsetof(struct wg_msg_resp, wgmr_mac2), 1864 NULL, 0); 1865 } 1866 1867 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1868 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1869 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1870 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1871 wgs->wgs_remote_index = wgmi->wgmi_sender; 1872 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1873 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1874 } 1875 1876 static void 1877 wg_swap_sessions(struct wg_peer *wgp) 1878 { 1879 struct wg_session *wgs, *wgs_prev; 1880 1881 KASSERT(mutex_owned(wgp->wgp_lock)); 1882 1883 wgs = wgp->wgp_session_unstable; 1884 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1885 1886 wgs_prev = wgp->wgp_session_stable; 1887 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1888 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1889 atomic_store_release(&wgp->wgp_session_stable, wgs); 1890 wgp->wgp_session_unstable = wgs_prev; 1891 } 1892 1893 static void __noinline 1894 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1895 const struct sockaddr *src) 1896 { 1897 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1898 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1899 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1900 struct wg_peer *wgp; 1901 struct wg_session *wgs; 1902 struct psref psref; 1903 int error; 1904 uint8_t mac1[WG_MAC_LEN]; 1905 struct wg_session *wgs_prev; 1906 struct mbuf *m; 1907 1908 wg_algo_mac_mac1(mac1, sizeof(mac1), 1909 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1910 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1911 1912 /* 1913 * [W] 5.3: Denial of Service Mitigation & Cookies 1914 * "the responder, ..., must always reject messages with an invalid 1915 * msg.mac1" 1916 */ 1917 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1918 WG_DLOG("mac1 is invalid\n"); 1919 return; 1920 } 1921 1922 WG_TRACE("resp msg received"); 1923 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1924 if (wgs == NULL) { 1925 WG_TRACE("No session found"); 1926 return; 1927 } 1928 1929 wgp = wgs->wgs_peer; 1930 1931 mutex_enter(wgp->wgp_lock); 1932 1933 /* If we weren't waiting for a handshake response, drop it. */ 1934 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1935 WG_TRACE("peer sent spurious handshake response, ignoring"); 1936 goto out; 1937 } 1938 1939 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1940 WG_TRACE("under load"); 1941 /* 1942 * [W] 5.3: Denial of Service Mitigation & Cookies 1943 * "the responder, ..., and when under load may reject messages 1944 * with an invalid msg.mac2. If the responder receives a 1945 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1946 * and is under load, it may respond with a cookie reply 1947 * message" 1948 */ 1949 uint8_t zero[WG_MAC_LEN] = {0}; 1950 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1951 WG_TRACE("sending a cookie message: no cookie included"); 1952 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1953 wgmr->wgmr_mac1, src); 1954 goto out; 1955 } 1956 if (!wgp->wgp_last_sent_cookie_valid) { 1957 WG_TRACE("sending a cookie message: no cookie sent ever"); 1958 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1959 wgmr->wgmr_mac1, src); 1960 goto out; 1961 } 1962 uint8_t mac2[WG_MAC_LEN]; 1963 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1964 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1965 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1966 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1967 WG_DLOG("mac2 is invalid\n"); 1968 goto out; 1969 } 1970 WG_TRACE("under load, but continue to sending"); 1971 } 1972 1973 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1974 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1975 1976 /* 1977 * [W] 5.4.3 Second Message: Responder to Initiator 1978 * "When the initiator receives this message, it does the same 1979 * operations so that its final state variables are identical, 1980 * replacing the operands of the DH function to produce equivalent 1981 * values." 1982 * Note that the following comments of operations are just copies of 1983 * the initiator's ones. 1984 */ 1985 1986 /* [N] 2.2: "e" */ 1987 /* Cr := KDF1(Cr, Er^pub) */ 1988 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1989 sizeof(wgmr->wgmr_ephemeral)); 1990 /* Hr := HASH(Hr || msg.ephemeral) */ 1991 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1992 1993 WG_DUMP_HASH("ckey", ckey); 1994 WG_DUMP_HASH("hash", hash); 1995 1996 /* [N] 2.2: "ee" */ 1997 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1998 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1999 wgmr->wgmr_ephemeral); 2000 2001 /* [N] 2.2: "se" */ 2002 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 2003 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 2004 2005 /* [N] 9.2: "psk" */ 2006 { 2007 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 2008 /* Cr, r, k := KDF3(Cr, Q) */ 2009 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 2010 sizeof(wgp->wgp_psk)); 2011 /* Hr := HASH(Hr || r) */ 2012 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 2013 } 2014 2015 { 2016 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 2017 /* msg.empty := AEAD(k, 0, e, Hr) */ 2018 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 2019 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 2020 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 2021 if (error != 0) { 2022 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2023 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 2024 if_name(&wg->wg_if), wgp->wgp_name); 2025 goto out; 2026 } 2027 /* Hr := HASH(Hr || msg.empty) */ 2028 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 2029 } 2030 2031 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 2032 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 2033 wgs->wgs_remote_index = wgmr->wgmr_sender; 2034 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 2035 2036 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 2037 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2038 wgs->wgs_time_established = time_uptime; 2039 wgs->wgs_time_last_data_sent = 0; 2040 wgs->wgs_is_initiator = true; 2041 wg_calculate_keys(wgs, true); 2042 wg_clear_states(wgs); 2043 WG_TRACE("WGS_STATE_ESTABLISHED"); 2044 2045 callout_stop(&wgp->wgp_handshake_timeout_timer); 2046 2047 wg_swap_sessions(wgp); 2048 KASSERT(wgs == wgp->wgp_session_stable); 2049 wgs_prev = wgp->wgp_session_unstable; 2050 getnanotime(&wgp->wgp_last_handshake_time); 2051 wgp->wgp_handshake_start_time = 0; 2052 wgp->wgp_last_sent_mac1_valid = false; 2053 wgp->wgp_last_sent_cookie_valid = false; 2054 2055 wg_schedule_rekey_timer(wgp); 2056 2057 wg_update_endpoint_if_necessary(wgp, src); 2058 2059 /* 2060 * If we had a data packet queued up, send it; otherwise send a 2061 * keepalive message -- either way we have to send something 2062 * immediately or else the responder will never answer. 2063 */ 2064 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2065 kpreempt_disable(); 2066 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2067 M_SETCTX(m, wgp); 2068 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2069 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2070 if_name(&wg->wg_if)); 2071 m_freem(m); 2072 } 2073 kpreempt_enable(); 2074 } else { 2075 wg_send_keepalive_msg(wgp, wgs); 2076 } 2077 2078 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2079 /* Wait for wg_get_stable_session to drain. */ 2080 pserialize_perform(wgp->wgp_psz); 2081 2082 /* Transition ESTABLISHED->DESTROYING. */ 2083 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2084 2085 /* We can't destroy the old session immediately */ 2086 wg_schedule_session_dtor_timer(wgp); 2087 } else { 2088 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2089 "state=%d", wgs_prev->wgs_state); 2090 } 2091 2092 out: 2093 mutex_exit(wgp->wgp_lock); 2094 wg_put_session(wgs, &psref); 2095 } 2096 2097 static int 2098 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2099 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2100 { 2101 int error; 2102 struct mbuf *m; 2103 struct wg_msg_resp *wgmr; 2104 2105 KASSERT(mutex_owned(wgp->wgp_lock)); 2106 KASSERT(wgs == wgp->wgp_session_unstable); 2107 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 2108 2109 m = m_gethdr(M_WAIT, MT_DATA); 2110 if (sizeof(*wgmr) > MHLEN) { 2111 m_clget(m, M_WAIT); 2112 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2113 } 2114 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2115 wgmr = mtod(m, struct wg_msg_resp *); 2116 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2117 2118 error = wg->wg_ops->send_hs_msg(wgp, m); 2119 if (error == 0) 2120 WG_TRACE("resp msg sent"); 2121 return error; 2122 } 2123 2124 static struct wg_peer * 2125 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2126 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2127 { 2128 struct wg_peer *wgp; 2129 2130 int s = pserialize_read_enter(); 2131 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2132 if (wgp != NULL) 2133 wg_get_peer(wgp, psref); 2134 pserialize_read_exit(s); 2135 2136 return wgp; 2137 } 2138 2139 static void 2140 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2141 struct wg_msg_cookie *wgmc, const uint32_t sender, 2142 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2143 { 2144 uint8_t cookie[WG_COOKIE_LEN]; 2145 uint8_t key[WG_HASH_LEN]; 2146 uint8_t addr[sizeof(struct in6_addr)]; 2147 size_t addrlen; 2148 uint16_t uh_sport; /* be */ 2149 2150 KASSERT(mutex_owned(wgp->wgp_lock)); 2151 2152 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2153 wgmc->wgmc_receiver = sender; 2154 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2155 2156 /* 2157 * [W] 5.4.7: Under Load: Cookie Reply Message 2158 * "The secret variable, Rm, changes every two minutes to a 2159 * random value" 2160 */ 2161 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2162 wgp->wgp_randval = cprng_strong32(); 2163 wgp->wgp_last_genrandval_time = time_uptime; 2164 } 2165 2166 switch (src->sa_family) { 2167 case AF_INET: { 2168 const struct sockaddr_in *sin = satocsin(src); 2169 addrlen = sizeof(sin->sin_addr); 2170 memcpy(addr, &sin->sin_addr, addrlen); 2171 uh_sport = sin->sin_port; 2172 break; 2173 } 2174 #ifdef INET6 2175 case AF_INET6: { 2176 const struct sockaddr_in6 *sin6 = satocsin6(src); 2177 addrlen = sizeof(sin6->sin6_addr); 2178 memcpy(addr, &sin6->sin6_addr, addrlen); 2179 uh_sport = sin6->sin6_port; 2180 break; 2181 } 2182 #endif 2183 default: 2184 panic("invalid af=%d", src->sa_family); 2185 } 2186 2187 wg_algo_mac(cookie, sizeof(cookie), 2188 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2189 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2190 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2191 sizeof(wg->wg_pubkey)); 2192 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2193 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2194 2195 /* Need to store to calculate mac2 */ 2196 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2197 wgp->wgp_last_sent_cookie_valid = true; 2198 } 2199 2200 static int 2201 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2202 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2203 const struct sockaddr *src) 2204 { 2205 int error; 2206 struct mbuf *m; 2207 struct wg_msg_cookie *wgmc; 2208 2209 KASSERT(mutex_owned(wgp->wgp_lock)); 2210 2211 m = m_gethdr(M_WAIT, MT_DATA); 2212 if (sizeof(*wgmc) > MHLEN) { 2213 m_clget(m, M_WAIT); 2214 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2215 } 2216 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2217 wgmc = mtod(m, struct wg_msg_cookie *); 2218 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2219 2220 error = wg->wg_ops->send_hs_msg(wgp, m); 2221 if (error == 0) 2222 WG_TRACE("cookie msg sent"); 2223 return error; 2224 } 2225 2226 static bool 2227 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2228 { 2229 #ifdef WG_DEBUG_PARAMS 2230 if (wg_force_underload) 2231 return true; 2232 #endif 2233 2234 /* 2235 * XXX we don't have a means of a load estimation. The purpose of 2236 * the mechanism is a DoS mitigation, so we consider frequent handshake 2237 * messages as (a kind of) load; if a message of the same type comes 2238 * to a peer within 1 second, we consider we are under load. 2239 */ 2240 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2241 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2242 return (time_uptime - last) == 0; 2243 } 2244 2245 static void 2246 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2247 { 2248 2249 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2250 2251 /* 2252 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2253 */ 2254 if (initiator) { 2255 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2256 wgs->wgs_chaining_key, NULL, 0); 2257 } else { 2258 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2259 wgs->wgs_chaining_key, NULL, 0); 2260 } 2261 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2262 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2263 } 2264 2265 static uint64_t 2266 wg_session_get_send_counter(struct wg_session *wgs) 2267 { 2268 #ifdef __HAVE_ATOMIC64_LOADSTORE 2269 return atomic_load_relaxed(&wgs->wgs_send_counter); 2270 #else 2271 uint64_t send_counter; 2272 2273 mutex_enter(&wgs->wgs_send_counter_lock); 2274 send_counter = wgs->wgs_send_counter; 2275 mutex_exit(&wgs->wgs_send_counter_lock); 2276 2277 return send_counter; 2278 #endif 2279 } 2280 2281 static uint64_t 2282 wg_session_inc_send_counter(struct wg_session *wgs) 2283 { 2284 #ifdef __HAVE_ATOMIC64_LOADSTORE 2285 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2286 #else 2287 uint64_t send_counter; 2288 2289 mutex_enter(&wgs->wgs_send_counter_lock); 2290 send_counter = wgs->wgs_send_counter++; 2291 mutex_exit(&wgs->wgs_send_counter_lock); 2292 2293 return send_counter; 2294 #endif 2295 } 2296 2297 static void 2298 wg_clear_states(struct wg_session *wgs) 2299 { 2300 2301 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2302 2303 wgs->wgs_send_counter = 0; 2304 sliwin_reset(&wgs->wgs_recvwin->window); 2305 2306 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2307 wgs_clear(handshake_hash); 2308 wgs_clear(chaining_key); 2309 wgs_clear(ephemeral_key_pub); 2310 wgs_clear(ephemeral_key_priv); 2311 wgs_clear(ephemeral_key_peer); 2312 #undef wgs_clear 2313 } 2314 2315 static struct wg_session * 2316 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2317 struct psref *psref) 2318 { 2319 struct wg_session *wgs; 2320 2321 int s = pserialize_read_enter(); 2322 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2323 if (wgs != NULL) { 2324 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2325 WGS_STATE_UNKNOWN); 2326 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2327 } 2328 pserialize_read_exit(s); 2329 2330 return wgs; 2331 } 2332 2333 static void 2334 wg_schedule_rekey_timer(struct wg_peer *wgp) 2335 { 2336 int timeout = MIN(wg_rekey_after_time, (unsigned)(INT_MAX / hz)); 2337 2338 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2339 } 2340 2341 static void 2342 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2343 { 2344 struct mbuf *m; 2345 2346 /* 2347 * [W] 6.5 Passive Keepalive 2348 * "A keepalive message is simply a transport data message with 2349 * a zero-length encapsulated encrypted inner-packet." 2350 */ 2351 WG_TRACE(""); 2352 m = m_gethdr(M_WAIT, MT_DATA); 2353 wg_send_data_msg(wgp, wgs, m); 2354 } 2355 2356 static bool 2357 wg_need_to_send_init_message(struct wg_session *wgs) 2358 { 2359 /* 2360 * [W] 6.2 Transport Message Limits 2361 * "if a peer is the initiator of a current secure session, 2362 * WireGuard will send a handshake initiation message to begin 2363 * a new secure session ... if after receiving a transport data 2364 * message, the current secure session is (REJECT-AFTER-TIME 2365 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2366 * not yet acted upon this event." 2367 */ 2368 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2369 (time_uptime - wgs->wgs_time_established) >= 2370 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2371 } 2372 2373 static void 2374 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2375 { 2376 2377 mutex_enter(wgp->wgp_intr_lock); 2378 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2379 if (wgp->wgp_tasks == 0) 2380 /* 2381 * XXX If the current CPU is already loaded -- e.g., if 2382 * there's already a bunch of handshakes queued up -- 2383 * consider tossing this over to another CPU to 2384 * distribute the load. 2385 */ 2386 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2387 wgp->wgp_tasks |= task; 2388 mutex_exit(wgp->wgp_intr_lock); 2389 } 2390 2391 static void 2392 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2393 { 2394 struct wg_sockaddr *wgsa_prev; 2395 2396 WG_TRACE("Changing endpoint"); 2397 2398 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2399 wgsa_prev = wgp->wgp_endpoint; 2400 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2401 wgp->wgp_endpoint0 = wgsa_prev; 2402 atomic_store_release(&wgp->wgp_endpoint_available, true); 2403 2404 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2405 } 2406 2407 static bool 2408 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2409 { 2410 uint16_t packet_len; 2411 const struct ip *ip; 2412 2413 if (__predict_false(decrypted_len < sizeof(*ip))) { 2414 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2415 sizeof(*ip)); 2416 return false; 2417 } 2418 2419 ip = (const struct ip *)packet; 2420 if (ip->ip_v == 4) 2421 *af = AF_INET; 2422 else if (ip->ip_v == 6) 2423 *af = AF_INET6; 2424 else { 2425 WG_DLOG("ip_v=%d\n", ip->ip_v); 2426 return false; 2427 } 2428 2429 WG_DLOG("af=%d\n", *af); 2430 2431 switch (*af) { 2432 #ifdef INET 2433 case AF_INET: 2434 packet_len = ntohs(ip->ip_len); 2435 break; 2436 #endif 2437 #ifdef INET6 2438 case AF_INET6: { 2439 const struct ip6_hdr *ip6; 2440 2441 if (__predict_false(decrypted_len < sizeof(*ip6))) { 2442 WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len, 2443 sizeof(*ip6)); 2444 return false; 2445 } 2446 2447 ip6 = (const struct ip6_hdr *)packet; 2448 packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen); 2449 break; 2450 } 2451 #endif 2452 default: 2453 return false; 2454 } 2455 2456 if (packet_len > decrypted_len) { 2457 WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len, 2458 decrypted_len); 2459 return false; 2460 } 2461 2462 return true; 2463 } 2464 2465 static bool 2466 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2467 int af, char *packet) 2468 { 2469 struct sockaddr_storage ss; 2470 struct sockaddr *sa; 2471 struct psref psref; 2472 struct wg_peer *wgp; 2473 bool ok; 2474 2475 /* 2476 * II CRYPTOKEY ROUTING 2477 * "it will only accept it if its source IP resolves in the 2478 * table to the public key used in the secure session for 2479 * decrypting it." 2480 */ 2481 2482 if (af == AF_INET) { 2483 const struct ip *ip = (const struct ip *)packet; 2484 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2485 sockaddr_in_init(sin, &ip->ip_src, 0); 2486 sa = sintosa(sin); 2487 #ifdef INET6 2488 } else { 2489 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2490 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2491 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2492 sa = sin6tosa(sin6); 2493 #endif 2494 } 2495 2496 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2497 ok = (wgp == wgp_expected); 2498 if (wgp != NULL) 2499 wg_put_peer(wgp, &psref); 2500 2501 return ok; 2502 } 2503 2504 static void 2505 wg_session_dtor_timer(void *arg) 2506 { 2507 struct wg_peer *wgp = arg; 2508 2509 WG_TRACE("enter"); 2510 2511 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2512 } 2513 2514 static void 2515 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2516 { 2517 2518 /* 1 second grace period */ 2519 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2520 } 2521 2522 static bool 2523 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2524 { 2525 if (sa1->sa_family != sa2->sa_family) 2526 return false; 2527 2528 switch (sa1->sa_family) { 2529 #ifdef INET 2530 case AF_INET: 2531 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2532 #endif 2533 #ifdef INET6 2534 case AF_INET6: 2535 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2536 #endif 2537 default: 2538 return false; 2539 } 2540 } 2541 2542 static void 2543 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2544 const struct sockaddr *src) 2545 { 2546 struct wg_sockaddr *wgsa; 2547 struct psref psref; 2548 2549 wgsa = wg_get_endpoint_sa(wgp, &psref); 2550 2551 #ifdef WG_DEBUG_LOG 2552 char oldaddr[128], newaddr[128]; 2553 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2554 sockaddr_format(src, newaddr, sizeof(newaddr)); 2555 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2556 #endif 2557 2558 /* 2559 * III: "Since the packet has authenticated correctly, the source IP of 2560 * the outer UDP/IP packet is used to update the endpoint for peer..." 2561 */ 2562 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2563 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2564 /* XXX We can't change the endpoint twice in a short period */ 2565 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2566 wg_change_endpoint(wgp, src); 2567 } 2568 } 2569 2570 wg_put_sa(wgp, wgsa, &psref); 2571 } 2572 2573 static void __noinline 2574 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2575 const struct sockaddr *src) 2576 { 2577 struct wg_msg_data *wgmd; 2578 char *encrypted_buf = NULL, *decrypted_buf; 2579 size_t encrypted_len, decrypted_len; 2580 struct wg_session *wgs; 2581 struct wg_peer *wgp; 2582 int state; 2583 size_t mlen; 2584 struct psref psref; 2585 int error, af; 2586 bool success, free_encrypted_buf = false, ok; 2587 struct mbuf *n; 2588 2589 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2590 wgmd = mtod(m, struct wg_msg_data *); 2591 2592 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2593 WG_TRACE("data"); 2594 2595 /* Find the putative session, or drop. */ 2596 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2597 if (wgs == NULL) { 2598 WG_TRACE("No session found"); 2599 m_freem(m); 2600 return; 2601 } 2602 2603 /* 2604 * We are only ready to handle data when in INIT_PASSIVE, 2605 * ESTABLISHED, or DESTROYING. All transitions out of that 2606 * state dissociate the session index and drain psrefs. 2607 */ 2608 state = atomic_load_relaxed(&wgs->wgs_state); 2609 switch (state) { 2610 case WGS_STATE_UNKNOWN: 2611 panic("wg session %p in unknown state has session index %u", 2612 wgs, wgmd->wgmd_receiver); 2613 case WGS_STATE_INIT_ACTIVE: 2614 WG_TRACE("not yet ready for data"); 2615 goto out; 2616 case WGS_STATE_INIT_PASSIVE: 2617 case WGS_STATE_ESTABLISHED: 2618 case WGS_STATE_DESTROYING: 2619 break; 2620 } 2621 2622 /* 2623 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2624 * to update the endpoint if authentication succeeds. 2625 */ 2626 wgp = wgs->wgs_peer; 2627 2628 /* 2629 * Reject outrageously wrong sequence numbers before doing any 2630 * crypto work or taking any locks. 2631 */ 2632 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2633 le64toh(wgmd->wgmd_counter)); 2634 if (error) { 2635 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2636 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2637 if_name(&wg->wg_if), wgp->wgp_name, 2638 le64toh(wgmd->wgmd_counter)); 2639 goto out; 2640 } 2641 2642 /* Ensure the payload and authenticator are contiguous. */ 2643 mlen = m_length(m); 2644 encrypted_len = mlen - sizeof(*wgmd); 2645 if (encrypted_len < WG_AUTHTAG_LEN) { 2646 WG_DLOG("Short encrypted_len: %zu\n", encrypted_len); 2647 goto out; 2648 } 2649 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2650 if (success) { 2651 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2652 } else { 2653 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2654 if (encrypted_buf == NULL) { 2655 WG_DLOG("failed to allocate encrypted_buf\n"); 2656 goto out; 2657 } 2658 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2659 free_encrypted_buf = true; 2660 } 2661 /* m_ensure_contig may change m regardless of its result */ 2662 KASSERT(m->m_len >= sizeof(*wgmd)); 2663 wgmd = mtod(m, struct wg_msg_data *); 2664 2665 #ifdef WG_DEBUG_PACKET 2666 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2667 hexdump(printf, "incoming packet", encrypted_buf, 2668 encrypted_len); 2669 } 2670 #endif 2671 /* 2672 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2673 * a zero-length buffer (XXX). Drop if plaintext is longer 2674 * than MCLBYTES (XXX). 2675 */ 2676 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2677 if (decrypted_len > MCLBYTES) { 2678 /* FIXME handle larger data than MCLBYTES */ 2679 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2680 goto out; 2681 } 2682 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2683 if (n == NULL) { 2684 WG_DLOG("wg_get_mbuf failed\n"); 2685 goto out; 2686 } 2687 decrypted_buf = mtod(n, char *); 2688 2689 /* Decrypt and verify the packet. */ 2690 WG_DLOG("mlen=%zu, encrypted_len=%zu\n", mlen, encrypted_len); 2691 error = wg_algo_aead_dec(decrypted_buf, 2692 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2693 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2694 encrypted_len, NULL, 0); 2695 if (error != 0) { 2696 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2697 "%s: peer %s: failed to wg_algo_aead_dec\n", 2698 if_name(&wg->wg_if), wgp->wgp_name); 2699 m_freem(n); 2700 goto out; 2701 } 2702 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2703 2704 /* Packet is genuine. Reject it if a replay or just too old. */ 2705 mutex_enter(&wgs->wgs_recvwin->lock); 2706 error = sliwin_update(&wgs->wgs_recvwin->window, 2707 le64toh(wgmd->wgmd_counter)); 2708 mutex_exit(&wgs->wgs_recvwin->lock); 2709 if (error) { 2710 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2711 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2712 if_name(&wg->wg_if), wgp->wgp_name, 2713 le64toh(wgmd->wgmd_counter)); 2714 m_freem(n); 2715 goto out; 2716 } 2717 2718 #ifdef WG_DEBUG_PACKET 2719 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 2720 hexdump(printf, "tkey_recv", wgs->wgs_tkey_recv, 2721 sizeof(wgs->wgs_tkey_recv)); 2722 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 2723 hexdump(printf, "decrypted_buf", decrypted_buf, 2724 decrypted_len); 2725 } 2726 #endif 2727 /* We're done with m now; free it and chuck the pointers. */ 2728 m_freem(m); 2729 m = NULL; 2730 wgmd = NULL; 2731 2732 /* 2733 * Validate the encapsulated packet header and get the address 2734 * family, or drop. 2735 */ 2736 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2737 if (!ok) { 2738 m_freem(n); 2739 goto out; 2740 } 2741 2742 /* 2743 * The packet is genuine. Update the peer's endpoint if the 2744 * source address changed. 2745 * 2746 * XXX How to prevent DoS by replaying genuine packets from the 2747 * wrong source address? 2748 */ 2749 wg_update_endpoint_if_necessary(wgp, src); 2750 2751 /* Submit it into our network stack if routable. */ 2752 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2753 if (ok) { 2754 wg->wg_ops->input(&wg->wg_if, n, af); 2755 } else { 2756 char addrstr[INET6_ADDRSTRLEN]; 2757 memset(addrstr, 0, sizeof(addrstr)); 2758 if (af == AF_INET) { 2759 const struct ip *ip = (const struct ip *)decrypted_buf; 2760 IN_PRINT(addrstr, &ip->ip_src); 2761 #ifdef INET6 2762 } else if (af == AF_INET6) { 2763 const struct ip6_hdr *ip6 = 2764 (const struct ip6_hdr *)decrypted_buf; 2765 IN6_PRINT(addrstr, &ip6->ip6_src); 2766 #endif 2767 } 2768 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2769 "%s: peer %s: invalid source address (%s)\n", 2770 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2771 m_freem(n); 2772 /* 2773 * The inner address is invalid however the session is valid 2774 * so continue the session processing below. 2775 */ 2776 } 2777 n = NULL; 2778 2779 /* Update the state machine if necessary. */ 2780 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2781 /* 2782 * We were waiting for the initiator to send their 2783 * first data transport message, and that has happened. 2784 * Schedule a task to establish this session. 2785 */ 2786 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2787 } else { 2788 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2789 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2790 } 2791 /* 2792 * [W] 6.5 Passive Keepalive 2793 * "If a peer has received a validly-authenticated transport 2794 * data message (section 5.4.6), but does not have any packets 2795 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2796 * a keepalive message." 2797 */ 2798 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2799 (uintmax_t)time_uptime, 2800 (uintmax_t)wgs->wgs_time_last_data_sent); 2801 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2802 wg_keepalive_timeout) { 2803 WG_TRACE("Schedule sending keepalive message"); 2804 /* 2805 * We can't send a keepalive message here to avoid 2806 * a deadlock; we already hold the solock of a socket 2807 * that is used to send the message. 2808 */ 2809 wg_schedule_peer_task(wgp, 2810 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2811 } 2812 } 2813 out: 2814 wg_put_session(wgs, &psref); 2815 m_freem(m); 2816 if (free_encrypted_buf) 2817 kmem_intr_free(encrypted_buf, encrypted_len); 2818 } 2819 2820 static void __noinline 2821 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2822 { 2823 struct wg_session *wgs; 2824 struct wg_peer *wgp; 2825 struct psref psref; 2826 int error; 2827 uint8_t key[WG_HASH_LEN]; 2828 uint8_t cookie[WG_COOKIE_LEN]; 2829 2830 WG_TRACE("cookie msg received"); 2831 2832 /* Find the putative session. */ 2833 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2834 if (wgs == NULL) { 2835 WG_TRACE("No session found"); 2836 return; 2837 } 2838 2839 /* Lock the peer so we can update the cookie state. */ 2840 wgp = wgs->wgs_peer; 2841 mutex_enter(wgp->wgp_lock); 2842 2843 if (!wgp->wgp_last_sent_mac1_valid) { 2844 WG_TRACE("No valid mac1 sent (or expired)"); 2845 goto out; 2846 } 2847 2848 /* Decrypt the cookie and store it for later handshake retry. */ 2849 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2850 sizeof(wgp->wgp_pubkey)); 2851 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2852 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2853 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2854 wgmc->wgmc_salt); 2855 if (error != 0) { 2856 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2857 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 2858 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 2859 goto out; 2860 } 2861 /* 2862 * [W] 6.6: Interaction with Cookie Reply System 2863 * "it should simply store the decrypted cookie value from the cookie 2864 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2865 * timer for retrying a handshake initiation message." 2866 */ 2867 wgp->wgp_latest_cookie_time = time_uptime; 2868 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2869 out: 2870 mutex_exit(wgp->wgp_lock); 2871 wg_put_session(wgs, &psref); 2872 } 2873 2874 static struct mbuf * 2875 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2876 { 2877 struct wg_msg wgm; 2878 size_t mbuflen; 2879 size_t msglen; 2880 2881 /* 2882 * Get the mbuf chain length. It is already guaranteed, by 2883 * wg_overudp_cb, to be large enough for a struct wg_msg. 2884 */ 2885 mbuflen = m_length(m); 2886 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2887 2888 /* 2889 * Copy the message header (32-bit message type) out -- we'll 2890 * worry about contiguity and alignment later. 2891 */ 2892 m_copydata(m, 0, sizeof(wgm), &wgm); 2893 switch (le32toh(wgm.wgm_type)) { 2894 case WG_MSG_TYPE_INIT: 2895 msglen = sizeof(struct wg_msg_init); 2896 break; 2897 case WG_MSG_TYPE_RESP: 2898 msglen = sizeof(struct wg_msg_resp); 2899 break; 2900 case WG_MSG_TYPE_COOKIE: 2901 msglen = sizeof(struct wg_msg_cookie); 2902 break; 2903 case WG_MSG_TYPE_DATA: 2904 msglen = sizeof(struct wg_msg_data); 2905 break; 2906 default: 2907 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2908 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 2909 le32toh(wgm.wgm_type)); 2910 goto error; 2911 } 2912 2913 /* Verify the mbuf chain is long enough for this type of message. */ 2914 if (__predict_false(mbuflen < msglen)) { 2915 WG_DLOG("Invalid msg size: mbuflen=%zu type=%u\n", mbuflen, 2916 le32toh(wgm.wgm_type)); 2917 goto error; 2918 } 2919 2920 /* Make the message header contiguous if necessary. */ 2921 if (__predict_false(m->m_len < msglen)) { 2922 m = m_pullup(m, msglen); 2923 if (m == NULL) 2924 return NULL; 2925 } 2926 2927 return m; 2928 2929 error: 2930 m_freem(m); 2931 return NULL; 2932 } 2933 2934 static void 2935 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2936 const struct sockaddr *src) 2937 { 2938 struct wg_msg *wgm; 2939 2940 KASSERT(curlwp->l_pflag & LP_BOUND); 2941 2942 m = wg_validate_msg_header(wg, m); 2943 if (__predict_false(m == NULL)) 2944 return; 2945 2946 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2947 wgm = mtod(m, struct wg_msg *); 2948 switch (le32toh(wgm->wgm_type)) { 2949 case WG_MSG_TYPE_INIT: 2950 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2951 break; 2952 case WG_MSG_TYPE_RESP: 2953 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2954 break; 2955 case WG_MSG_TYPE_COOKIE: 2956 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2957 break; 2958 case WG_MSG_TYPE_DATA: 2959 wg_handle_msg_data(wg, m, src); 2960 /* wg_handle_msg_data frees m for us */ 2961 return; 2962 default: 2963 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2964 } 2965 2966 m_freem(m); 2967 } 2968 2969 static void 2970 wg_receive_packets(struct wg_softc *wg, const int af) 2971 { 2972 2973 for (;;) { 2974 int error, flags; 2975 struct socket *so; 2976 struct mbuf *m = NULL; 2977 struct uio dummy_uio; 2978 struct mbuf *paddr = NULL; 2979 struct sockaddr *src; 2980 2981 so = wg_get_so_by_af(wg, af); 2982 flags = MSG_DONTWAIT; 2983 dummy_uio.uio_resid = 1000000000; 2984 2985 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2986 &flags); 2987 if (error || m == NULL) { 2988 //if (error == EWOULDBLOCK) 2989 return; 2990 } 2991 2992 KASSERT(paddr != NULL); 2993 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2994 src = mtod(paddr, struct sockaddr *); 2995 2996 wg_handle_packet(wg, m, src); 2997 } 2998 } 2999 3000 static void 3001 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 3002 { 3003 3004 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 3005 } 3006 3007 static void 3008 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 3009 { 3010 3011 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 3012 } 3013 3014 static void 3015 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 3016 { 3017 struct wg_session *wgs; 3018 3019 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 3020 3021 KASSERT(mutex_owned(wgp->wgp_lock)); 3022 3023 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 3024 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 3025 if_name(&wg->wg_if)); 3026 /* XXX should do something? */ 3027 return; 3028 } 3029 3030 wgs = wgp->wgp_session_stable; 3031 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 3032 /* XXX What if the unstable session is already INIT_ACTIVE? */ 3033 wg_send_handshake_msg_init(wg, wgp); 3034 } else { 3035 /* rekey */ 3036 wgs = wgp->wgp_session_unstable; 3037 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3038 wg_send_handshake_msg_init(wg, wgp); 3039 } 3040 } 3041 3042 static void 3043 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 3044 { 3045 struct wg_session *wgs; 3046 3047 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 3048 3049 KASSERT(mutex_owned(wgp->wgp_lock)); 3050 KASSERT(wgp->wgp_handshake_start_time != 0); 3051 3052 wgs = wgp->wgp_session_unstable; 3053 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 3054 return; 3055 3056 /* 3057 * XXX no real need to assign a new index here, but we do need 3058 * to transition to UNKNOWN temporarily 3059 */ 3060 wg_put_session_index(wg, wgs); 3061 3062 /* [W] 6.4 Handshake Initiation Retransmission */ 3063 if ((time_uptime - wgp->wgp_handshake_start_time) > 3064 wg_rekey_attempt_time) { 3065 /* Give up handshaking */ 3066 wgp->wgp_handshake_start_time = 0; 3067 WG_TRACE("give up"); 3068 3069 /* 3070 * If a new data packet comes, handshaking will be retried 3071 * and a new session would be established at that time, 3072 * however we don't want to send pending packets then. 3073 */ 3074 wg_purge_pending_packets(wgp); 3075 return; 3076 } 3077 3078 wg_task_send_init_message(wg, wgp); 3079 } 3080 3081 static void 3082 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3083 { 3084 struct wg_session *wgs, *wgs_prev; 3085 struct mbuf *m; 3086 3087 KASSERT(mutex_owned(wgp->wgp_lock)); 3088 3089 wgs = wgp->wgp_session_unstable; 3090 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3091 /* XXX Can this happen? */ 3092 return; 3093 3094 wgs->wgs_state = WGS_STATE_ESTABLISHED; 3095 wgs->wgs_time_established = time_uptime; 3096 wgs->wgs_time_last_data_sent = 0; 3097 wgs->wgs_is_initiator = false; 3098 WG_TRACE("WGS_STATE_ESTABLISHED"); 3099 3100 wg_swap_sessions(wgp); 3101 KASSERT(wgs == wgp->wgp_session_stable); 3102 wgs_prev = wgp->wgp_session_unstable; 3103 getnanotime(&wgp->wgp_last_handshake_time); 3104 wgp->wgp_handshake_start_time = 0; 3105 wgp->wgp_last_sent_mac1_valid = false; 3106 wgp->wgp_last_sent_cookie_valid = false; 3107 3108 /* If we had a data packet queued up, send it. */ 3109 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3110 kpreempt_disable(); 3111 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3112 M_SETCTX(m, wgp); 3113 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3114 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3115 if_name(&wg->wg_if)); 3116 m_freem(m); 3117 } 3118 kpreempt_enable(); 3119 } 3120 3121 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3122 /* Wait for wg_get_stable_session to drain. */ 3123 pserialize_perform(wgp->wgp_psz); 3124 3125 /* Transition ESTABLISHED->DESTROYING. */ 3126 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 3127 3128 /* We can't destroy the old session immediately */ 3129 wg_schedule_session_dtor_timer(wgp); 3130 } else { 3131 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3132 "state=%d", wgs_prev->wgs_state); 3133 wg_clear_states(wgs_prev); 3134 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3135 } 3136 } 3137 3138 static void 3139 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3140 { 3141 3142 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3143 3144 KASSERT(mutex_owned(wgp->wgp_lock)); 3145 3146 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3147 pserialize_perform(wgp->wgp_psz); 3148 mutex_exit(wgp->wgp_lock); 3149 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3150 wg_psref_class); 3151 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3152 wg_psref_class); 3153 mutex_enter(wgp->wgp_lock); 3154 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3155 } 3156 } 3157 3158 static void 3159 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3160 { 3161 struct wg_session *wgs; 3162 3163 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3164 3165 KASSERT(mutex_owned(wgp->wgp_lock)); 3166 3167 wgs = wgp->wgp_session_stable; 3168 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3169 return; 3170 3171 wg_send_keepalive_msg(wgp, wgs); 3172 } 3173 3174 static void 3175 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3176 { 3177 struct wg_session *wgs; 3178 3179 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3180 3181 KASSERT(mutex_owned(wgp->wgp_lock)); 3182 3183 wgs = wgp->wgp_session_unstable; 3184 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 3185 wg_put_session_index(wg, wgs); 3186 } 3187 } 3188 3189 static void 3190 wg_peer_work(struct work *wk, void *cookie) 3191 { 3192 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3193 struct wg_softc *wg = wgp->wgp_sc; 3194 unsigned int tasks; 3195 3196 mutex_enter(wgp->wgp_intr_lock); 3197 while ((tasks = wgp->wgp_tasks) != 0) { 3198 wgp->wgp_tasks = 0; 3199 mutex_exit(wgp->wgp_intr_lock); 3200 3201 mutex_enter(wgp->wgp_lock); 3202 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3203 wg_task_send_init_message(wg, wgp); 3204 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3205 wg_task_retry_handshake(wg, wgp); 3206 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3207 wg_task_establish_session(wg, wgp); 3208 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3209 wg_task_endpoint_changed(wg, wgp); 3210 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3211 wg_task_send_keepalive_message(wg, wgp); 3212 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3213 wg_task_destroy_prev_session(wg, wgp); 3214 mutex_exit(wgp->wgp_lock); 3215 3216 mutex_enter(wgp->wgp_intr_lock); 3217 } 3218 mutex_exit(wgp->wgp_intr_lock); 3219 } 3220 3221 static void 3222 wg_job(struct threadpool_job *job) 3223 { 3224 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3225 int bound, upcalls; 3226 3227 mutex_enter(wg->wg_intr_lock); 3228 while ((upcalls = wg->wg_upcalls) != 0) { 3229 wg->wg_upcalls = 0; 3230 mutex_exit(wg->wg_intr_lock); 3231 bound = curlwp_bind(); 3232 if (ISSET(upcalls, WG_UPCALL_INET)) 3233 wg_receive_packets(wg, AF_INET); 3234 if (ISSET(upcalls, WG_UPCALL_INET6)) 3235 wg_receive_packets(wg, AF_INET6); 3236 curlwp_bindx(bound); 3237 mutex_enter(wg->wg_intr_lock); 3238 } 3239 threadpool_job_done(job); 3240 mutex_exit(wg->wg_intr_lock); 3241 } 3242 3243 static int 3244 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3245 { 3246 int error; 3247 uint16_t old_port = wg->wg_listen_port; 3248 3249 if (port != 0 && old_port == port) 3250 return 0; 3251 3252 struct sockaddr_in _sin, *sin = &_sin; 3253 sin->sin_len = sizeof(*sin); 3254 sin->sin_family = AF_INET; 3255 sin->sin_addr.s_addr = INADDR_ANY; 3256 sin->sin_port = htons(port); 3257 3258 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3259 if (error != 0) 3260 return error; 3261 3262 #ifdef INET6 3263 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3264 sin6->sin6_len = sizeof(*sin6); 3265 sin6->sin6_family = AF_INET6; 3266 sin6->sin6_addr = in6addr_any; 3267 sin6->sin6_port = htons(port); 3268 3269 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3270 if (error != 0) 3271 return error; 3272 #endif 3273 3274 wg->wg_listen_port = port; 3275 3276 return 0; 3277 } 3278 3279 static void 3280 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3281 { 3282 struct wg_softc *wg = cookie; 3283 int reason; 3284 3285 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3286 WG_UPCALL_INET : 3287 WG_UPCALL_INET6; 3288 3289 mutex_enter(wg->wg_intr_lock); 3290 wg->wg_upcalls |= reason; 3291 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3292 mutex_exit(wg->wg_intr_lock); 3293 } 3294 3295 static int 3296 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3297 struct sockaddr *src, void *arg) 3298 { 3299 struct wg_softc *wg = arg; 3300 struct wg_msg wgm; 3301 struct mbuf *m = *mp; 3302 3303 WG_TRACE("enter"); 3304 3305 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3306 KASSERT(offset <= m_length(m)); 3307 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3308 /* drop on the floor */ 3309 m_freem(m); 3310 return -1; 3311 } 3312 3313 /* 3314 * Copy the message header (32-bit message type) out -- we'll 3315 * worry about contiguity and alignment later. 3316 */ 3317 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3318 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3319 3320 /* 3321 * Handle DATA packets promptly as they arrive. Other packets 3322 * may require expensive public-key crypto and are not as 3323 * sensitive to latency, so defer them to the worker thread. 3324 */ 3325 switch (le32toh(wgm.wgm_type)) { 3326 case WG_MSG_TYPE_DATA: 3327 /* handle immediately */ 3328 m_adj(m, offset); 3329 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3330 m = m_pullup(m, sizeof(struct wg_msg_data)); 3331 if (m == NULL) 3332 return -1; 3333 } 3334 wg_handle_msg_data(wg, m, src); 3335 *mp = NULL; 3336 return 1; 3337 case WG_MSG_TYPE_INIT: 3338 case WG_MSG_TYPE_RESP: 3339 case WG_MSG_TYPE_COOKIE: 3340 /* pass through to so_receive in wg_receive_packets */ 3341 return 0; 3342 default: 3343 /* drop on the floor */ 3344 m_freem(m); 3345 return -1; 3346 } 3347 } 3348 3349 static int 3350 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3351 { 3352 int error; 3353 struct socket *so; 3354 3355 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3356 if (error != 0) 3357 return error; 3358 3359 solock(so); 3360 so->so_upcallarg = wg; 3361 so->so_upcall = wg_so_upcall; 3362 so->so_rcv.sb_flags |= SB_UPCALL; 3363 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3364 sounlock(so); 3365 3366 *sop = so; 3367 3368 return 0; 3369 } 3370 3371 static bool 3372 wg_session_hit_limits(struct wg_session *wgs) 3373 { 3374 3375 /* 3376 * [W] 6.2: Transport Message Limits 3377 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3378 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3379 * comes first, WireGuard will refuse to send any more transport data 3380 * messages using the current secure session, ..." 3381 */ 3382 KASSERT(wgs->wgs_time_established != 0); 3383 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3384 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3385 return true; 3386 } else if (wg_session_get_send_counter(wgs) > 3387 wg_reject_after_messages) { 3388 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3389 return true; 3390 } 3391 3392 return false; 3393 } 3394 3395 static void 3396 wgintr(void *cookie) 3397 { 3398 struct wg_peer *wgp; 3399 struct wg_session *wgs; 3400 struct mbuf *m; 3401 struct psref psref; 3402 3403 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3404 wgp = M_GETCTX(m, struct wg_peer *); 3405 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3406 WG_TRACE("no stable session"); 3407 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3408 goto next0; 3409 } 3410 if (__predict_false(wg_session_hit_limits(wgs))) { 3411 WG_TRACE("stable session hit limits"); 3412 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3413 goto next1; 3414 } 3415 wg_send_data_msg(wgp, wgs, m); 3416 m = NULL; /* consumed */ 3417 next1: wg_put_session(wgs, &psref); 3418 next0: m_freem(m); 3419 /* XXX Yield to avoid userland starvation? */ 3420 } 3421 } 3422 3423 static void 3424 wg_rekey_timer(void *arg) 3425 { 3426 struct wg_peer *wgp = arg; 3427 3428 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3429 } 3430 3431 static void 3432 wg_purge_pending_packets(struct wg_peer *wgp) 3433 { 3434 struct mbuf *m; 3435 3436 m = atomic_swap_ptr(&wgp->wgp_pending, NULL); 3437 m_freem(m); 3438 pktq_barrier(wg_pktq); 3439 } 3440 3441 static void 3442 wg_handshake_timeout_timer(void *arg) 3443 { 3444 struct wg_peer *wgp = arg; 3445 3446 WG_TRACE("enter"); 3447 3448 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3449 } 3450 3451 static struct wg_peer * 3452 wg_alloc_peer(struct wg_softc *wg) 3453 { 3454 struct wg_peer *wgp; 3455 3456 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3457 3458 wgp->wgp_sc = wg; 3459 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3460 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3461 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3462 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3463 wg_handshake_timeout_timer, wgp); 3464 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3465 callout_setfunc(&wgp->wgp_session_dtor_timer, 3466 wg_session_dtor_timer, wgp); 3467 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3468 wgp->wgp_endpoint_changing = false; 3469 wgp->wgp_endpoint_available = false; 3470 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3471 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3472 wgp->wgp_psz = pserialize_create(); 3473 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3474 3475 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3476 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3477 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3478 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3479 3480 struct wg_session *wgs; 3481 wgp->wgp_session_stable = 3482 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3483 wgp->wgp_session_unstable = 3484 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3485 wgs = wgp->wgp_session_stable; 3486 wgs->wgs_peer = wgp; 3487 wgs->wgs_state = WGS_STATE_UNKNOWN; 3488 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3489 #ifndef __HAVE_ATOMIC64_LOADSTORE 3490 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3491 #endif 3492 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3493 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3494 3495 wgs = wgp->wgp_session_unstable; 3496 wgs->wgs_peer = wgp; 3497 wgs->wgs_state = WGS_STATE_UNKNOWN; 3498 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3499 #ifndef __HAVE_ATOMIC64_LOADSTORE 3500 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3501 #endif 3502 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3503 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3504 3505 return wgp; 3506 } 3507 3508 static void 3509 wg_destroy_peer(struct wg_peer *wgp) 3510 { 3511 struct wg_session *wgs; 3512 struct wg_softc *wg = wgp->wgp_sc; 3513 3514 /* Prevent new packets from this peer on any source address. */ 3515 rw_enter(wg->wg_rwlock, RW_WRITER); 3516 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3517 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3518 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3519 struct radix_node *rn; 3520 3521 KASSERT(rnh != NULL); 3522 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3523 &wga->wga_sa_mask, rnh); 3524 if (rn == NULL) { 3525 char addrstr[128]; 3526 sockaddr_format(&wga->wga_sa_addr, addrstr, 3527 sizeof(addrstr)); 3528 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3529 if_name(&wg->wg_if), addrstr); 3530 } 3531 } 3532 rw_exit(wg->wg_rwlock); 3533 3534 /* Purge pending packets. */ 3535 wg_purge_pending_packets(wgp); 3536 3537 /* Halt all packet processing and timeouts. */ 3538 callout_halt(&wgp->wgp_rekey_timer, NULL); 3539 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3540 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3541 3542 /* Wait for any queued work to complete. */ 3543 workqueue_wait(wg_wq, &wgp->wgp_work); 3544 3545 wgs = wgp->wgp_session_unstable; 3546 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3547 mutex_enter(wgp->wgp_lock); 3548 wg_destroy_session(wg, wgs); 3549 mutex_exit(wgp->wgp_lock); 3550 } 3551 mutex_destroy(&wgs->wgs_recvwin->lock); 3552 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3553 #ifndef __HAVE_ATOMIC64_LOADSTORE 3554 mutex_destroy(&wgs->wgs_send_counter_lock); 3555 #endif 3556 kmem_free(wgs, sizeof(*wgs)); 3557 3558 wgs = wgp->wgp_session_stable; 3559 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3560 mutex_enter(wgp->wgp_lock); 3561 wg_destroy_session(wg, wgs); 3562 mutex_exit(wgp->wgp_lock); 3563 } 3564 mutex_destroy(&wgs->wgs_recvwin->lock); 3565 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3566 #ifndef __HAVE_ATOMIC64_LOADSTORE 3567 mutex_destroy(&wgs->wgs_send_counter_lock); 3568 #endif 3569 kmem_free(wgs, sizeof(*wgs)); 3570 3571 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3572 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3573 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3574 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3575 3576 pserialize_destroy(wgp->wgp_psz); 3577 mutex_obj_free(wgp->wgp_intr_lock); 3578 mutex_obj_free(wgp->wgp_lock); 3579 3580 kmem_free(wgp, sizeof(*wgp)); 3581 } 3582 3583 static void 3584 wg_destroy_all_peers(struct wg_softc *wg) 3585 { 3586 struct wg_peer *wgp, *wgp0 __diagused; 3587 void *garbage_byname, *garbage_bypubkey; 3588 3589 restart: 3590 garbage_byname = garbage_bypubkey = NULL; 3591 mutex_enter(wg->wg_lock); 3592 WG_PEER_WRITER_FOREACH(wgp, wg) { 3593 if (wgp->wgp_name[0]) { 3594 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3595 strlen(wgp->wgp_name)); 3596 KASSERT(wgp0 == wgp); 3597 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3598 } 3599 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3600 sizeof(wgp->wgp_pubkey)); 3601 KASSERT(wgp0 == wgp); 3602 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3603 WG_PEER_WRITER_REMOVE(wgp); 3604 wg->wg_npeers--; 3605 mutex_enter(wgp->wgp_lock); 3606 pserialize_perform(wgp->wgp_psz); 3607 mutex_exit(wgp->wgp_lock); 3608 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3609 break; 3610 } 3611 mutex_exit(wg->wg_lock); 3612 3613 if (wgp == NULL) 3614 return; 3615 3616 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3617 3618 wg_destroy_peer(wgp); 3619 thmap_gc(wg->wg_peers_byname, garbage_byname); 3620 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3621 3622 goto restart; 3623 } 3624 3625 static int 3626 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3627 { 3628 struct wg_peer *wgp, *wgp0 __diagused; 3629 void *garbage_byname, *garbage_bypubkey; 3630 3631 mutex_enter(wg->wg_lock); 3632 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3633 if (wgp != NULL) { 3634 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3635 sizeof(wgp->wgp_pubkey)); 3636 KASSERT(wgp0 == wgp); 3637 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3638 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3639 WG_PEER_WRITER_REMOVE(wgp); 3640 wg->wg_npeers--; 3641 if (wg->wg_npeers == 0) 3642 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3643 mutex_enter(wgp->wgp_lock); 3644 pserialize_perform(wgp->wgp_psz); 3645 mutex_exit(wgp->wgp_lock); 3646 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3647 } 3648 mutex_exit(wg->wg_lock); 3649 3650 if (wgp == NULL) 3651 return ENOENT; 3652 3653 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3654 3655 wg_destroy_peer(wgp); 3656 thmap_gc(wg->wg_peers_byname, garbage_byname); 3657 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3658 3659 return 0; 3660 } 3661 3662 static int 3663 wg_if_attach(struct wg_softc *wg) 3664 { 3665 3666 wg->wg_if.if_addrlen = 0; 3667 wg->wg_if.if_mtu = WG_MTU; 3668 wg->wg_if.if_flags = IFF_MULTICAST; 3669 wg->wg_if.if_extflags = IFEF_MPSAFE; 3670 wg->wg_if.if_ioctl = wg_ioctl; 3671 wg->wg_if.if_output = wg_output; 3672 wg->wg_if.if_init = wg_init; 3673 #ifdef ALTQ 3674 wg->wg_if.if_start = wg_start; 3675 #endif 3676 wg->wg_if.if_stop = wg_stop; 3677 wg->wg_if.if_type = IFT_OTHER; 3678 wg->wg_if.if_dlt = DLT_NULL; 3679 wg->wg_if.if_softc = wg; 3680 #ifdef ALTQ 3681 IFQ_SET_READY(&wg->wg_if.if_snd); 3682 #endif 3683 if_initialize(&wg->wg_if); 3684 3685 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3686 if_alloc_sadl(&wg->wg_if); 3687 if_register(&wg->wg_if); 3688 3689 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3690 3691 return 0; 3692 } 3693 3694 static void 3695 wg_if_detach(struct wg_softc *wg) 3696 { 3697 struct ifnet *ifp = &wg->wg_if; 3698 3699 bpf_detach(ifp); 3700 if_detach(ifp); 3701 } 3702 3703 static int 3704 wg_clone_create(struct if_clone *ifc, int unit) 3705 { 3706 struct wg_softc *wg; 3707 int error; 3708 3709 wg_guarantee_initialized(); 3710 3711 error = wg_count_inc(); 3712 if (error) 3713 return error; 3714 3715 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3716 3717 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3718 3719 PSLIST_INIT(&wg->wg_peers); 3720 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3721 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3722 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3723 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3724 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3725 wg->wg_rwlock = rw_obj_alloc(); 3726 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3727 "%s", if_name(&wg->wg_if)); 3728 wg->wg_ops = &wg_ops_rumpkernel; 3729 3730 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3731 if (error) 3732 goto fail0; 3733 3734 #ifdef INET 3735 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3736 if (error) 3737 goto fail1; 3738 rn_inithead((void **)&wg->wg_rtable_ipv4, 3739 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3740 #endif 3741 #ifdef INET6 3742 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3743 if (error) 3744 goto fail2; 3745 rn_inithead((void **)&wg->wg_rtable_ipv6, 3746 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3747 #endif 3748 3749 error = wg_if_attach(wg); 3750 if (error) 3751 goto fail3; 3752 3753 return 0; 3754 3755 fail4: __unused 3756 wg_if_detach(wg); 3757 fail3: wg_destroy_all_peers(wg); 3758 #ifdef INET6 3759 solock(wg->wg_so6); 3760 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3761 sounlock(wg->wg_so6); 3762 #endif 3763 #ifdef INET 3764 solock(wg->wg_so4); 3765 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3766 sounlock(wg->wg_so4); 3767 #endif 3768 mutex_enter(wg->wg_intr_lock); 3769 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3770 mutex_exit(wg->wg_intr_lock); 3771 #ifdef INET6 3772 if (wg->wg_rtable_ipv6 != NULL) 3773 free(wg->wg_rtable_ipv6, M_RTABLE); 3774 soclose(wg->wg_so6); 3775 fail2: 3776 #endif 3777 #ifdef INET 3778 if (wg->wg_rtable_ipv4 != NULL) 3779 free(wg->wg_rtable_ipv4, M_RTABLE); 3780 soclose(wg->wg_so4); 3781 fail1: 3782 #endif 3783 threadpool_put(wg->wg_threadpool, PRI_NONE); 3784 fail0: threadpool_job_destroy(&wg->wg_job); 3785 rw_obj_free(wg->wg_rwlock); 3786 mutex_obj_free(wg->wg_intr_lock); 3787 mutex_obj_free(wg->wg_lock); 3788 thmap_destroy(wg->wg_sessions_byindex); 3789 thmap_destroy(wg->wg_peers_byname); 3790 thmap_destroy(wg->wg_peers_bypubkey); 3791 PSLIST_DESTROY(&wg->wg_peers); 3792 kmem_free(wg, sizeof(*wg)); 3793 wg_count_dec(); 3794 return error; 3795 } 3796 3797 static int 3798 wg_clone_destroy(struct ifnet *ifp) 3799 { 3800 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3801 3802 #ifdef WG_RUMPKERNEL 3803 if (wg_user_mode(wg)) { 3804 rumpuser_wg_destroy(wg->wg_user); 3805 wg->wg_user = NULL; 3806 } 3807 #endif 3808 3809 wg_if_detach(wg); 3810 wg_destroy_all_peers(wg); 3811 #ifdef INET6 3812 solock(wg->wg_so6); 3813 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3814 sounlock(wg->wg_so6); 3815 #endif 3816 #ifdef INET 3817 solock(wg->wg_so4); 3818 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3819 sounlock(wg->wg_so4); 3820 #endif 3821 mutex_enter(wg->wg_intr_lock); 3822 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3823 mutex_exit(wg->wg_intr_lock); 3824 #ifdef INET6 3825 if (wg->wg_rtable_ipv6 != NULL) 3826 free(wg->wg_rtable_ipv6, M_RTABLE); 3827 soclose(wg->wg_so6); 3828 #endif 3829 #ifdef INET 3830 if (wg->wg_rtable_ipv4 != NULL) 3831 free(wg->wg_rtable_ipv4, M_RTABLE); 3832 soclose(wg->wg_so4); 3833 #endif 3834 threadpool_put(wg->wg_threadpool, PRI_NONE); 3835 threadpool_job_destroy(&wg->wg_job); 3836 rw_obj_free(wg->wg_rwlock); 3837 mutex_obj_free(wg->wg_intr_lock); 3838 mutex_obj_free(wg->wg_lock); 3839 thmap_destroy(wg->wg_sessions_byindex); 3840 thmap_destroy(wg->wg_peers_byname); 3841 thmap_destroy(wg->wg_peers_bypubkey); 3842 PSLIST_DESTROY(&wg->wg_peers); 3843 kmem_free(wg, sizeof(*wg)); 3844 wg_count_dec(); 3845 3846 return 0; 3847 } 3848 3849 static struct wg_peer * 3850 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3851 struct psref *psref) 3852 { 3853 struct radix_node_head *rnh; 3854 struct radix_node *rn; 3855 struct wg_peer *wgp = NULL; 3856 struct wg_allowedip *wga; 3857 3858 #ifdef WG_DEBUG_LOG 3859 char addrstr[128]; 3860 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3861 WG_DLOG("sa=%s\n", addrstr); 3862 #endif 3863 3864 rw_enter(wg->wg_rwlock, RW_READER); 3865 3866 rnh = wg_rnh(wg, sa->sa_family); 3867 if (rnh == NULL) 3868 goto out; 3869 3870 rn = rnh->rnh_matchaddr(sa, rnh); 3871 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3872 goto out; 3873 3874 WG_TRACE("success"); 3875 3876 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3877 wgp = wga->wga_peer; 3878 wg_get_peer(wgp, psref); 3879 3880 out: 3881 rw_exit(wg->wg_rwlock); 3882 return wgp; 3883 } 3884 3885 static void 3886 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3887 struct wg_session *wgs, struct wg_msg_data *wgmd) 3888 { 3889 3890 memset(wgmd, 0, sizeof(*wgmd)); 3891 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3892 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3893 /* [W] 5.4.6: msg.counter := Nm^send */ 3894 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3895 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3896 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3897 } 3898 3899 static int 3900 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3901 const struct rtentry *rt) 3902 { 3903 struct wg_softc *wg = ifp->if_softc; 3904 struct wg_peer *wgp = NULL; 3905 struct wg_session *wgs = NULL; 3906 struct psref wgp_psref, wgs_psref; 3907 int bound; 3908 int error; 3909 3910 bound = curlwp_bind(); 3911 3912 /* TODO make the nest limit configurable via sysctl */ 3913 error = if_tunnel_check_nesting(ifp, m, 1); 3914 if (error) { 3915 WGLOG(LOG_ERR, 3916 "%s: tunneling loop detected and packet dropped\n", 3917 if_name(&wg->wg_if)); 3918 goto out0; 3919 } 3920 3921 #ifdef ALTQ 3922 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 3923 & ALTQF_ENABLED; 3924 if (altq) 3925 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3926 #endif 3927 3928 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3929 3930 m->m_flags &= ~(M_BCAST|M_MCAST); 3931 3932 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3933 if (wgp == NULL) { 3934 WG_TRACE("peer not found"); 3935 error = EHOSTUNREACH; 3936 goto out0; 3937 } 3938 3939 /* Clear checksum-offload flags. */ 3940 m->m_pkthdr.csum_flags = 0; 3941 m->m_pkthdr.csum_data = 0; 3942 3943 /* Check whether there's an established session. */ 3944 wgs = wg_get_stable_session(wgp, &wgs_psref); 3945 if (wgs == NULL) { 3946 /* 3947 * No established session. If we're the first to try 3948 * sending data, schedule a handshake and queue the 3949 * packet for when the handshake is done; otherwise 3950 * just drop the packet and let the ongoing handshake 3951 * attempt continue. We could queue more data packets 3952 * but it's not clear that's worthwhile. 3953 */ 3954 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 3955 m = NULL; /* consume */ 3956 WG_TRACE("queued first packet; init handshake"); 3957 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3958 } else { 3959 WG_TRACE("first packet already queued, dropping"); 3960 } 3961 goto out1; 3962 } 3963 3964 /* There's an established session. Toss it in the queue. */ 3965 #ifdef ALTQ 3966 if (altq) { 3967 mutex_enter(ifp->if_snd.ifq_lock); 3968 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 3969 M_SETCTX(m, wgp); 3970 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 3971 m = NULL; /* consume */ 3972 } 3973 mutex_exit(ifp->if_snd.ifq_lock); 3974 if (m == NULL) { 3975 wg_start(ifp); 3976 goto out2; 3977 } 3978 } 3979 #endif 3980 kpreempt_disable(); 3981 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3982 M_SETCTX(m, wgp); 3983 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3984 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3985 if_name(&wg->wg_if)); 3986 error = ENOBUFS; 3987 goto out3; 3988 } 3989 m = NULL; /* consumed */ 3990 error = 0; 3991 out3: kpreempt_enable(); 3992 3993 #ifdef ALTQ 3994 out2: 3995 #endif 3996 wg_put_session(wgs, &wgs_psref); 3997 out1: wg_put_peer(wgp, &wgp_psref); 3998 out0: m_freem(m); 3999 curlwp_bindx(bound); 4000 return error; 4001 } 4002 4003 static int 4004 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 4005 { 4006 struct psref psref; 4007 struct wg_sockaddr *wgsa; 4008 int error; 4009 struct socket *so; 4010 4011 wgsa = wg_get_endpoint_sa(wgp, &psref); 4012 so = wg_get_so_by_peer(wgp, wgsa); 4013 solock(so); 4014 if (wgsatosa(wgsa)->sa_family == AF_INET) { 4015 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 4016 } else { 4017 #ifdef INET6 4018 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 4019 NULL, curlwp); 4020 #else 4021 m_freem(m); 4022 error = EPFNOSUPPORT; 4023 #endif 4024 } 4025 sounlock(so); 4026 wg_put_sa(wgp, wgsa, &psref); 4027 4028 return error; 4029 } 4030 4031 /* Inspired by pppoe_get_mbuf */ 4032 static struct mbuf * 4033 wg_get_mbuf(size_t leading_len, size_t len) 4034 { 4035 struct mbuf *m; 4036 4037 KASSERT(leading_len <= MCLBYTES); 4038 KASSERT(len <= MCLBYTES - leading_len); 4039 4040 m = m_gethdr(M_DONTWAIT, MT_DATA); 4041 if (m == NULL) 4042 return NULL; 4043 if (len + leading_len > MHLEN) { 4044 m_clget(m, M_DONTWAIT); 4045 if ((m->m_flags & M_EXT) == 0) { 4046 m_free(m); 4047 return NULL; 4048 } 4049 } 4050 m->m_data += leading_len; 4051 m->m_pkthdr.len = m->m_len = len; 4052 4053 return m; 4054 } 4055 4056 static int 4057 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 4058 struct mbuf *m) 4059 { 4060 struct wg_softc *wg = wgp->wgp_sc; 4061 int error; 4062 size_t inner_len, padded_len, encrypted_len; 4063 char *padded_buf = NULL; 4064 size_t mlen; 4065 struct wg_msg_data *wgmd; 4066 bool free_padded_buf = false; 4067 struct mbuf *n; 4068 size_t leading_len = max_hdr + sizeof(struct udphdr); 4069 4070 mlen = m_length(m); 4071 inner_len = mlen; 4072 padded_len = roundup(mlen, 16); 4073 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4074 WG_DLOG("inner=%zu, padded=%zu, encrypted_len=%zu\n", 4075 inner_len, padded_len, encrypted_len); 4076 if (mlen != 0) { 4077 bool success; 4078 success = m_ensure_contig(&m, padded_len); 4079 if (success) { 4080 padded_buf = mtod(m, char *); 4081 } else { 4082 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4083 if (padded_buf == NULL) { 4084 error = ENOBUFS; 4085 goto end; 4086 } 4087 free_padded_buf = true; 4088 m_copydata(m, 0, mlen, padded_buf); 4089 } 4090 memset(padded_buf + mlen, 0, padded_len - inner_len); 4091 } 4092 4093 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4094 if (n == NULL) { 4095 error = ENOBUFS; 4096 goto end; 4097 } 4098 KASSERT(n->m_len >= sizeof(*wgmd)); 4099 wgmd = mtod(n, struct wg_msg_data *); 4100 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4101 #ifdef WG_DEBUG_PACKET 4102 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4103 hexdump(printf, "padded_buf", padded_buf, 4104 padded_len); 4105 } 4106 #endif 4107 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4108 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4109 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4110 padded_buf, padded_len, 4111 NULL, 0); 4112 #ifdef WG_DEBUG_PACKET 4113 if (wg_debug & WG_DEBUG_FLAGS_PACKET) { 4114 hexdump(printf, "tkey_send", wgs->wgs_tkey_send, 4115 sizeof(wgs->wgs_tkey_send)); 4116 hexdump(printf, "wgmd", wgmd, sizeof(*wgmd)); 4117 hexdump(printf, "outgoing packet", 4118 (char *)wgmd + sizeof(*wgmd), encrypted_len); 4119 size_t decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 4120 char *decrypted_buf = kmem_intr_alloc((decrypted_len + 4121 WG_AUTHTAG_LEN/*XXX*/), KM_NOSLEEP); 4122 if (decrypted_buf != NULL) { 4123 error = wg_algo_aead_dec( 4124 1 + decrypted_buf /* force misalignment */, 4125 encrypted_len - WG_AUTHTAG_LEN /* XXX */, 4126 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4127 (char *)wgmd + sizeof(*wgmd), encrypted_len, 4128 NULL, 0); 4129 if (error) { 4130 WG_DLOG("wg_algo_aead_dec failed: %d\n", 4131 error); 4132 } 4133 if (!consttime_memequal(1 + decrypted_buf, 4134 (char *)wgmd + sizeof(*wgmd), 4135 decrypted_len)) { 4136 WG_DLOG("wg_algo_aead_dec returned garbage\n"); 4137 } 4138 kmem_intr_free(decrypted_buf, (decrypted_len + 4139 WG_AUTHTAG_LEN/*XXX*/)); 4140 } 4141 } 4142 #endif 4143 4144 error = wg->wg_ops->send_data_msg(wgp, n); 4145 if (error == 0) { 4146 struct ifnet *ifp = &wg->wg_if; 4147 if_statadd(ifp, if_obytes, mlen); 4148 if_statinc(ifp, if_opackets); 4149 if (wgs->wgs_is_initiator && 4150 wgs->wgs_time_last_data_sent == 0) { 4151 /* 4152 * [W] 6.2 Transport Message Limits 4153 * "if a peer is the initiator of a current secure 4154 * session, WireGuard will send a handshake initiation 4155 * message to begin a new secure session if, after 4156 * transmitting a transport data message, the current 4157 * secure session is REKEY-AFTER-TIME seconds old," 4158 */ 4159 wg_schedule_rekey_timer(wgp); 4160 } 4161 wgs->wgs_time_last_data_sent = time_uptime; 4162 if (wg_session_get_send_counter(wgs) >= 4163 wg_rekey_after_messages) { 4164 /* 4165 * [W] 6.2 Transport Message Limits 4166 * "WireGuard will try to create a new session, by 4167 * sending a handshake initiation message (section 4168 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4169 * transport data messages..." 4170 */ 4171 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4172 } 4173 } 4174 end: 4175 m_freem(m); 4176 if (free_padded_buf) 4177 kmem_intr_free(padded_buf, padded_len); 4178 return error; 4179 } 4180 4181 static void 4182 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4183 { 4184 pktqueue_t *pktq; 4185 size_t pktlen; 4186 4187 KASSERT(af == AF_INET || af == AF_INET6); 4188 4189 WG_TRACE(""); 4190 4191 m_set_rcvif(m, ifp); 4192 pktlen = m->m_pkthdr.len; 4193 4194 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4195 4196 switch (af) { 4197 case AF_INET: 4198 pktq = ip_pktq; 4199 break; 4200 #ifdef INET6 4201 case AF_INET6: 4202 pktq = ip6_pktq; 4203 break; 4204 #endif 4205 default: 4206 panic("invalid af=%d", af); 4207 } 4208 4209 kpreempt_disable(); 4210 const u_int h = curcpu()->ci_index; 4211 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4212 if_statadd(ifp, if_ibytes, pktlen); 4213 if_statinc(ifp, if_ipackets); 4214 } else { 4215 m_freem(m); 4216 } 4217 kpreempt_enable(); 4218 } 4219 4220 static void 4221 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4222 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4223 { 4224 4225 crypto_scalarmult_base(pubkey, privkey); 4226 } 4227 4228 static int 4229 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4230 { 4231 struct radix_node_head *rnh; 4232 struct radix_node *rn; 4233 int error = 0; 4234 4235 rw_enter(wg->wg_rwlock, RW_WRITER); 4236 rnh = wg_rnh(wg, wga->wga_family); 4237 KASSERT(rnh != NULL); 4238 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4239 wga->wga_nodes); 4240 rw_exit(wg->wg_rwlock); 4241 4242 if (rn == NULL) 4243 error = EEXIST; 4244 4245 return error; 4246 } 4247 4248 static int 4249 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4250 struct wg_peer **wgpp) 4251 { 4252 int error = 0; 4253 const void *pubkey; 4254 size_t pubkey_len; 4255 const void *psk; 4256 size_t psk_len; 4257 const char *name = NULL; 4258 4259 if (prop_dictionary_get_string(peer, "name", &name)) { 4260 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4261 error = EINVAL; 4262 goto out; 4263 } 4264 } 4265 4266 if (!prop_dictionary_get_data(peer, "public_key", 4267 &pubkey, &pubkey_len)) { 4268 error = EINVAL; 4269 goto out; 4270 } 4271 #ifdef WG_DEBUG_DUMP 4272 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4273 char *hex = gethexdump(pubkey, pubkey_len); 4274 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%zu\n%s\n", 4275 pubkey, pubkey_len, hex); 4276 puthexdump(hex, pubkey, pubkey_len); 4277 } 4278 #endif 4279 4280 struct wg_peer *wgp = wg_alloc_peer(wg); 4281 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4282 if (name != NULL) 4283 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4284 4285 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4286 if (psk_len != sizeof(wgp->wgp_psk)) { 4287 error = EINVAL; 4288 goto out; 4289 } 4290 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4291 } 4292 4293 const void *addr; 4294 size_t addr_len; 4295 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4296 4297 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4298 goto skip_endpoint; 4299 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4300 addr_len > sizeof(*wgsatoss(wgsa))) { 4301 error = EINVAL; 4302 goto out; 4303 } 4304 memcpy(wgsatoss(wgsa), addr, addr_len); 4305 switch (wgsa_family(wgsa)) { 4306 case AF_INET: 4307 #ifdef INET6 4308 case AF_INET6: 4309 #endif 4310 break; 4311 default: 4312 error = EPFNOSUPPORT; 4313 goto out; 4314 } 4315 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4316 error = EINVAL; 4317 goto out; 4318 } 4319 { 4320 char addrstr[128]; 4321 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4322 WG_DLOG("addr=%s\n", addrstr); 4323 } 4324 wgp->wgp_endpoint_available = true; 4325 4326 prop_array_t allowedips; 4327 skip_endpoint: 4328 allowedips = prop_dictionary_get(peer, "allowedips"); 4329 if (allowedips == NULL) 4330 goto skip; 4331 4332 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4333 prop_dictionary_t prop_allowedip; 4334 int j = 0; 4335 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4336 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4337 4338 if (!prop_dictionary_get_int(prop_allowedip, "family", 4339 &wga->wga_family)) 4340 continue; 4341 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4342 &addr, &addr_len)) 4343 continue; 4344 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4345 &wga->wga_cidr)) 4346 continue; 4347 4348 switch (wga->wga_family) { 4349 case AF_INET: { 4350 struct sockaddr_in sin; 4351 char addrstr[128]; 4352 struct in_addr mask; 4353 struct sockaddr_in sin_mask; 4354 4355 if (addr_len != sizeof(struct in_addr)) 4356 return EINVAL; 4357 memcpy(&wga->wga_addr4, addr, addr_len); 4358 4359 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4360 0); 4361 sockaddr_copy(&wga->wga_sa_addr, 4362 sizeof(sin), sintosa(&sin)); 4363 4364 sockaddr_format(sintosa(&sin), 4365 addrstr, sizeof(addrstr)); 4366 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4367 4368 in_len2mask(&mask, wga->wga_cidr); 4369 sockaddr_in_init(&sin_mask, &mask, 0); 4370 sockaddr_copy(&wga->wga_sa_mask, 4371 sizeof(sin_mask), sintosa(&sin_mask)); 4372 4373 break; 4374 } 4375 #ifdef INET6 4376 case AF_INET6: { 4377 struct sockaddr_in6 sin6; 4378 char addrstr[128]; 4379 struct in6_addr mask; 4380 struct sockaddr_in6 sin6_mask; 4381 4382 if (addr_len != sizeof(struct in6_addr)) 4383 return EINVAL; 4384 memcpy(&wga->wga_addr6, addr, addr_len); 4385 4386 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4387 0, 0, 0); 4388 sockaddr_copy(&wga->wga_sa_addr, 4389 sizeof(sin6), sin6tosa(&sin6)); 4390 4391 sockaddr_format(sin6tosa(&sin6), 4392 addrstr, sizeof(addrstr)); 4393 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4394 4395 in6_prefixlen2mask(&mask, wga->wga_cidr); 4396 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4397 sockaddr_copy(&wga->wga_sa_mask, 4398 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4399 4400 break; 4401 } 4402 #endif 4403 default: 4404 error = EINVAL; 4405 goto out; 4406 } 4407 wga->wga_peer = wgp; 4408 4409 error = wg_rtable_add_route(wg, wga); 4410 if (error != 0) 4411 goto out; 4412 4413 j++; 4414 } 4415 wgp->wgp_n_allowedips = j; 4416 skip: 4417 *wgpp = wgp; 4418 out: 4419 return error; 4420 } 4421 4422 static int 4423 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4424 { 4425 int error; 4426 char *buf; 4427 4428 WG_DLOG("buf=%p, len=%zu\n", ifd->ifd_data, ifd->ifd_len); 4429 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4430 return E2BIG; 4431 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4432 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4433 if (error != 0) 4434 return error; 4435 buf[ifd->ifd_len] = '\0'; 4436 #ifdef WG_DEBUG_DUMP 4437 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4438 log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len), 4439 (const char *)buf); 4440 } 4441 #endif 4442 *_buf = buf; 4443 return 0; 4444 } 4445 4446 static int 4447 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4448 { 4449 int error; 4450 prop_dictionary_t prop_dict; 4451 char *buf = NULL; 4452 const void *privkey; 4453 size_t privkey_len; 4454 4455 error = wg_alloc_prop_buf(&buf, ifd); 4456 if (error != 0) 4457 return error; 4458 error = EINVAL; 4459 prop_dict = prop_dictionary_internalize(buf); 4460 if (prop_dict == NULL) 4461 goto out; 4462 if (!prop_dictionary_get_data(prop_dict, "private_key", 4463 &privkey, &privkey_len)) 4464 goto out; 4465 #ifdef WG_DEBUG_DUMP 4466 if (wg_debug & WG_DEBUG_FLAGS_DUMP) { 4467 char *hex = gethexdump(privkey, privkey_len); 4468 log(LOG_DEBUG, "privkey=%p, privkey_len=%zu\n%s\n", 4469 privkey, privkey_len, hex); 4470 puthexdump(hex, privkey, privkey_len); 4471 } 4472 #endif 4473 if (privkey_len != WG_STATIC_KEY_LEN) 4474 goto out; 4475 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4476 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4477 error = 0; 4478 4479 out: 4480 kmem_free(buf, ifd->ifd_len + 1); 4481 return error; 4482 } 4483 4484 static int 4485 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4486 { 4487 int error; 4488 prop_dictionary_t prop_dict; 4489 char *buf = NULL; 4490 uint16_t port; 4491 4492 error = wg_alloc_prop_buf(&buf, ifd); 4493 if (error != 0) 4494 return error; 4495 error = EINVAL; 4496 prop_dict = prop_dictionary_internalize(buf); 4497 if (prop_dict == NULL) 4498 goto out; 4499 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4500 goto out; 4501 4502 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4503 4504 out: 4505 kmem_free(buf, ifd->ifd_len + 1); 4506 return error; 4507 } 4508 4509 static int 4510 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4511 { 4512 int error; 4513 prop_dictionary_t prop_dict; 4514 char *buf = NULL; 4515 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4516 4517 error = wg_alloc_prop_buf(&buf, ifd); 4518 if (error != 0) 4519 return error; 4520 error = EINVAL; 4521 prop_dict = prop_dictionary_internalize(buf); 4522 if (prop_dict == NULL) 4523 goto out; 4524 4525 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4526 if (error != 0) 4527 goto out; 4528 4529 mutex_enter(wg->wg_lock); 4530 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4531 sizeof(wgp->wgp_pubkey)) != NULL || 4532 (wgp->wgp_name[0] && 4533 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4534 strlen(wgp->wgp_name)) != NULL)) { 4535 mutex_exit(wg->wg_lock); 4536 wg_destroy_peer(wgp); 4537 error = EEXIST; 4538 goto out; 4539 } 4540 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4541 sizeof(wgp->wgp_pubkey), wgp); 4542 KASSERT(wgp0 == wgp); 4543 if (wgp->wgp_name[0]) { 4544 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4545 strlen(wgp->wgp_name), wgp); 4546 KASSERT(wgp0 == wgp); 4547 } 4548 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4549 wg->wg_npeers++; 4550 mutex_exit(wg->wg_lock); 4551 4552 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4553 4554 out: 4555 kmem_free(buf, ifd->ifd_len + 1); 4556 return error; 4557 } 4558 4559 static int 4560 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4561 { 4562 int error; 4563 prop_dictionary_t prop_dict; 4564 char *buf = NULL; 4565 const char *name; 4566 4567 error = wg_alloc_prop_buf(&buf, ifd); 4568 if (error != 0) 4569 return error; 4570 error = EINVAL; 4571 prop_dict = prop_dictionary_internalize(buf); 4572 if (prop_dict == NULL) 4573 goto out; 4574 4575 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4576 goto out; 4577 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4578 goto out; 4579 4580 error = wg_destroy_peer_name(wg, name); 4581 out: 4582 kmem_free(buf, ifd->ifd_len + 1); 4583 return error; 4584 } 4585 4586 static bool 4587 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4588 { 4589 int au = cmd == SIOCGDRVSPEC ? 4590 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4591 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4592 return kauth_authorize_network(kauth_cred_get(), 4593 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4594 (void *)cmd, NULL) == 0; 4595 } 4596 4597 static int 4598 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4599 { 4600 int error = ENOMEM; 4601 prop_dictionary_t prop_dict; 4602 prop_array_t peers = NULL; 4603 char *buf; 4604 struct wg_peer *wgp; 4605 int s, i; 4606 4607 prop_dict = prop_dictionary_create(); 4608 if (prop_dict == NULL) 4609 goto error; 4610 4611 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4612 if (!prop_dictionary_set_data(prop_dict, "private_key", 4613 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4614 goto error; 4615 } 4616 4617 if (wg->wg_listen_port != 0) { 4618 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4619 wg->wg_listen_port)) 4620 goto error; 4621 } 4622 4623 if (wg->wg_npeers == 0) 4624 goto skip_peers; 4625 4626 peers = prop_array_create(); 4627 if (peers == NULL) 4628 goto error; 4629 4630 s = pserialize_read_enter(); 4631 i = 0; 4632 WG_PEER_READER_FOREACH(wgp, wg) { 4633 struct wg_sockaddr *wgsa; 4634 struct psref wgp_psref, wgsa_psref; 4635 prop_dictionary_t prop_peer; 4636 4637 wg_get_peer(wgp, &wgp_psref); 4638 pserialize_read_exit(s); 4639 4640 prop_peer = prop_dictionary_create(); 4641 if (prop_peer == NULL) 4642 goto next; 4643 4644 if (strlen(wgp->wgp_name) > 0) { 4645 if (!prop_dictionary_set_string(prop_peer, "name", 4646 wgp->wgp_name)) 4647 goto next; 4648 } 4649 4650 if (!prop_dictionary_set_data(prop_peer, "public_key", 4651 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4652 goto next; 4653 4654 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4655 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4656 sizeof(wgp->wgp_psk))) { 4657 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4658 if (!prop_dictionary_set_data(prop_peer, 4659 "preshared_key", 4660 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4661 goto next; 4662 } 4663 } 4664 4665 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4666 CTASSERT(AF_UNSPEC == 0); 4667 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4668 !prop_dictionary_set_data(prop_peer, "endpoint", 4669 wgsatoss(wgsa), 4670 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4671 wg_put_sa(wgp, wgsa, &wgsa_psref); 4672 goto next; 4673 } 4674 wg_put_sa(wgp, wgsa, &wgsa_psref); 4675 4676 const struct timespec *t = &wgp->wgp_last_handshake_time; 4677 4678 if (!prop_dictionary_set_uint64(prop_peer, 4679 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4680 goto next; 4681 if (!prop_dictionary_set_uint32(prop_peer, 4682 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4683 goto next; 4684 4685 if (wgp->wgp_n_allowedips == 0) 4686 goto skip_allowedips; 4687 4688 prop_array_t allowedips = prop_array_create(); 4689 if (allowedips == NULL) 4690 goto next; 4691 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4692 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4693 prop_dictionary_t prop_allowedip; 4694 4695 prop_allowedip = prop_dictionary_create(); 4696 if (prop_allowedip == NULL) 4697 break; 4698 4699 if (!prop_dictionary_set_int(prop_allowedip, "family", 4700 wga->wga_family)) 4701 goto _next; 4702 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4703 wga->wga_cidr)) 4704 goto _next; 4705 4706 switch (wga->wga_family) { 4707 case AF_INET: 4708 if (!prop_dictionary_set_data(prop_allowedip, 4709 "ip", &wga->wga_addr4, 4710 sizeof(wga->wga_addr4))) 4711 goto _next; 4712 break; 4713 #ifdef INET6 4714 case AF_INET6: 4715 if (!prop_dictionary_set_data(prop_allowedip, 4716 "ip", &wga->wga_addr6, 4717 sizeof(wga->wga_addr6))) 4718 goto _next; 4719 break; 4720 #endif 4721 default: 4722 break; 4723 } 4724 prop_array_set(allowedips, j, prop_allowedip); 4725 _next: 4726 prop_object_release(prop_allowedip); 4727 } 4728 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4729 prop_object_release(allowedips); 4730 4731 skip_allowedips: 4732 4733 prop_array_set(peers, i, prop_peer); 4734 next: 4735 if (prop_peer) 4736 prop_object_release(prop_peer); 4737 i++; 4738 4739 s = pserialize_read_enter(); 4740 wg_put_peer(wgp, &wgp_psref); 4741 } 4742 pserialize_read_exit(s); 4743 4744 prop_dictionary_set(prop_dict, "peers", peers); 4745 prop_object_release(peers); 4746 peers = NULL; 4747 4748 skip_peers: 4749 buf = prop_dictionary_externalize(prop_dict); 4750 if (buf == NULL) 4751 goto error; 4752 if (ifd->ifd_len < (strlen(buf) + 1)) { 4753 error = EINVAL; 4754 goto error; 4755 } 4756 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4757 4758 free(buf, 0); 4759 error: 4760 if (peers != NULL) 4761 prop_object_release(peers); 4762 if (prop_dict != NULL) 4763 prop_object_release(prop_dict); 4764 4765 return error; 4766 } 4767 4768 static int 4769 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4770 { 4771 struct wg_softc *wg = ifp->if_softc; 4772 struct ifreq *ifr = data; 4773 struct ifaddr *ifa = data; 4774 struct ifdrv *ifd = data; 4775 int error = 0; 4776 4777 switch (cmd) { 4778 case SIOCINITIFADDR: 4779 if (ifa->ifa_addr->sa_family != AF_LINK && 4780 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4781 (IFF_UP | IFF_RUNNING)) { 4782 ifp->if_flags |= IFF_UP; 4783 error = if_init(ifp); 4784 } 4785 return error; 4786 case SIOCADDMULTI: 4787 case SIOCDELMULTI: 4788 switch (ifr->ifr_addr.sa_family) { 4789 case AF_INET: /* IP supports Multicast */ 4790 break; 4791 #ifdef INET6 4792 case AF_INET6: /* IP6 supports Multicast */ 4793 break; 4794 #endif 4795 default: /* Other protocols doesn't support Multicast */ 4796 error = EAFNOSUPPORT; 4797 break; 4798 } 4799 return error; 4800 case SIOCSDRVSPEC: 4801 if (!wg_is_authorized(wg, cmd)) { 4802 return EPERM; 4803 } 4804 switch (ifd->ifd_cmd) { 4805 case WG_IOCTL_SET_PRIVATE_KEY: 4806 error = wg_ioctl_set_private_key(wg, ifd); 4807 break; 4808 case WG_IOCTL_SET_LISTEN_PORT: 4809 error = wg_ioctl_set_listen_port(wg, ifd); 4810 break; 4811 case WG_IOCTL_ADD_PEER: 4812 error = wg_ioctl_add_peer(wg, ifd); 4813 break; 4814 case WG_IOCTL_DELETE_PEER: 4815 error = wg_ioctl_delete_peer(wg, ifd); 4816 break; 4817 default: 4818 error = EINVAL; 4819 break; 4820 } 4821 return error; 4822 case SIOCGDRVSPEC: 4823 return wg_ioctl_get(wg, ifd); 4824 case SIOCSIFFLAGS: 4825 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4826 break; 4827 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4828 case IFF_RUNNING: 4829 /* 4830 * If interface is marked down and it is running, 4831 * then stop and disable it. 4832 */ 4833 if_stop(ifp, 1); 4834 break; 4835 case IFF_UP: 4836 /* 4837 * If interface is marked up and it is stopped, then 4838 * start it. 4839 */ 4840 error = if_init(ifp); 4841 break; 4842 default: 4843 break; 4844 } 4845 return error; 4846 #ifdef WG_RUMPKERNEL 4847 case SIOCSLINKSTR: 4848 error = wg_ioctl_linkstr(wg, ifd); 4849 if (error == 0) 4850 wg->wg_ops = &wg_ops_rumpuser; 4851 return error; 4852 #endif 4853 default: 4854 break; 4855 } 4856 4857 error = ifioctl_common(ifp, cmd, data); 4858 4859 #ifdef WG_RUMPKERNEL 4860 if (!wg_user_mode(wg)) 4861 return error; 4862 4863 /* Do the same to the corresponding tun device on the host */ 4864 /* 4865 * XXX Actually the command has not been handled yet. It 4866 * will be handled via pr_ioctl form doifioctl later. 4867 */ 4868 switch (cmd) { 4869 case SIOCAIFADDR: 4870 case SIOCDIFADDR: { 4871 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4872 struct in_aliasreq *ifra = &_ifra; 4873 KASSERT(error == ENOTTY); 4874 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4875 IFNAMSIZ); 4876 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4877 if (error == 0) 4878 error = ENOTTY; 4879 break; 4880 } 4881 #ifdef INET6 4882 case SIOCAIFADDR_IN6: 4883 case SIOCDIFADDR_IN6: { 4884 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4885 struct in6_aliasreq *ifra = &_ifra; 4886 KASSERT(error == ENOTTY); 4887 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4888 IFNAMSIZ); 4889 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4890 if (error == 0) 4891 error = ENOTTY; 4892 break; 4893 } 4894 #endif 4895 } 4896 #endif /* WG_RUMPKERNEL */ 4897 4898 return error; 4899 } 4900 4901 static int 4902 wg_init(struct ifnet *ifp) 4903 { 4904 4905 ifp->if_flags |= IFF_RUNNING; 4906 4907 /* TODO flush pending packets. */ 4908 return 0; 4909 } 4910 4911 #ifdef ALTQ 4912 static void 4913 wg_start(struct ifnet *ifp) 4914 { 4915 struct mbuf *m; 4916 4917 for (;;) { 4918 IFQ_DEQUEUE(&ifp->if_snd, m); 4919 if (m == NULL) 4920 break; 4921 4922 kpreempt_disable(); 4923 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4924 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4925 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4926 if_name(ifp)); 4927 m_freem(m); 4928 } 4929 kpreempt_enable(); 4930 } 4931 } 4932 #endif 4933 4934 static void 4935 wg_stop(struct ifnet *ifp, int disable) 4936 { 4937 4938 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4939 ifp->if_flags &= ~IFF_RUNNING; 4940 4941 /* Need to do something? */ 4942 } 4943 4944 #ifdef WG_DEBUG_PARAMS 4945 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4946 { 4947 const struct sysctlnode *node = NULL; 4948 4949 sysctl_createv(clog, 0, NULL, &node, 4950 CTLFLAG_PERMANENT, 4951 CTLTYPE_NODE, "wg", 4952 SYSCTL_DESCR("wg(4)"), 4953 NULL, 0, NULL, 0, 4954 CTL_NET, CTL_CREATE, CTL_EOL); 4955 sysctl_createv(clog, 0, &node, NULL, 4956 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4957 CTLTYPE_QUAD, "rekey_after_messages", 4958 SYSCTL_DESCR("session liftime by messages"), 4959 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4960 sysctl_createv(clog, 0, &node, NULL, 4961 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4962 CTLTYPE_INT, "rekey_after_time", 4963 SYSCTL_DESCR("session liftime"), 4964 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4965 sysctl_createv(clog, 0, &node, NULL, 4966 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4967 CTLTYPE_INT, "rekey_timeout", 4968 SYSCTL_DESCR("session handshake retry time"), 4969 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4970 sysctl_createv(clog, 0, &node, NULL, 4971 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4972 CTLTYPE_INT, "rekey_attempt_time", 4973 SYSCTL_DESCR("session handshake timeout"), 4974 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4975 sysctl_createv(clog, 0, &node, NULL, 4976 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4977 CTLTYPE_INT, "keepalive_timeout", 4978 SYSCTL_DESCR("keepalive timeout"), 4979 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4980 sysctl_createv(clog, 0, &node, NULL, 4981 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4982 CTLTYPE_BOOL, "force_underload", 4983 SYSCTL_DESCR("force to detemine under load"), 4984 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4985 sysctl_createv(clog, 0, &node, NULL, 4986 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4987 CTLTYPE_INT, "debug", 4988 SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"), 4989 NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL); 4990 } 4991 #endif 4992 4993 #ifdef WG_RUMPKERNEL 4994 static bool 4995 wg_user_mode(struct wg_softc *wg) 4996 { 4997 4998 return wg->wg_user != NULL; 4999 } 5000 5001 static int 5002 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 5003 { 5004 struct ifnet *ifp = &wg->wg_if; 5005 int error; 5006 5007 if (ifp->if_flags & IFF_UP) 5008 return EBUSY; 5009 5010 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 5011 /* XXX do nothing */ 5012 return 0; 5013 } else if (ifd->ifd_cmd != 0) { 5014 return EINVAL; 5015 } else if (wg->wg_user != NULL) { 5016 return EBUSY; 5017 } 5018 5019 /* Assume \0 included */ 5020 if (ifd->ifd_len > IFNAMSIZ) { 5021 return E2BIG; 5022 } else if (ifd->ifd_len < 1) { 5023 return EINVAL; 5024 } 5025 5026 char tun_name[IFNAMSIZ]; 5027 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 5028 if (error != 0) 5029 return error; 5030 5031 if (strncmp(tun_name, "tun", 3) != 0) 5032 return EINVAL; 5033 5034 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 5035 5036 return error; 5037 } 5038 5039 static int 5040 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 5041 { 5042 int error; 5043 struct psref psref; 5044 struct wg_sockaddr *wgsa; 5045 struct wg_softc *wg = wgp->wgp_sc; 5046 struct iovec iov[1]; 5047 5048 wgsa = wg_get_endpoint_sa(wgp, &psref); 5049 5050 iov[0].iov_base = mtod(m, void *); 5051 iov[0].iov_len = m->m_len; 5052 5053 /* Send messages to a peer via an ordinary socket. */ 5054 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 5055 5056 wg_put_sa(wgp, wgsa, &psref); 5057 5058 m_freem(m); 5059 5060 return error; 5061 } 5062 5063 static void 5064 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 5065 { 5066 struct wg_softc *wg = ifp->if_softc; 5067 struct iovec iov[2]; 5068 struct sockaddr_storage ss; 5069 5070 KASSERT(af == AF_INET || af == AF_INET6); 5071 5072 WG_TRACE(""); 5073 5074 if (af == AF_INET) { 5075 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 5076 struct ip *ip; 5077 5078 KASSERT(m->m_len >= sizeof(struct ip)); 5079 ip = mtod(m, struct ip *); 5080 sockaddr_in_init(sin, &ip->ip_dst, 0); 5081 } else { 5082 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 5083 struct ip6_hdr *ip6; 5084 5085 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 5086 ip6 = mtod(m, struct ip6_hdr *); 5087 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 5088 } 5089 5090 iov[0].iov_base = &ss; 5091 iov[0].iov_len = ss.ss_len; 5092 iov[1].iov_base = mtod(m, void *); 5093 iov[1].iov_len = m->m_len; 5094 5095 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5096 5097 /* Send decrypted packets to users via a tun. */ 5098 rumpuser_wg_send_user(wg->wg_user, iov, 2); 5099 5100 m_freem(m); 5101 } 5102 5103 static int 5104 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 5105 { 5106 int error; 5107 uint16_t old_port = wg->wg_listen_port; 5108 5109 if (port != 0 && old_port == port) 5110 return 0; 5111 5112 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5113 if (error == 0) 5114 wg->wg_listen_port = port; 5115 return error; 5116 } 5117 5118 /* 5119 * Receive user packets. 5120 */ 5121 void 5122 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5123 { 5124 struct ifnet *ifp = &wg->wg_if; 5125 struct mbuf *m; 5126 const struct sockaddr *dst; 5127 5128 WG_TRACE(""); 5129 5130 dst = iov[0].iov_base; 5131 5132 m = m_gethdr(M_DONTWAIT, MT_DATA); 5133 if (m == NULL) 5134 return; 5135 m->m_len = m->m_pkthdr.len = 0; 5136 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5137 5138 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5139 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5140 5141 (void)wg_output(ifp, m, dst, NULL); 5142 } 5143 5144 /* 5145 * Receive packets from a peer. 5146 */ 5147 void 5148 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5149 { 5150 struct mbuf *m; 5151 const struct sockaddr *src; 5152 int bound; 5153 5154 WG_TRACE(""); 5155 5156 src = iov[0].iov_base; 5157 5158 m = m_gethdr(M_DONTWAIT, MT_DATA); 5159 if (m == NULL) 5160 return; 5161 m->m_len = m->m_pkthdr.len = 0; 5162 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5163 5164 WG_DLOG("iov_len=%zu\n", iov[1].iov_len); 5165 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5166 5167 bound = curlwp_bind(); 5168 wg_handle_packet(wg, m, src); 5169 curlwp_bindx(bound); 5170 } 5171 #endif /* WG_RUMPKERNEL */ 5172 5173 /* 5174 * Module infrastructure 5175 */ 5176 #include "if_module.h" 5177 5178 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5179
Indexes created Tue Sep 23 07:09:52 GMT 2025