if_wg.c revision 1.71.2.3
1 /* $NetBSD: if_wg.c,v 1.71.2.3 2024/03/11 19:34:00 martin Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.71.2.3 2024/03/11 19:34:00 martin Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_altq_enabled.h" 48 #include "opt_inet.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/types.h> 53 54 #include <sys/atomic.h> 55 #include <sys/callout.h> 56 #include <sys/cprng.h> 57 #include <sys/cpu.h> 58 #include <sys/device.h> 59 #include <sys/domain.h> 60 #include <sys/errno.h> 61 #include <sys/intr.h> 62 #include <sys/ioctl.h> 63 #include <sys/kernel.h> 64 #include <sys/kmem.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/once.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/threadpool.h> 82 #include <sys/time.h> 83 #include <sys/timespec.h> 84 #include <sys/workqueue.h> 85 86 #include <net/bpf.h> 87 #include <net/if.h> 88 #include <net/if_types.h> 89 #include <net/if_wg.h> 90 #include <net/pktqueue.h> 91 #include <net/route.h> 92 93 #include <netinet/in.h> 94 #include <netinet/in_pcb.h> 95 #include <netinet/in_var.h> 96 #include <netinet/ip.h> 97 #include <netinet/ip_var.h> 98 #include <netinet/udp.h> 99 #include <netinet/udp_var.h> 100 101 #ifdef INET6 102 #include <netinet/ip6.h> 103 #include <netinet6/in6_pcb.h> 104 #include <netinet6/in6_var.h> 105 #include <netinet6/ip6_var.h> 106 #include <netinet6/udp6_var.h> 107 #endif /* INET6 */ 108 109 #include <prop/proplib.h> 110 111 #include <crypto/blake2/blake2s.h> 112 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 113 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 114 #include <crypto/sodium/crypto_scalarmult.h> 115 116 #include "ioconf.h" 117 118 #ifdef WG_RUMPKERNEL 119 #include "wg_user.h" 120 #endif 121 122 /* 123 * Data structures 124 * - struct wg_softc is an instance of wg interfaces 125 * - It has a list of peers (struct wg_peer) 126 * - It has a threadpool job that sends/receives handshake messages and 127 * runs event handlers 128 * - It has its own two routing tables: one is for IPv4 and the other IPv6 129 * - struct wg_peer is a representative of a peer 130 * - It has a struct work to handle handshakes and timer tasks 131 * - It has a pair of session instances (struct wg_session) 132 * - It has a pair of endpoint instances (struct wg_sockaddr) 133 * - Normally one endpoint is used and the second one is used only on 134 * a peer migration (a change of peer's IP address) 135 * - It has a list of IP addresses and sub networks called allowedips 136 * (struct wg_allowedip) 137 * - A packets sent over a session is allowed if its destination matches 138 * any IP addresses or sub networks of the list 139 * - struct wg_session represents a session of a secure tunnel with a peer 140 * - Two instances of sessions belong to a peer; a stable session and a 141 * unstable session 142 * - A handshake process of a session always starts with a unstable instance 143 * - Once a session is established, its instance becomes stable and the 144 * other becomes unstable instead 145 * - Data messages are always sent via a stable session 146 * 147 * Locking notes: 148 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 149 * - Changes to the peer list are serialized by wg_lock 150 * - The peer list may be read with pserialize(9) and psref(9) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 152 * => XXX replace by pserialize when routing table is psz-safe 153 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 154 * only in thread context and serializes: 155 * - the stable and unstable session pointers 156 * - all unstable session state 157 * - Packet processing may be done in softint context: 158 * - The stable session can be read under pserialize(9) or psref(9) 159 * - The stable session is always ESTABLISHED 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 * - Lock order: wg_lock -> wgp_lock 164 */ 165 166 167 #define WGLOG(level, fmt, args...) \ 168 log(level, "%s: " fmt, __func__, ##args) 169 170 /* Debug options */ 171 #ifdef WG_DEBUG 172 /* Output debug logs */ 173 #ifndef WG_DEBUG_LOG 174 #define WG_DEBUG_LOG 175 #endif 176 /* Output trace logs */ 177 #ifndef WG_DEBUG_TRACE 178 #define WG_DEBUG_TRACE 179 #endif 180 /* Output hash values, etc. */ 181 #ifndef WG_DEBUG_DUMP 182 #define WG_DEBUG_DUMP 183 #endif 184 /* Make some internal parameters configurable for testing and debugging */ 185 #ifndef WG_DEBUG_PARAMS 186 #define WG_DEBUG_PARAMS 187 #endif 188 #endif 189 190 #ifdef WG_DEBUG_TRACE 191 #define WG_TRACE(msg) \ 192 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)) 193 #else 194 #define WG_TRACE(msg) __nothing 195 #endif 196 197 #ifdef WG_DEBUG_LOG 198 #define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args) 199 #else 200 #define WG_DLOG(fmt, args...) __nothing 201 #endif 202 203 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 204 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 205 &(wgprc)->wgprc_curpps, 1)) { \ 206 log(level, fmt, ##args); \ 207 } \ 208 } while (0) 209 210 #ifdef WG_DEBUG_PARAMS 211 static bool wg_force_underload = false; 212 #endif 213 214 #ifdef WG_DEBUG_DUMP 215 216 static char * 217 gethexdump(const char *p, size_t n) 218 { 219 char *buf; 220 size_t i; 221 222 if (n > SIZE_MAX/3 - 1) 223 return NULL; 224 buf = kmem_alloc(3*n + 1, KM_NOSLEEP); 225 if (buf == NULL) 226 return NULL; 227 for (i = 0; i < n; i++) 228 snprintf(buf + 3*i, 3 + 1, " %02hhx", p[i]); 229 return buf; 230 } 231 232 static void 233 puthexdump(char *buf, const void *p, size_t n) 234 { 235 236 if (buf == NULL) 237 return; 238 kmem_free(buf, 3*n + 1); 239 } 240 241 #ifdef WG_RUMPKERNEL 242 static void 243 wg_dump_buf(const char *func, const char *buf, const size_t size) 244 { 245 char *hex = gethexdump(buf, size); 246 247 log(LOG_DEBUG, "%s: %s\n", func, hex ? hex : "(enomem)"); 248 puthexdump(hex, buf, size); 249 } 250 #endif 251 252 static void 253 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 254 const size_t size) 255 { 256 char *hex = gethexdump(hash, size); 257 258 log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex ? hex : "(enomem)"); 259 puthexdump(hex, hash, size); 260 } 261 262 #define WG_DUMP_HASH(name, hash) \ 263 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 264 #define WG_DUMP_HASH48(name, hash) \ 265 wg_dump_hash(__func__, name, hash, 48) 266 #define WG_DUMP_BUF(buf, size) \ 267 wg_dump_buf(__func__, buf, size) 268 #else 269 #define WG_DUMP_HASH(name, hash) __nothing 270 #define WG_DUMP_HASH48(name, hash) __nothing 271 #define WG_DUMP_BUF(buf, size) __nothing 272 #endif /* WG_DEBUG_DUMP */ 273 274 /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-termintaed */ 275 #define WG_MAX_PROPLEN 32766 276 277 #define WG_MTU 1420 278 #define WG_ALLOWEDIPS 16 279 280 #define CURVE25519_KEY_LEN 32 281 #define TAI64N_LEN sizeof(uint32_t) * 3 282 #define POLY1305_AUTHTAG_LEN 16 283 #define HMAC_BLOCK_LEN 64 284 285 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 286 /* [N] 4.3: Hash functions */ 287 #define NOISE_DHLEN 32 288 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 289 #define NOISE_HASHLEN 32 290 #define NOISE_BLOCKLEN 64 291 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 292 /* [N] 5.1: "k" */ 293 #define NOISE_CIPHER_KEY_LEN 32 294 /* 295 * [N] 9.2: "psk" 296 * "... psk is a 32-byte secret value provided by the application." 297 */ 298 #define NOISE_PRESHARED_KEY_LEN 32 299 300 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 301 #define WG_TIMESTAMP_LEN TAI64N_LEN 302 303 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 304 305 #define WG_COOKIE_LEN 16 306 #define WG_MAC_LEN 16 307 #define WG_RANDVAL_LEN 24 308 309 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 310 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 311 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 312 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 313 #define WG_HASH_LEN NOISE_HASHLEN 314 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 315 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 316 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 317 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 318 #define WG_DATA_KEY_LEN 32 319 #define WG_SALT_LEN 24 320 321 /* 322 * The protocol messages 323 */ 324 struct wg_msg { 325 uint32_t wgm_type; 326 } __packed; 327 328 /* [W] 5.4.2 First Message: Initiator to Responder */ 329 struct wg_msg_init { 330 uint32_t wgmi_type; 331 uint32_t wgmi_sender; 332 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 333 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 334 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 335 uint8_t wgmi_mac1[WG_MAC_LEN]; 336 uint8_t wgmi_mac2[WG_MAC_LEN]; 337 } __packed; 338 339 /* [W] 5.4.3 Second Message: Responder to Initiator */ 340 struct wg_msg_resp { 341 uint32_t wgmr_type; 342 uint32_t wgmr_sender; 343 uint32_t wgmr_receiver; 344 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 345 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 346 uint8_t wgmr_mac1[WG_MAC_LEN]; 347 uint8_t wgmr_mac2[WG_MAC_LEN]; 348 } __packed; 349 350 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 351 struct wg_msg_data { 352 uint32_t wgmd_type; 353 uint32_t wgmd_receiver; 354 uint64_t wgmd_counter; 355 uint32_t wgmd_packet[0]; 356 } __packed; 357 358 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 359 struct wg_msg_cookie { 360 uint32_t wgmc_type; 361 uint32_t wgmc_receiver; 362 uint8_t wgmc_salt[WG_SALT_LEN]; 363 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 364 } __packed; 365 366 #define WG_MSG_TYPE_INIT 1 367 #define WG_MSG_TYPE_RESP 2 368 #define WG_MSG_TYPE_COOKIE 3 369 #define WG_MSG_TYPE_DATA 4 370 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 371 372 /* Sliding windows */ 373 374 #define SLIWIN_BITS 2048u 375 #define SLIWIN_TYPE uint32_t 376 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 377 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 378 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 379 380 struct sliwin { 381 SLIWIN_TYPE B[SLIWIN_WORDS]; 382 uint64_t T; 383 }; 384 385 static void 386 sliwin_reset(struct sliwin *W) 387 { 388 389 memset(W, 0, sizeof(*W)); 390 } 391 392 static int 393 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 394 { 395 396 /* 397 * If it's more than one window older than the highest sequence 398 * number we've seen, reject. 399 */ 400 #ifdef __HAVE_ATOMIC64_LOADSTORE 401 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 402 return EAUTH; 403 #endif 404 405 /* 406 * Otherwise, we need to take the lock to decide, so don't 407 * reject just yet. Caller must serialize a call to 408 * sliwin_update in this case. 409 */ 410 return 0; 411 } 412 413 static int 414 sliwin_update(struct sliwin *W, uint64_t S) 415 { 416 unsigned word, bit; 417 418 /* 419 * If it's more than one window older than the highest sequence 420 * number we've seen, reject. 421 */ 422 if (S + SLIWIN_NPKT < W->T) 423 return EAUTH; 424 425 /* 426 * If it's higher than the highest sequence number we've seen, 427 * advance the window. 428 */ 429 if (S > W->T) { 430 uint64_t i = W->T / SLIWIN_BPW; 431 uint64_t j = S / SLIWIN_BPW; 432 unsigned k; 433 434 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 435 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 436 #ifdef __HAVE_ATOMIC64_LOADSTORE 437 atomic_store_relaxed(&W->T, S); 438 #else 439 W->T = S; 440 #endif 441 } 442 443 /* Test and set the bit -- if already set, reject. */ 444 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 445 bit = S % SLIWIN_BPW; 446 if (W->B[word] & (1UL << bit)) 447 return EAUTH; 448 W->B[word] |= 1U << bit; 449 450 /* Accept! */ 451 return 0; 452 } 453 454 struct wg_session { 455 struct wg_peer *wgs_peer; 456 struct psref_target 457 wgs_psref; 458 459 int wgs_state; 460 #define WGS_STATE_UNKNOWN 0 461 #define WGS_STATE_INIT_ACTIVE 1 462 #define WGS_STATE_INIT_PASSIVE 2 463 #define WGS_STATE_ESTABLISHED 3 464 #define WGS_STATE_DESTROYING 4 465 466 time_t wgs_time_established; 467 time_t wgs_time_last_data_sent; 468 bool wgs_is_initiator; 469 470 uint32_t wgs_local_index; 471 uint32_t wgs_remote_index; 472 #ifdef __HAVE_ATOMIC64_LOADSTORE 473 volatile uint64_t 474 wgs_send_counter; 475 #else 476 kmutex_t wgs_send_counter_lock; 477 uint64_t wgs_send_counter; 478 #endif 479 480 struct { 481 kmutex_t lock; 482 struct sliwin window; 483 } *wgs_recvwin; 484 485 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 486 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 487 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 488 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 489 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 490 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 491 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 492 }; 493 494 struct wg_sockaddr { 495 union { 496 struct sockaddr_storage _ss; 497 struct sockaddr _sa; 498 struct sockaddr_in _sin; 499 struct sockaddr_in6 _sin6; 500 }; 501 struct psref_target wgsa_psref; 502 }; 503 504 #define wgsatoss(wgsa) (&(wgsa)->_ss) 505 #define wgsatosa(wgsa) (&(wgsa)->_sa) 506 #define wgsatosin(wgsa) (&(wgsa)->_sin) 507 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 508 509 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 510 511 struct wg_peer; 512 struct wg_allowedip { 513 struct radix_node wga_nodes[2]; 514 struct wg_sockaddr _wga_sa_addr; 515 struct wg_sockaddr _wga_sa_mask; 516 #define wga_sa_addr _wga_sa_addr._sa 517 #define wga_sa_mask _wga_sa_mask._sa 518 519 int wga_family; 520 uint8_t wga_cidr; 521 union { 522 struct in_addr _ip4; 523 struct in6_addr _ip6; 524 } wga_addr; 525 #define wga_addr4 wga_addr._ip4 526 #define wga_addr6 wga_addr._ip6 527 528 struct wg_peer *wga_peer; 529 }; 530 531 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 532 533 struct wg_ppsratecheck { 534 struct timeval wgprc_lasttime; 535 int wgprc_curpps; 536 }; 537 538 struct wg_softc; 539 struct wg_peer { 540 struct wg_softc *wgp_sc; 541 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 542 struct pslist_entry wgp_peerlist_entry; 543 pserialize_t wgp_psz; 544 struct psref_target wgp_psref; 545 kmutex_t *wgp_lock; 546 kmutex_t *wgp_intr_lock; 547 548 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 549 struct wg_sockaddr *wgp_endpoint; 550 struct wg_sockaddr *wgp_endpoint0; 551 volatile unsigned wgp_endpoint_changing; 552 bool wgp_endpoint_available; 553 554 /* The preshared key (optional) */ 555 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 556 557 struct wg_session *wgp_session_stable; 558 struct wg_session *wgp_session_unstable; 559 560 /* first outgoing packet awaiting session initiation */ 561 struct mbuf *wgp_pending; 562 563 /* timestamp in big-endian */ 564 wg_timestamp_t wgp_timestamp_latest_init; 565 566 struct timespec wgp_last_handshake_time; 567 568 callout_t wgp_rekey_timer; 569 callout_t wgp_handshake_timeout_timer; 570 callout_t wgp_session_dtor_timer; 571 572 time_t wgp_handshake_start_time; 573 574 int wgp_n_allowedips; 575 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 576 577 time_t wgp_latest_cookie_time; 578 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 579 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 580 bool wgp_last_sent_mac1_valid; 581 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 582 bool wgp_last_sent_cookie_valid; 583 584 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 585 586 time_t wgp_last_genrandval_time; 587 uint32_t wgp_randval; 588 589 struct wg_ppsratecheck wgp_ppsratecheck; 590 591 struct work wgp_work; 592 unsigned int wgp_tasks; 593 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 594 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 595 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 596 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 597 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 598 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 599 }; 600 601 struct wg_ops; 602 603 struct wg_softc { 604 struct ifnet wg_if; 605 LIST_ENTRY(wg_softc) wg_list; 606 kmutex_t *wg_lock; 607 kmutex_t *wg_intr_lock; 608 krwlock_t *wg_rwlock; 609 610 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 611 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 612 613 int wg_npeers; 614 struct pslist_head wg_peers; 615 struct thmap *wg_peers_bypubkey; 616 struct thmap *wg_peers_byname; 617 struct thmap *wg_sessions_byindex; 618 uint16_t wg_listen_port; 619 620 struct threadpool *wg_threadpool; 621 622 struct threadpool_job wg_job; 623 int wg_upcalls; 624 #define WG_UPCALL_INET __BIT(0) 625 #define WG_UPCALL_INET6 __BIT(1) 626 627 #ifdef INET 628 struct socket *wg_so4; 629 struct radix_node_head *wg_rtable_ipv4; 630 #endif 631 #ifdef INET6 632 struct socket *wg_so6; 633 struct radix_node_head *wg_rtable_ipv6; 634 #endif 635 636 struct wg_ppsratecheck wg_ppsratecheck; 637 638 struct wg_ops *wg_ops; 639 640 #ifdef WG_RUMPKERNEL 641 struct wg_user *wg_user; 642 #endif 643 }; 644 645 /* [W] 6.1 Preliminaries */ 646 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 647 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 648 #define WG_REKEY_AFTER_TIME 120 649 #define WG_REJECT_AFTER_TIME 180 650 #define WG_REKEY_ATTEMPT_TIME 90 651 #define WG_REKEY_TIMEOUT 5 652 #define WG_KEEPALIVE_TIMEOUT 10 653 654 #define WG_COOKIE_TIME 120 655 #define WG_RANDVAL_TIME (2 * 60) 656 657 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 658 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 659 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 660 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 661 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 662 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 663 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 664 665 static struct mbuf * 666 wg_get_mbuf(size_t, size_t); 667 668 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 669 struct mbuf *); 670 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 671 const uint32_t, const uint8_t [], const struct sockaddr *); 672 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 673 struct wg_session *, const struct wg_msg_init *); 674 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 675 676 static struct wg_peer * 677 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 678 struct psref *); 679 static struct wg_peer * 680 wg_lookup_peer_by_pubkey(struct wg_softc *, 681 const uint8_t [], struct psref *); 682 683 static struct wg_session * 684 wg_lookup_session_by_index(struct wg_softc *, 685 const uint32_t, struct psref *); 686 687 static void wg_update_endpoint_if_necessary(struct wg_peer *, 688 const struct sockaddr *); 689 690 static void wg_schedule_rekey_timer(struct wg_peer *); 691 static void wg_schedule_session_dtor_timer(struct wg_peer *); 692 693 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 694 static void wg_calculate_keys(struct wg_session *, const bool); 695 696 static void wg_clear_states(struct wg_session *); 697 698 static void wg_get_peer(struct wg_peer *, struct psref *); 699 static void wg_put_peer(struct wg_peer *, struct psref *); 700 701 static int wg_send_so(struct wg_peer *, struct mbuf *); 702 static int wg_send_udp(struct wg_peer *, struct mbuf *); 703 static int wg_output(struct ifnet *, struct mbuf *, 704 const struct sockaddr *, const struct rtentry *); 705 static void wg_input(struct ifnet *, struct mbuf *, const int); 706 static int wg_ioctl(struct ifnet *, u_long, void *); 707 static int wg_bind_port(struct wg_softc *, const uint16_t); 708 static int wg_init(struct ifnet *); 709 #ifdef ALTQ 710 static void wg_start(struct ifnet *); 711 #endif 712 static void wg_stop(struct ifnet *, int); 713 714 static void wg_peer_work(struct work *, void *); 715 static void wg_job(struct threadpool_job *); 716 static void wgintr(void *); 717 static void wg_purge_pending_packets(struct wg_peer *); 718 719 static int wg_clone_create(struct if_clone *, int); 720 static int wg_clone_destroy(struct ifnet *); 721 722 struct wg_ops { 723 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 724 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 725 void (*input)(struct ifnet *, struct mbuf *, const int); 726 int (*bind_port)(struct wg_softc *, const uint16_t); 727 }; 728 729 struct wg_ops wg_ops_rumpkernel = { 730 .send_hs_msg = wg_send_so, 731 .send_data_msg = wg_send_udp, 732 .input = wg_input, 733 .bind_port = wg_bind_port, 734 }; 735 736 #ifdef WG_RUMPKERNEL 737 static bool wg_user_mode(struct wg_softc *); 738 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 739 740 static int wg_send_user(struct wg_peer *, struct mbuf *); 741 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 742 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 743 744 struct wg_ops wg_ops_rumpuser = { 745 .send_hs_msg = wg_send_user, 746 .send_data_msg = wg_send_user, 747 .input = wg_input_user, 748 .bind_port = wg_bind_port_user, 749 }; 750 #endif 751 752 #define WG_PEER_READER_FOREACH(wgp, wg) \ 753 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 754 wgp_peerlist_entry) 755 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 756 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 757 wgp_peerlist_entry) 758 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 759 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 760 #define WG_PEER_WRITER_REMOVE(wgp) \ 761 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 762 763 struct wg_route { 764 struct radix_node wgr_nodes[2]; 765 struct wg_peer *wgr_peer; 766 }; 767 768 static struct radix_node_head * 769 wg_rnh(struct wg_softc *wg, const int family) 770 { 771 772 switch (family) { 773 case AF_INET: 774 return wg->wg_rtable_ipv4; 775 #ifdef INET6 776 case AF_INET6: 777 return wg->wg_rtable_ipv6; 778 #endif 779 default: 780 return NULL; 781 } 782 } 783 784 785 /* 786 * Global variables 787 */ 788 static volatile unsigned wg_count __cacheline_aligned; 789 790 struct psref_class *wg_psref_class __read_mostly; 791 792 static struct if_clone wg_cloner = 793 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 794 795 static struct pktqueue *wg_pktq __read_mostly; 796 static struct workqueue *wg_wq __read_mostly; 797 798 void wgattach(int); 799 /* ARGSUSED */ 800 void 801 wgattach(int count) 802 { 803 /* 804 * Nothing to do here, initialization is handled by the 805 * module initialization code in wginit() below). 806 */ 807 } 808 809 static void 810 wginit(void) 811 { 812 813 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 814 815 if_clone_attach(&wg_cloner); 816 } 817 818 /* 819 * XXX Kludge: This should just happen in wginit, but workqueue_create 820 * cannot be run until after CPUs have been detected, and wginit runs 821 * before configure. 822 */ 823 static int 824 wginitqueues(void) 825 { 826 int error __diagused; 827 828 wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL); 829 KASSERT(wg_pktq != NULL); 830 831 error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL, 832 PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 833 KASSERT(error == 0); 834 835 return 0; 836 } 837 838 static void 839 wg_guarantee_initialized(void) 840 { 841 static ONCE_DECL(init); 842 int error __diagused; 843 844 error = RUN_ONCE(&init, wginitqueues); 845 KASSERT(error == 0); 846 } 847 848 static int 849 wg_count_inc(void) 850 { 851 unsigned o, n; 852 853 do { 854 o = atomic_load_relaxed(&wg_count); 855 if (o == UINT_MAX) 856 return ENFILE; 857 n = o + 1; 858 } while (atomic_cas_uint(&wg_count, o, n) != o); 859 860 return 0; 861 } 862 863 static void 864 wg_count_dec(void) 865 { 866 unsigned c __diagused; 867 868 c = atomic_dec_uint_nv(&wg_count); 869 KASSERT(c != UINT_MAX); 870 } 871 872 static int 873 wgdetach(void) 874 { 875 876 /* Prevent new interface creation. */ 877 if_clone_detach(&wg_cloner); 878 879 /* Check whether there are any existing interfaces. */ 880 if (atomic_load_relaxed(&wg_count)) { 881 /* Back out -- reattach the cloner. */ 882 if_clone_attach(&wg_cloner); 883 return EBUSY; 884 } 885 886 /* No interfaces left. Nuke it. */ 887 workqueue_destroy(wg_wq); 888 pktq_destroy(wg_pktq); 889 psref_class_destroy(wg_psref_class); 890 891 return 0; 892 } 893 894 static void 895 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 896 uint8_t hash[WG_HASH_LEN]) 897 { 898 /* [W] 5.4: CONSTRUCTION */ 899 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 900 /* [W] 5.4: IDENTIFIER */ 901 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 902 struct blake2s state; 903 904 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 905 signature, strlen(signature)); 906 907 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 908 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 909 910 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 911 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 912 blake2s_update(&state, id, strlen(id)); 913 blake2s_final(&state, hash); 914 915 WG_DUMP_HASH("ckey", ckey); 916 WG_DUMP_HASH("hash", hash); 917 } 918 919 static void 920 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 921 const size_t inputsize) 922 { 923 struct blake2s state; 924 925 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 926 blake2s_update(&state, hash, WG_HASH_LEN); 927 blake2s_update(&state, input, inputsize); 928 blake2s_final(&state, hash); 929 } 930 931 static void 932 wg_algo_mac(uint8_t out[], const size_t outsize, 933 const uint8_t key[], const size_t keylen, 934 const uint8_t input1[], const size_t input1len, 935 const uint8_t input2[], const size_t input2len) 936 { 937 struct blake2s state; 938 939 blake2s_init(&state, outsize, key, keylen); 940 941 blake2s_update(&state, input1, input1len); 942 if (input2 != NULL) 943 blake2s_update(&state, input2, input2len); 944 blake2s_final(&state, out); 945 } 946 947 static void 948 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 949 const uint8_t input1[], const size_t input1len, 950 const uint8_t input2[], const size_t input2len) 951 { 952 struct blake2s state; 953 /* [W] 5.4: LABEL-MAC1 */ 954 const char *label = "mac1----"; 955 uint8_t key[WG_HASH_LEN]; 956 957 blake2s_init(&state, sizeof(key), NULL, 0); 958 blake2s_update(&state, label, strlen(label)); 959 blake2s_update(&state, input1, input1len); 960 blake2s_final(&state, key); 961 962 blake2s_init(&state, outsize, key, sizeof(key)); 963 if (input2 != NULL) 964 blake2s_update(&state, input2, input2len); 965 blake2s_final(&state, out); 966 } 967 968 static void 969 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 970 const uint8_t input1[], const size_t input1len) 971 { 972 struct blake2s state; 973 /* [W] 5.4: LABEL-COOKIE */ 974 const char *label = "cookie--"; 975 976 blake2s_init(&state, outsize, NULL, 0); 977 blake2s_update(&state, label, strlen(label)); 978 blake2s_update(&state, input1, input1len); 979 blake2s_final(&state, out); 980 } 981 982 static void 983 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 984 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 985 { 986 987 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 988 989 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 990 crypto_scalarmult_base(pubkey, privkey); 991 } 992 993 static void 994 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 995 const uint8_t privkey[WG_STATIC_KEY_LEN], 996 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 997 { 998 999 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 1000 1001 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 1002 KASSERT(ret == 0); 1003 } 1004 1005 static void 1006 wg_algo_hmac(uint8_t out[], const size_t outlen, 1007 const uint8_t key[], const size_t keylen, 1008 const uint8_t in[], const size_t inlen) 1009 { 1010 #define IPAD 0x36 1011 #define OPAD 0x5c 1012 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 1013 uint8_t ipad[HMAC_BLOCK_LEN]; 1014 uint8_t opad[HMAC_BLOCK_LEN]; 1015 size_t i; 1016 struct blake2s state; 1017 1018 KASSERT(outlen == WG_HASH_LEN); 1019 KASSERT(keylen <= HMAC_BLOCK_LEN); 1020 1021 memcpy(hmackey, key, keylen); 1022 1023 for (i = 0; i < sizeof(hmackey); i++) { 1024 ipad[i] = hmackey[i] ^ IPAD; 1025 opad[i] = hmackey[i] ^ OPAD; 1026 } 1027 1028 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1029 blake2s_update(&state, ipad, sizeof(ipad)); 1030 blake2s_update(&state, in, inlen); 1031 blake2s_final(&state, out); 1032 1033 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 1034 blake2s_update(&state, opad, sizeof(opad)); 1035 blake2s_update(&state, out, WG_HASH_LEN); 1036 blake2s_final(&state, out); 1037 #undef IPAD 1038 #undef OPAD 1039 } 1040 1041 static void 1042 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 1043 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 1044 const uint8_t input[], const size_t inputlen) 1045 { 1046 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 1047 uint8_t one[1]; 1048 1049 /* 1050 * [N] 4.3: "an input_key_material byte sequence with length 1051 * either zero bytes, 32 bytes, or DHLEN bytes." 1052 */ 1053 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 1054 1055 WG_DUMP_HASH("ckey", ckey); 1056 if (input != NULL) 1057 WG_DUMP_HASH("input", input); 1058 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 1059 input, inputlen); 1060 WG_DUMP_HASH("tmp1", tmp1); 1061 one[0] = 1; 1062 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1063 one, sizeof(one)); 1064 WG_DUMP_HASH("out1", out1); 1065 if (out2 == NULL) 1066 return; 1067 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 1068 tmp2[WG_KDF_OUTPUT_LEN] = 2; 1069 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1070 tmp2, sizeof(tmp2)); 1071 WG_DUMP_HASH("out2", out2); 1072 if (out3 == NULL) 1073 return; 1074 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 1075 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1076 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1077 tmp2, sizeof(tmp2)); 1078 WG_DUMP_HASH("out3", out3); 1079 } 1080 1081 static void __noinline 1082 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1083 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1084 const uint8_t local_key[WG_STATIC_KEY_LEN], 1085 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1086 { 1087 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1088 1089 wg_algo_dh(dhout, local_key, remote_key); 1090 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1091 1092 WG_DUMP_HASH("dhout", dhout); 1093 WG_DUMP_HASH("ckey", ckey); 1094 if (cipher_key != NULL) 1095 WG_DUMP_HASH("cipher_key", cipher_key); 1096 } 1097 1098 static void 1099 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1100 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1101 const uint8_t auth[], size_t authlen) 1102 { 1103 uint8_t nonce[(32 + 64) / 8] = {0}; 1104 long long unsigned int outsize; 1105 int error __diagused; 1106 1107 le64enc(&nonce[4], counter); 1108 1109 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1110 plainsize, auth, authlen, NULL, nonce, key); 1111 KASSERT(error == 0); 1112 KASSERT(outsize == expected_outsize); 1113 } 1114 1115 static int 1116 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1117 const uint64_t counter, const uint8_t encrypted[], 1118 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1119 { 1120 uint8_t nonce[(32 + 64) / 8] = {0}; 1121 long long unsigned int outsize; 1122 int error; 1123 1124 le64enc(&nonce[4], counter); 1125 1126 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1127 encrypted, encryptedsize, auth, authlen, nonce, key); 1128 if (error == 0) 1129 KASSERT(outsize == expected_outsize); 1130 return error; 1131 } 1132 1133 static void 1134 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1135 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1136 const uint8_t auth[], size_t authlen, 1137 const uint8_t nonce[WG_SALT_LEN]) 1138 { 1139 long long unsigned int outsize; 1140 int error __diagused; 1141 1142 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1143 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1144 plain, plainsize, auth, authlen, NULL, nonce, key); 1145 KASSERT(error == 0); 1146 KASSERT(outsize == expected_outsize); 1147 } 1148 1149 static int 1150 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1151 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1152 const uint8_t auth[], size_t authlen, 1153 const uint8_t nonce[WG_SALT_LEN]) 1154 { 1155 long long unsigned int outsize; 1156 int error; 1157 1158 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1159 encrypted, encryptedsize, auth, authlen, nonce, key); 1160 if (error == 0) 1161 KASSERT(outsize == expected_outsize); 1162 return error; 1163 } 1164 1165 static void 1166 wg_algo_tai64n(wg_timestamp_t timestamp) 1167 { 1168 struct timespec ts; 1169 1170 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1171 getnanotime(&ts); 1172 /* TAI64 label in external TAI64 format */ 1173 be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32)); 1174 /* second beginning from 1970 TAI */ 1175 be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU)); 1176 /* nanosecond in big-endian format */ 1177 be32enc(timestamp + 8, (uint32_t)ts.tv_nsec); 1178 } 1179 1180 /* 1181 * wg_get_stable_session(wgp, psref) 1182 * 1183 * Get a passive reference to the current stable session, or 1184 * return NULL if there is no current stable session. 1185 * 1186 * The pointer is always there but the session is not necessarily 1187 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1188 * the session may transition from ESTABLISHED to DESTROYING while 1189 * holding the passive reference. 1190 */ 1191 static struct wg_session * 1192 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1193 { 1194 int s; 1195 struct wg_session *wgs; 1196 1197 s = pserialize_read_enter(); 1198 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1199 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1200 wgs = NULL; 1201 else 1202 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1203 pserialize_read_exit(s); 1204 1205 return wgs; 1206 } 1207 1208 static void 1209 wg_put_session(struct wg_session *wgs, struct psref *psref) 1210 { 1211 1212 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1213 } 1214 1215 static void 1216 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1217 { 1218 struct wg_peer *wgp = wgs->wgs_peer; 1219 struct wg_session *wgs0 __diagused; 1220 void *garbage; 1221 1222 KASSERT(mutex_owned(wgp->wgp_lock)); 1223 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1224 1225 /* Remove the session from the table. */ 1226 wgs0 = thmap_del(wg->wg_sessions_byindex, 1227 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1228 KASSERT(wgs0 == wgs); 1229 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1230 1231 /* Wait for passive references to drain. */ 1232 pserialize_perform(wgp->wgp_psz); 1233 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1234 1235 /* Free memory, zero state, and transition to UNKNOWN. */ 1236 thmap_gc(wg->wg_sessions_byindex, garbage); 1237 wg_clear_states(wgs); 1238 wgs->wgs_state = WGS_STATE_UNKNOWN; 1239 } 1240 1241 /* 1242 * wg_get_session_index(wg, wgs) 1243 * 1244 * Choose a session index for wgs->wgs_local_index, and store it 1245 * in wg's table of sessions by index. 1246 * 1247 * wgs must be the unstable session of its peer, and must be 1248 * transitioning out of the UNKNOWN state. 1249 */ 1250 static void 1251 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1252 { 1253 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1254 struct wg_session *wgs0; 1255 uint32_t index; 1256 1257 KASSERT(mutex_owned(wgp->wgp_lock)); 1258 KASSERT(wgs == wgp->wgp_session_unstable); 1259 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1260 1261 do { 1262 /* Pick a uniform random index. */ 1263 index = cprng_strong32(); 1264 1265 /* Try to take it. */ 1266 wgs->wgs_local_index = index; 1267 wgs0 = thmap_put(wg->wg_sessions_byindex, 1268 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1269 1270 /* If someone else beat us, start over. */ 1271 } while (__predict_false(wgs0 != wgs)); 1272 } 1273 1274 /* 1275 * wg_put_session_index(wg, wgs) 1276 * 1277 * Remove wgs from the table of sessions by index, wait for any 1278 * passive references to drain, and transition the session to the 1279 * UNKNOWN state. 1280 * 1281 * wgs must be the unstable session of its peer, and must not be 1282 * UNKNOWN or ESTABLISHED. 1283 */ 1284 static void 1285 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1286 { 1287 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1288 1289 KASSERT(mutex_owned(wgp->wgp_lock)); 1290 KASSERT(wgs == wgp->wgp_session_unstable); 1291 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1292 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1293 1294 wg_destroy_session(wg, wgs); 1295 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1296 } 1297 1298 /* 1299 * Handshake patterns 1300 * 1301 * [W] 5: "These messages use the "IK" pattern from Noise" 1302 * [N] 7.5. Interactive handshake patterns (fundamental) 1303 * "The first character refers to the initiators static key:" 1304 * "I = Static key for initiator Immediately transmitted to responder, 1305 * despite reduced or absent identity hiding" 1306 * "The second character refers to the responders static key:" 1307 * "K = Static key for responder Known to initiator" 1308 * "IK: 1309 * <- s 1310 * ... 1311 * -> e, es, s, ss 1312 * <- e, ee, se" 1313 * [N] 9.4. Pattern modifiers 1314 * "IKpsk2: 1315 * <- s 1316 * ... 1317 * -> e, es, s, ss 1318 * <- e, ee, se, psk" 1319 */ 1320 static void 1321 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1322 struct wg_session *wgs, struct wg_msg_init *wgmi) 1323 { 1324 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1325 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1326 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1327 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1328 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1329 1330 KASSERT(mutex_owned(wgp->wgp_lock)); 1331 KASSERT(wgs == wgp->wgp_session_unstable); 1332 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1333 1334 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1335 wgmi->wgmi_sender = wgs->wgs_local_index; 1336 1337 /* [W] 5.4.2: First Message: Initiator to Responder */ 1338 1339 /* Ci := HASH(CONSTRUCTION) */ 1340 /* Hi := HASH(Ci || IDENTIFIER) */ 1341 wg_init_key_and_hash(ckey, hash); 1342 /* Hi := HASH(Hi || Sr^pub) */ 1343 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1344 1345 WG_DUMP_HASH("hash", hash); 1346 1347 /* [N] 2.2: "e" */ 1348 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1349 wg_algo_generate_keypair(pubkey, privkey); 1350 /* Ci := KDF1(Ci, Ei^pub) */ 1351 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1352 /* msg.ephemeral := Ei^pub */ 1353 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1354 /* Hi := HASH(Hi || msg.ephemeral) */ 1355 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1356 1357 WG_DUMP_HASH("ckey", ckey); 1358 WG_DUMP_HASH("hash", hash); 1359 1360 /* [N] 2.2: "es" */ 1361 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1362 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1363 1364 /* [N] 2.2: "s" */ 1365 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1366 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1367 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1368 hash, sizeof(hash)); 1369 /* Hi := HASH(Hi || msg.static) */ 1370 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1371 1372 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1373 1374 /* [N] 2.2: "ss" */ 1375 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1376 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1377 1378 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1379 wg_timestamp_t timestamp; 1380 wg_algo_tai64n(timestamp); 1381 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1382 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1383 /* Hi := HASH(Hi || msg.timestamp) */ 1384 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1385 1386 /* [W] 5.4.4 Cookie MACs */ 1387 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1388 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1389 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1390 /* Need mac1 to decrypt a cookie from a cookie message */ 1391 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1392 sizeof(wgp->wgp_last_sent_mac1)); 1393 wgp->wgp_last_sent_mac1_valid = true; 1394 1395 if (wgp->wgp_latest_cookie_time == 0 || 1396 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1397 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1398 else { 1399 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1400 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1401 (const uint8_t *)wgmi, 1402 offsetof(struct wg_msg_init, wgmi_mac2), 1403 NULL, 0); 1404 } 1405 1406 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1407 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1408 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1409 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1410 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1411 } 1412 1413 static void __noinline 1414 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1415 const struct sockaddr *src) 1416 { 1417 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1418 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1419 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1420 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1421 struct wg_peer *wgp; 1422 struct wg_session *wgs; 1423 int error, ret; 1424 struct psref psref_peer; 1425 uint8_t mac1[WG_MAC_LEN]; 1426 1427 WG_TRACE("init msg received"); 1428 1429 wg_algo_mac_mac1(mac1, sizeof(mac1), 1430 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1431 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1432 1433 /* 1434 * [W] 5.3: Denial of Service Mitigation & Cookies 1435 * "the responder, ..., must always reject messages with an invalid 1436 * msg.mac1" 1437 */ 1438 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1439 WG_DLOG("mac1 is invalid\n"); 1440 return; 1441 } 1442 1443 /* 1444 * [W] 5.4.2: First Message: Initiator to Responder 1445 * "When the responder receives this message, it does the same 1446 * operations so that its final state variables are identical, 1447 * replacing the operands of the DH function to produce equivalent 1448 * values." 1449 * Note that the following comments of operations are just copies of 1450 * the initiator's ones. 1451 */ 1452 1453 /* Ci := HASH(CONSTRUCTION) */ 1454 /* Hi := HASH(Ci || IDENTIFIER) */ 1455 wg_init_key_and_hash(ckey, hash); 1456 /* Hi := HASH(Hi || Sr^pub) */ 1457 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1458 1459 /* [N] 2.2: "e" */ 1460 /* Ci := KDF1(Ci, Ei^pub) */ 1461 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1462 sizeof(wgmi->wgmi_ephemeral)); 1463 /* Hi := HASH(Hi || msg.ephemeral) */ 1464 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1465 1466 WG_DUMP_HASH("ckey", ckey); 1467 1468 /* [N] 2.2: "es" */ 1469 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1470 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1471 1472 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1473 1474 /* [N] 2.2: "s" */ 1475 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1476 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1477 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1478 if (error != 0) { 1479 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1480 "%s: wg_algo_aead_dec for secret key failed\n", 1481 if_name(&wg->wg_if)); 1482 return; 1483 } 1484 /* Hi := HASH(Hi || msg.static) */ 1485 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1486 1487 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1488 if (wgp == NULL) { 1489 WG_DLOG("peer not found\n"); 1490 return; 1491 } 1492 1493 /* 1494 * Lock the peer to serialize access to cookie state. 1495 * 1496 * XXX Can we safely avoid holding the lock across DH? Take it 1497 * just to verify mac2 and then unlock/DH/lock? 1498 */ 1499 mutex_enter(wgp->wgp_lock); 1500 1501 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1502 WG_TRACE("under load"); 1503 /* 1504 * [W] 5.3: Denial of Service Mitigation & Cookies 1505 * "the responder, ..., and when under load may reject messages 1506 * with an invalid msg.mac2. If the responder receives a 1507 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1508 * and is under load, it may respond with a cookie reply 1509 * message" 1510 */ 1511 uint8_t zero[WG_MAC_LEN] = {0}; 1512 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1513 WG_TRACE("sending a cookie message: no cookie included"); 1514 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1515 wgmi->wgmi_mac1, src); 1516 goto out; 1517 } 1518 if (!wgp->wgp_last_sent_cookie_valid) { 1519 WG_TRACE("sending a cookie message: no cookie sent ever"); 1520 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1521 wgmi->wgmi_mac1, src); 1522 goto out; 1523 } 1524 uint8_t mac2[WG_MAC_LEN]; 1525 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1526 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1527 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1528 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1529 WG_DLOG("mac2 is invalid\n"); 1530 goto out; 1531 } 1532 WG_TRACE("under load, but continue to sending"); 1533 } 1534 1535 /* [N] 2.2: "ss" */ 1536 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1537 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1538 1539 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1540 wg_timestamp_t timestamp; 1541 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1542 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1543 hash, sizeof(hash)); 1544 if (error != 0) { 1545 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1546 "%s: peer %s: wg_algo_aead_dec for timestamp failed\n", 1547 if_name(&wg->wg_if), wgp->wgp_name); 1548 goto out; 1549 } 1550 /* Hi := HASH(Hi || msg.timestamp) */ 1551 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1552 1553 /* 1554 * [W] 5.1 "The responder keeps track of the greatest timestamp 1555 * received per peer and discards packets containing 1556 * timestamps less than or equal to it." 1557 */ 1558 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1559 sizeof(timestamp)); 1560 if (ret <= 0) { 1561 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1562 "%s: peer %s: invalid init msg: timestamp is old\n", 1563 if_name(&wg->wg_if), wgp->wgp_name); 1564 goto out; 1565 } 1566 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1567 1568 /* 1569 * Message is good -- we're committing to handle it now, unless 1570 * we were already initiating a session. 1571 */ 1572 wgs = wgp->wgp_session_unstable; 1573 switch (wgs->wgs_state) { 1574 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1575 wg_get_session_index(wg, wgs); 1576 break; 1577 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1578 WG_TRACE("Session already initializing, ignoring the message"); 1579 goto out; 1580 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1581 WG_TRACE("Session already initializing, destroying old states"); 1582 wg_clear_states(wgs); 1583 /* keep session index */ 1584 break; 1585 case WGS_STATE_ESTABLISHED: /* can't happen */ 1586 panic("unstable session can't be established"); 1587 break; 1588 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1589 WG_TRACE("Session destroying, but force to clear"); 1590 callout_stop(&wgp->wgp_session_dtor_timer); 1591 wg_clear_states(wgs); 1592 /* keep session index */ 1593 break; 1594 default: 1595 panic("invalid session state: %d", wgs->wgs_state); 1596 } 1597 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1598 1599 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1600 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1601 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1602 sizeof(wgmi->wgmi_ephemeral)); 1603 1604 wg_update_endpoint_if_necessary(wgp, src); 1605 1606 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1607 1608 wg_calculate_keys(wgs, false); 1609 wg_clear_states(wgs); 1610 1611 out: 1612 mutex_exit(wgp->wgp_lock); 1613 wg_put_peer(wgp, &psref_peer); 1614 } 1615 1616 static struct socket * 1617 wg_get_so_by_af(struct wg_softc *wg, const int af) 1618 { 1619 1620 switch (af) { 1621 #ifdef INET 1622 case AF_INET: 1623 return wg->wg_so4; 1624 #endif 1625 #ifdef INET6 1626 case AF_INET6: 1627 return wg->wg_so6; 1628 #endif 1629 default: 1630 panic("wg: no such af: %d", af); 1631 } 1632 } 1633 1634 static struct socket * 1635 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1636 { 1637 1638 return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa)); 1639 } 1640 1641 static struct wg_sockaddr * 1642 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1643 { 1644 struct wg_sockaddr *wgsa; 1645 int s; 1646 1647 s = pserialize_read_enter(); 1648 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1649 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1650 pserialize_read_exit(s); 1651 1652 return wgsa; 1653 } 1654 1655 static void 1656 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1657 { 1658 1659 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1660 } 1661 1662 static int 1663 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1664 { 1665 int error; 1666 struct socket *so; 1667 struct psref psref; 1668 struct wg_sockaddr *wgsa; 1669 1670 wgsa = wg_get_endpoint_sa(wgp, &psref); 1671 so = wg_get_so_by_peer(wgp, wgsa); 1672 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1673 wg_put_sa(wgp, wgsa, &psref); 1674 1675 return error; 1676 } 1677 1678 static int 1679 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1680 { 1681 int error; 1682 struct mbuf *m; 1683 struct wg_msg_init *wgmi; 1684 struct wg_session *wgs; 1685 1686 KASSERT(mutex_owned(wgp->wgp_lock)); 1687 1688 wgs = wgp->wgp_session_unstable; 1689 /* XXX pull dispatch out into wg_task_send_init_message */ 1690 switch (wgs->wgs_state) { 1691 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1692 wg_get_session_index(wg, wgs); 1693 break; 1694 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1695 WG_TRACE("Session already initializing, skip starting new one"); 1696 return EBUSY; 1697 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1698 WG_TRACE("Session already initializing, destroying old states"); 1699 wg_clear_states(wgs); 1700 /* keep session index */ 1701 break; 1702 case WGS_STATE_ESTABLISHED: /* can't happen */ 1703 panic("unstable session can't be established"); 1704 break; 1705 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1706 WG_TRACE("Session destroying"); 1707 /* XXX should wait? */ 1708 return EBUSY; 1709 } 1710 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1711 1712 m = m_gethdr(M_WAIT, MT_DATA); 1713 if (sizeof(*wgmi) > MHLEN) { 1714 m_clget(m, M_WAIT); 1715 CTASSERT(sizeof(*wgmi) <= MCLBYTES); 1716 } 1717 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1718 wgmi = mtod(m, struct wg_msg_init *); 1719 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1720 1721 error = wg->wg_ops->send_hs_msg(wgp, m); 1722 if (error == 0) { 1723 WG_TRACE("init msg sent"); 1724 1725 if (wgp->wgp_handshake_start_time == 0) 1726 wgp->wgp_handshake_start_time = time_uptime; 1727 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1728 MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz); 1729 } else { 1730 wg_put_session_index(wg, wgs); 1731 /* Initiation failed; toss packet waiting for it if any. */ 1732 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) 1733 m_freem(m); 1734 } 1735 1736 return error; 1737 } 1738 1739 static void 1740 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1741 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1742 const struct wg_msg_init *wgmi) 1743 { 1744 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1745 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1746 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1747 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1748 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1749 1750 KASSERT(mutex_owned(wgp->wgp_lock)); 1751 KASSERT(wgs == wgp->wgp_session_unstable); 1752 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1753 1754 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1755 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1756 1757 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1758 wgmr->wgmr_sender = wgs->wgs_local_index; 1759 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1760 1761 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1762 1763 /* [N] 2.2: "e" */ 1764 /* Er^priv, Er^pub := DH-GENERATE() */ 1765 wg_algo_generate_keypair(pubkey, privkey); 1766 /* Cr := KDF1(Cr, Er^pub) */ 1767 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1768 /* msg.ephemeral := Er^pub */ 1769 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1770 /* Hr := HASH(Hr || msg.ephemeral) */ 1771 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1772 1773 WG_DUMP_HASH("ckey", ckey); 1774 WG_DUMP_HASH("hash", hash); 1775 1776 /* [N] 2.2: "ee" */ 1777 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1778 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1779 1780 /* [N] 2.2: "se" */ 1781 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1782 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1783 1784 /* [N] 9.2: "psk" */ 1785 { 1786 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1787 /* Cr, r, k := KDF3(Cr, Q) */ 1788 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1789 sizeof(wgp->wgp_psk)); 1790 /* Hr := HASH(Hr || r) */ 1791 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1792 } 1793 1794 /* msg.empty := AEAD(k, 0, e, Hr) */ 1795 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1796 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1797 /* Hr := HASH(Hr || msg.empty) */ 1798 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1799 1800 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1801 1802 /* [W] 5.4.4: Cookie MACs */ 1803 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1804 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1805 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1806 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1807 /* Need mac1 to decrypt a cookie from a cookie message */ 1808 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1809 sizeof(wgp->wgp_last_sent_mac1)); 1810 wgp->wgp_last_sent_mac1_valid = true; 1811 1812 if (wgp->wgp_latest_cookie_time == 0 || 1813 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1814 /* msg.mac2 := 0^16 */ 1815 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1816 else { 1817 /* msg.mac2 := MAC(Lm, msg_b) */ 1818 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1819 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1820 (const uint8_t *)wgmr, 1821 offsetof(struct wg_msg_resp, wgmr_mac2), 1822 NULL, 0); 1823 } 1824 1825 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1826 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1827 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1828 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1829 wgs->wgs_remote_index = wgmi->wgmi_sender; 1830 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1831 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1832 } 1833 1834 static void 1835 wg_swap_sessions(struct wg_peer *wgp) 1836 { 1837 struct wg_session *wgs, *wgs_prev; 1838 1839 KASSERT(mutex_owned(wgp->wgp_lock)); 1840 1841 wgs = wgp->wgp_session_unstable; 1842 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1843 1844 wgs_prev = wgp->wgp_session_stable; 1845 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1846 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1847 atomic_store_release(&wgp->wgp_session_stable, wgs); 1848 wgp->wgp_session_unstable = wgs_prev; 1849 } 1850 1851 static void __noinline 1852 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1853 const struct sockaddr *src) 1854 { 1855 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1856 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1857 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1858 struct wg_peer *wgp; 1859 struct wg_session *wgs; 1860 struct psref psref; 1861 int error; 1862 uint8_t mac1[WG_MAC_LEN]; 1863 struct wg_session *wgs_prev; 1864 struct mbuf *m; 1865 1866 wg_algo_mac_mac1(mac1, sizeof(mac1), 1867 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1868 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1869 1870 /* 1871 * [W] 5.3: Denial of Service Mitigation & Cookies 1872 * "the responder, ..., must always reject messages with an invalid 1873 * msg.mac1" 1874 */ 1875 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1876 WG_DLOG("mac1 is invalid\n"); 1877 return; 1878 } 1879 1880 WG_TRACE("resp msg received"); 1881 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1882 if (wgs == NULL) { 1883 WG_TRACE("No session found"); 1884 return; 1885 } 1886 1887 wgp = wgs->wgs_peer; 1888 1889 mutex_enter(wgp->wgp_lock); 1890 1891 /* If we weren't waiting for a handshake response, drop it. */ 1892 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1893 WG_TRACE("peer sent spurious handshake response, ignoring"); 1894 goto out; 1895 } 1896 1897 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1898 WG_TRACE("under load"); 1899 /* 1900 * [W] 5.3: Denial of Service Mitigation & Cookies 1901 * "the responder, ..., and when under load may reject messages 1902 * with an invalid msg.mac2. If the responder receives a 1903 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1904 * and is under load, it may respond with a cookie reply 1905 * message" 1906 */ 1907 uint8_t zero[WG_MAC_LEN] = {0}; 1908 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1909 WG_TRACE("sending a cookie message: no cookie included"); 1910 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1911 wgmr->wgmr_mac1, src); 1912 goto out; 1913 } 1914 if (!wgp->wgp_last_sent_cookie_valid) { 1915 WG_TRACE("sending a cookie message: no cookie sent ever"); 1916 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1917 wgmr->wgmr_mac1, src); 1918 goto out; 1919 } 1920 uint8_t mac2[WG_MAC_LEN]; 1921 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1922 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1923 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1924 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1925 WG_DLOG("mac2 is invalid\n"); 1926 goto out; 1927 } 1928 WG_TRACE("under load, but continue to sending"); 1929 } 1930 1931 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1932 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1933 1934 /* 1935 * [W] 5.4.3 Second Message: Responder to Initiator 1936 * "When the initiator receives this message, it does the same 1937 * operations so that its final state variables are identical, 1938 * replacing the operands of the DH function to produce equivalent 1939 * values." 1940 * Note that the following comments of operations are just copies of 1941 * the initiator's ones. 1942 */ 1943 1944 /* [N] 2.2: "e" */ 1945 /* Cr := KDF1(Cr, Er^pub) */ 1946 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1947 sizeof(wgmr->wgmr_ephemeral)); 1948 /* Hr := HASH(Hr || msg.ephemeral) */ 1949 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1950 1951 WG_DUMP_HASH("ckey", ckey); 1952 WG_DUMP_HASH("hash", hash); 1953 1954 /* [N] 2.2: "ee" */ 1955 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1956 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1957 wgmr->wgmr_ephemeral); 1958 1959 /* [N] 2.2: "se" */ 1960 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1961 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1962 1963 /* [N] 9.2: "psk" */ 1964 { 1965 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1966 /* Cr, r, k := KDF3(Cr, Q) */ 1967 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1968 sizeof(wgp->wgp_psk)); 1969 /* Hr := HASH(Hr || r) */ 1970 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1971 } 1972 1973 { 1974 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 1975 /* msg.empty := AEAD(k, 0, e, Hr) */ 1976 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 1977 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 1978 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1979 if (error != 0) { 1980 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1981 "%s: peer %s: wg_algo_aead_dec for empty message failed\n", 1982 if_name(&wg->wg_if), wgp->wgp_name); 1983 goto out; 1984 } 1985 /* Hr := HASH(Hr || msg.empty) */ 1986 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1987 } 1988 1989 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 1990 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 1991 wgs->wgs_remote_index = wgmr->wgmr_sender; 1992 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1993 1994 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1995 wgs->wgs_state = WGS_STATE_ESTABLISHED; 1996 wgs->wgs_time_established = time_uptime; 1997 wgs->wgs_time_last_data_sent = 0; 1998 wgs->wgs_is_initiator = true; 1999 wg_calculate_keys(wgs, true); 2000 wg_clear_states(wgs); 2001 WG_TRACE("WGS_STATE_ESTABLISHED"); 2002 2003 callout_stop(&wgp->wgp_handshake_timeout_timer); 2004 2005 wg_swap_sessions(wgp); 2006 KASSERT(wgs == wgp->wgp_session_stable); 2007 wgs_prev = wgp->wgp_session_unstable; 2008 getnanotime(&wgp->wgp_last_handshake_time); 2009 wgp->wgp_handshake_start_time = 0; 2010 wgp->wgp_last_sent_mac1_valid = false; 2011 wgp->wgp_last_sent_cookie_valid = false; 2012 2013 wg_schedule_rekey_timer(wgp); 2014 2015 wg_update_endpoint_if_necessary(wgp, src); 2016 2017 /* 2018 * If we had a data packet queued up, send it; otherwise send a 2019 * keepalive message -- either way we have to send something 2020 * immediately or else the responder will never answer. 2021 */ 2022 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 2023 kpreempt_disable(); 2024 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 2025 M_SETCTX(m, wgp); 2026 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 2027 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 2028 if_name(&wg->wg_if)); 2029 m_freem(m); 2030 } 2031 kpreempt_enable(); 2032 } else { 2033 wg_send_keepalive_msg(wgp, wgs); 2034 } 2035 2036 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2037 /* Wait for wg_get_stable_session to drain. */ 2038 pserialize_perform(wgp->wgp_psz); 2039 2040 /* Transition ESTABLISHED->DESTROYING. */ 2041 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2042 2043 /* We can't destroy the old session immediately */ 2044 wg_schedule_session_dtor_timer(wgp); 2045 } else { 2046 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2047 "state=%d", wgs_prev->wgs_state); 2048 } 2049 2050 out: 2051 mutex_exit(wgp->wgp_lock); 2052 wg_put_session(wgs, &psref); 2053 } 2054 2055 static int 2056 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 2057 struct wg_session *wgs, const struct wg_msg_init *wgmi) 2058 { 2059 int error; 2060 struct mbuf *m; 2061 struct wg_msg_resp *wgmr; 2062 2063 KASSERT(mutex_owned(wgp->wgp_lock)); 2064 KASSERT(wgs == wgp->wgp_session_unstable); 2065 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 2066 2067 m = m_gethdr(M_WAIT, MT_DATA); 2068 if (sizeof(*wgmr) > MHLEN) { 2069 m_clget(m, M_WAIT); 2070 CTASSERT(sizeof(*wgmr) <= MCLBYTES); 2071 } 2072 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 2073 wgmr = mtod(m, struct wg_msg_resp *); 2074 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 2075 2076 error = wg->wg_ops->send_hs_msg(wgp, m); 2077 if (error == 0) 2078 WG_TRACE("resp msg sent"); 2079 return error; 2080 } 2081 2082 static struct wg_peer * 2083 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 2084 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 2085 { 2086 struct wg_peer *wgp; 2087 2088 int s = pserialize_read_enter(); 2089 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 2090 if (wgp != NULL) 2091 wg_get_peer(wgp, psref); 2092 pserialize_read_exit(s); 2093 2094 return wgp; 2095 } 2096 2097 static void 2098 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 2099 struct wg_msg_cookie *wgmc, const uint32_t sender, 2100 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 2101 { 2102 uint8_t cookie[WG_COOKIE_LEN]; 2103 uint8_t key[WG_HASH_LEN]; 2104 uint8_t addr[sizeof(struct in6_addr)]; 2105 size_t addrlen; 2106 uint16_t uh_sport; /* be */ 2107 2108 KASSERT(mutex_owned(wgp->wgp_lock)); 2109 2110 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 2111 wgmc->wgmc_receiver = sender; 2112 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 2113 2114 /* 2115 * [W] 5.4.7: Under Load: Cookie Reply Message 2116 * "The secret variable, Rm, changes every two minutes to a 2117 * random value" 2118 */ 2119 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2120 wgp->wgp_randval = cprng_strong32(); 2121 wgp->wgp_last_genrandval_time = time_uptime; 2122 } 2123 2124 switch (src->sa_family) { 2125 case AF_INET: { 2126 const struct sockaddr_in *sin = satocsin(src); 2127 addrlen = sizeof(sin->sin_addr); 2128 memcpy(addr, &sin->sin_addr, addrlen); 2129 uh_sport = sin->sin_port; 2130 break; 2131 } 2132 #ifdef INET6 2133 case AF_INET6: { 2134 const struct sockaddr_in6 *sin6 = satocsin6(src); 2135 addrlen = sizeof(sin6->sin6_addr); 2136 memcpy(addr, &sin6->sin6_addr, addrlen); 2137 uh_sport = sin6->sin6_port; 2138 break; 2139 } 2140 #endif 2141 default: 2142 panic("invalid af=%d", src->sa_family); 2143 } 2144 2145 wg_algo_mac(cookie, sizeof(cookie), 2146 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2147 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2148 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2149 sizeof(wg->wg_pubkey)); 2150 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2151 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2152 2153 /* Need to store to calculate mac2 */ 2154 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2155 wgp->wgp_last_sent_cookie_valid = true; 2156 } 2157 2158 static int 2159 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2160 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2161 const struct sockaddr *src) 2162 { 2163 int error; 2164 struct mbuf *m; 2165 struct wg_msg_cookie *wgmc; 2166 2167 KASSERT(mutex_owned(wgp->wgp_lock)); 2168 2169 m = m_gethdr(M_WAIT, MT_DATA); 2170 if (sizeof(*wgmc) > MHLEN) { 2171 m_clget(m, M_WAIT); 2172 CTASSERT(sizeof(*wgmc) <= MCLBYTES); 2173 } 2174 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2175 wgmc = mtod(m, struct wg_msg_cookie *); 2176 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2177 2178 error = wg->wg_ops->send_hs_msg(wgp, m); 2179 if (error == 0) 2180 WG_TRACE("cookie msg sent"); 2181 return error; 2182 } 2183 2184 static bool 2185 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2186 { 2187 #ifdef WG_DEBUG_PARAMS 2188 if (wg_force_underload) 2189 return true; 2190 #endif 2191 2192 /* 2193 * XXX we don't have a means of a load estimation. The purpose of 2194 * the mechanism is a DoS mitigation, so we consider frequent handshake 2195 * messages as (a kind of) load; if a message of the same type comes 2196 * to a peer within 1 second, we consider we are under load. 2197 */ 2198 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2199 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2200 return (time_uptime - last) == 0; 2201 } 2202 2203 static void 2204 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2205 { 2206 2207 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2208 2209 /* 2210 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2211 */ 2212 if (initiator) { 2213 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2214 wgs->wgs_chaining_key, NULL, 0); 2215 } else { 2216 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2217 wgs->wgs_chaining_key, NULL, 0); 2218 } 2219 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2220 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2221 } 2222 2223 static uint64_t 2224 wg_session_get_send_counter(struct wg_session *wgs) 2225 { 2226 #ifdef __HAVE_ATOMIC64_LOADSTORE 2227 return atomic_load_relaxed(&wgs->wgs_send_counter); 2228 #else 2229 uint64_t send_counter; 2230 2231 mutex_enter(&wgs->wgs_send_counter_lock); 2232 send_counter = wgs->wgs_send_counter; 2233 mutex_exit(&wgs->wgs_send_counter_lock); 2234 2235 return send_counter; 2236 #endif 2237 } 2238 2239 static uint64_t 2240 wg_session_inc_send_counter(struct wg_session *wgs) 2241 { 2242 #ifdef __HAVE_ATOMIC64_LOADSTORE 2243 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2244 #else 2245 uint64_t send_counter; 2246 2247 mutex_enter(&wgs->wgs_send_counter_lock); 2248 send_counter = wgs->wgs_send_counter++; 2249 mutex_exit(&wgs->wgs_send_counter_lock); 2250 2251 return send_counter; 2252 #endif 2253 } 2254 2255 static void 2256 wg_clear_states(struct wg_session *wgs) 2257 { 2258 2259 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2260 2261 wgs->wgs_send_counter = 0; 2262 sliwin_reset(&wgs->wgs_recvwin->window); 2263 2264 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2265 wgs_clear(handshake_hash); 2266 wgs_clear(chaining_key); 2267 wgs_clear(ephemeral_key_pub); 2268 wgs_clear(ephemeral_key_priv); 2269 wgs_clear(ephemeral_key_peer); 2270 #undef wgs_clear 2271 } 2272 2273 static struct wg_session * 2274 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2275 struct psref *psref) 2276 { 2277 struct wg_session *wgs; 2278 2279 int s = pserialize_read_enter(); 2280 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2281 if (wgs != NULL) { 2282 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2283 WGS_STATE_UNKNOWN); 2284 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2285 } 2286 pserialize_read_exit(s); 2287 2288 return wgs; 2289 } 2290 2291 static void 2292 wg_schedule_rekey_timer(struct wg_peer *wgp) 2293 { 2294 int timeout = MIN(wg_rekey_after_time, (unsigned)(INT_MAX / hz)); 2295 2296 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2297 } 2298 2299 static void 2300 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2301 { 2302 struct mbuf *m; 2303 2304 /* 2305 * [W] 6.5 Passive Keepalive 2306 * "A keepalive message is simply a transport data message with 2307 * a zero-length encapsulated encrypted inner-packet." 2308 */ 2309 m = m_gethdr(M_WAIT, MT_DATA); 2310 wg_send_data_msg(wgp, wgs, m); 2311 } 2312 2313 static bool 2314 wg_need_to_send_init_message(struct wg_session *wgs) 2315 { 2316 /* 2317 * [W] 6.2 Transport Message Limits 2318 * "if a peer is the initiator of a current secure session, 2319 * WireGuard will send a handshake initiation message to begin 2320 * a new secure session ... if after receiving a transport data 2321 * message, the current secure session is (REJECT-AFTER-TIME 2322 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2323 * not yet acted upon this event." 2324 */ 2325 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2326 (time_uptime - wgs->wgs_time_established) >= 2327 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2328 } 2329 2330 static void 2331 wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task) 2332 { 2333 2334 mutex_enter(wgp->wgp_intr_lock); 2335 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2336 if (wgp->wgp_tasks == 0) 2337 /* 2338 * XXX If the current CPU is already loaded -- e.g., if 2339 * there's already a bunch of handshakes queued up -- 2340 * consider tossing this over to another CPU to 2341 * distribute the load. 2342 */ 2343 workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL); 2344 wgp->wgp_tasks |= task; 2345 mutex_exit(wgp->wgp_intr_lock); 2346 } 2347 2348 static void 2349 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2350 { 2351 struct wg_sockaddr *wgsa_prev; 2352 2353 WG_TRACE("Changing endpoint"); 2354 2355 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2356 wgsa_prev = wgp->wgp_endpoint; 2357 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2358 wgp->wgp_endpoint0 = wgsa_prev; 2359 atomic_store_release(&wgp->wgp_endpoint_available, true); 2360 2361 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2362 } 2363 2364 static bool 2365 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2366 { 2367 uint16_t packet_len; 2368 const struct ip *ip; 2369 2370 if (__predict_false(decrypted_len < sizeof(struct ip))) 2371 return false; 2372 2373 ip = (const struct ip *)packet; 2374 if (ip->ip_v == 4) 2375 *af = AF_INET; 2376 else if (ip->ip_v == 6) 2377 *af = AF_INET6; 2378 else 2379 return false; 2380 2381 WG_DLOG("af=%d\n", *af); 2382 2383 switch (*af) { 2384 #ifdef INET 2385 case AF_INET: 2386 packet_len = ntohs(ip->ip_len); 2387 break; 2388 #endif 2389 #ifdef INET6 2390 case AF_INET6: { 2391 const struct ip6_hdr *ip6; 2392 2393 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr))) 2394 return false; 2395 2396 ip6 = (const struct ip6_hdr *)packet; 2397 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen); 2398 break; 2399 } 2400 #endif 2401 default: 2402 return false; 2403 } 2404 2405 WG_DLOG("packet_len=%u\n", packet_len); 2406 if (packet_len > decrypted_len) 2407 return false; 2408 2409 return true; 2410 } 2411 2412 static bool 2413 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2414 int af, char *packet) 2415 { 2416 struct sockaddr_storage ss; 2417 struct sockaddr *sa; 2418 struct psref psref; 2419 struct wg_peer *wgp; 2420 bool ok; 2421 2422 /* 2423 * II CRYPTOKEY ROUTING 2424 * "it will only accept it if its source IP resolves in the 2425 * table to the public key used in the secure session for 2426 * decrypting it." 2427 */ 2428 2429 if (af == AF_INET) { 2430 const struct ip *ip = (const struct ip *)packet; 2431 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2432 sockaddr_in_init(sin, &ip->ip_src, 0); 2433 sa = sintosa(sin); 2434 #ifdef INET6 2435 } else { 2436 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2437 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2438 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2439 sa = sin6tosa(sin6); 2440 #endif 2441 } 2442 2443 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2444 ok = (wgp == wgp_expected); 2445 if (wgp != NULL) 2446 wg_put_peer(wgp, &psref); 2447 2448 return ok; 2449 } 2450 2451 static void 2452 wg_session_dtor_timer(void *arg) 2453 { 2454 struct wg_peer *wgp = arg; 2455 2456 WG_TRACE("enter"); 2457 2458 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2459 } 2460 2461 static void 2462 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2463 { 2464 2465 /* 1 second grace period */ 2466 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2467 } 2468 2469 static bool 2470 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2471 { 2472 if (sa1->sa_family != sa2->sa_family) 2473 return false; 2474 2475 switch (sa1->sa_family) { 2476 #ifdef INET 2477 case AF_INET: 2478 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2479 #endif 2480 #ifdef INET6 2481 case AF_INET6: 2482 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2483 #endif 2484 default: 2485 return false; 2486 } 2487 } 2488 2489 static void 2490 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2491 const struct sockaddr *src) 2492 { 2493 struct wg_sockaddr *wgsa; 2494 struct psref psref; 2495 2496 wgsa = wg_get_endpoint_sa(wgp, &psref); 2497 2498 #ifdef WG_DEBUG_LOG 2499 char oldaddr[128], newaddr[128]; 2500 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2501 sockaddr_format(src, newaddr, sizeof(newaddr)); 2502 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2503 #endif 2504 2505 /* 2506 * III: "Since the packet has authenticated correctly, the source IP of 2507 * the outer UDP/IP packet is used to update the endpoint for peer..." 2508 */ 2509 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2510 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2511 /* XXX We can't change the endpoint twice in a short period */ 2512 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2513 wg_change_endpoint(wgp, src); 2514 } 2515 } 2516 2517 wg_put_sa(wgp, wgsa, &psref); 2518 } 2519 2520 static void __noinline 2521 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2522 const struct sockaddr *src) 2523 { 2524 struct wg_msg_data *wgmd; 2525 char *encrypted_buf = NULL, *decrypted_buf; 2526 size_t encrypted_len, decrypted_len; 2527 struct wg_session *wgs; 2528 struct wg_peer *wgp; 2529 int state; 2530 size_t mlen; 2531 struct psref psref; 2532 int error, af; 2533 bool success, free_encrypted_buf = false, ok; 2534 struct mbuf *n; 2535 2536 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2537 wgmd = mtod(m, struct wg_msg_data *); 2538 2539 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2540 WG_TRACE("data"); 2541 2542 /* Find the putative session, or drop. */ 2543 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2544 if (wgs == NULL) { 2545 WG_TRACE("No session found"); 2546 m_freem(m); 2547 return; 2548 } 2549 2550 /* 2551 * We are only ready to handle data when in INIT_PASSIVE, 2552 * ESTABLISHED, or DESTROYING. All transitions out of that 2553 * state dissociate the session index and drain psrefs. 2554 */ 2555 state = atomic_load_relaxed(&wgs->wgs_state); 2556 switch (state) { 2557 case WGS_STATE_UNKNOWN: 2558 panic("wg session %p in unknown state has session index %u", 2559 wgs, wgmd->wgmd_receiver); 2560 case WGS_STATE_INIT_ACTIVE: 2561 WG_TRACE("not yet ready for data"); 2562 goto out; 2563 case WGS_STATE_INIT_PASSIVE: 2564 case WGS_STATE_ESTABLISHED: 2565 case WGS_STATE_DESTROYING: 2566 break; 2567 } 2568 2569 /* 2570 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2571 * to update the endpoint if authentication succeeds. 2572 */ 2573 wgp = wgs->wgs_peer; 2574 2575 /* 2576 * Reject outrageously wrong sequence numbers before doing any 2577 * crypto work or taking any locks. 2578 */ 2579 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2580 le64toh(wgmd->wgmd_counter)); 2581 if (error) { 2582 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2583 "%s: peer %s: out-of-window packet: %"PRIu64"\n", 2584 if_name(&wg->wg_if), wgp->wgp_name, 2585 le64toh(wgmd->wgmd_counter)); 2586 goto out; 2587 } 2588 2589 /* Ensure the payload and authenticator are contiguous. */ 2590 mlen = m_length(m); 2591 encrypted_len = mlen - sizeof(*wgmd); 2592 if (encrypted_len < WG_AUTHTAG_LEN) { 2593 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2594 goto out; 2595 } 2596 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2597 if (success) { 2598 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2599 } else { 2600 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2601 if (encrypted_buf == NULL) { 2602 WG_DLOG("failed to allocate encrypted_buf\n"); 2603 goto out; 2604 } 2605 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2606 free_encrypted_buf = true; 2607 } 2608 /* m_ensure_contig may change m regardless of its result */ 2609 KASSERT(m->m_len >= sizeof(*wgmd)); 2610 wgmd = mtod(m, struct wg_msg_data *); 2611 2612 /* 2613 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2614 * a zero-length buffer (XXX). Drop if plaintext is longer 2615 * than MCLBYTES (XXX). 2616 */ 2617 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2618 if (decrypted_len > MCLBYTES) { 2619 /* FIXME handle larger data than MCLBYTES */ 2620 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2621 goto out; 2622 } 2623 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2624 if (n == NULL) { 2625 WG_DLOG("wg_get_mbuf failed\n"); 2626 goto out; 2627 } 2628 decrypted_buf = mtod(n, char *); 2629 2630 /* Decrypt and verify the packet. */ 2631 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2632 error = wg_algo_aead_dec(decrypted_buf, 2633 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2634 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2635 encrypted_len, NULL, 0); 2636 if (error != 0) { 2637 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2638 "%s: peer %s: failed to wg_algo_aead_dec\n", 2639 if_name(&wg->wg_if), wgp->wgp_name); 2640 m_freem(n); 2641 goto out; 2642 } 2643 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2644 2645 /* Packet is genuine. Reject it if a replay or just too old. */ 2646 mutex_enter(&wgs->wgs_recvwin->lock); 2647 error = sliwin_update(&wgs->wgs_recvwin->window, 2648 le64toh(wgmd->wgmd_counter)); 2649 mutex_exit(&wgs->wgs_recvwin->lock); 2650 if (error) { 2651 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2652 "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n", 2653 if_name(&wg->wg_if), wgp->wgp_name, 2654 le64toh(wgmd->wgmd_counter)); 2655 m_freem(n); 2656 goto out; 2657 } 2658 2659 /* We're done with m now; free it and chuck the pointers. */ 2660 m_freem(m); 2661 m = NULL; 2662 wgmd = NULL; 2663 2664 /* 2665 * Validate the encapsulated packet header and get the address 2666 * family, or drop. 2667 */ 2668 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2669 if (!ok) { 2670 m_freem(n); 2671 goto out; 2672 } 2673 2674 /* 2675 * The packet is genuine. Update the peer's endpoint if the 2676 * source address changed. 2677 * 2678 * XXX How to prevent DoS by replaying genuine packets from the 2679 * wrong source address? 2680 */ 2681 wg_update_endpoint_if_necessary(wgp, src); 2682 2683 /* Submit it into our network stack if routable. */ 2684 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2685 if (ok) { 2686 wg->wg_ops->input(&wg->wg_if, n, af); 2687 } else { 2688 char addrstr[INET6_ADDRSTRLEN]; 2689 memset(addrstr, 0, sizeof(addrstr)); 2690 if (af == AF_INET) { 2691 const struct ip *ip = (const struct ip *)decrypted_buf; 2692 IN_PRINT(addrstr, &ip->ip_src); 2693 #ifdef INET6 2694 } else if (af == AF_INET6) { 2695 const struct ip6_hdr *ip6 = 2696 (const struct ip6_hdr *)decrypted_buf; 2697 IN6_PRINT(addrstr, &ip6->ip6_src); 2698 #endif 2699 } 2700 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2701 "%s: peer %s: invalid source address (%s)\n", 2702 if_name(&wg->wg_if), wgp->wgp_name, addrstr); 2703 m_freem(n); 2704 /* 2705 * The inner address is invalid however the session is valid 2706 * so continue the session processing below. 2707 */ 2708 } 2709 n = NULL; 2710 2711 /* Update the state machine if necessary. */ 2712 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2713 /* 2714 * We were waiting for the initiator to send their 2715 * first data transport message, and that has happened. 2716 * Schedule a task to establish this session. 2717 */ 2718 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2719 } else { 2720 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2721 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2722 } 2723 /* 2724 * [W] 6.5 Passive Keepalive 2725 * "If a peer has received a validly-authenticated transport 2726 * data message (section 5.4.6), but does not have any packets 2727 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2728 * a keepalive message." 2729 */ 2730 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2731 (uintmax_t)time_uptime, 2732 (uintmax_t)wgs->wgs_time_last_data_sent); 2733 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2734 wg_keepalive_timeout) { 2735 WG_TRACE("Schedule sending keepalive message"); 2736 /* 2737 * We can't send a keepalive message here to avoid 2738 * a deadlock; we already hold the solock of a socket 2739 * that is used to send the message. 2740 */ 2741 wg_schedule_peer_task(wgp, 2742 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2743 } 2744 } 2745 out: 2746 wg_put_session(wgs, &psref); 2747 if (m != NULL) 2748 m_freem(m); 2749 if (free_encrypted_buf) 2750 kmem_intr_free(encrypted_buf, encrypted_len); 2751 } 2752 2753 static void __noinline 2754 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2755 { 2756 struct wg_session *wgs; 2757 struct wg_peer *wgp; 2758 struct psref psref; 2759 int error; 2760 uint8_t key[WG_HASH_LEN]; 2761 uint8_t cookie[WG_COOKIE_LEN]; 2762 2763 WG_TRACE("cookie msg received"); 2764 2765 /* Find the putative session. */ 2766 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2767 if (wgs == NULL) { 2768 WG_TRACE("No session found"); 2769 return; 2770 } 2771 2772 /* Lock the peer so we can update the cookie state. */ 2773 wgp = wgs->wgs_peer; 2774 mutex_enter(wgp->wgp_lock); 2775 2776 if (!wgp->wgp_last_sent_mac1_valid) { 2777 WG_TRACE("No valid mac1 sent (or expired)"); 2778 goto out; 2779 } 2780 2781 /* Decrypt the cookie and store it for later handshake retry. */ 2782 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2783 sizeof(wgp->wgp_pubkey)); 2784 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2785 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2786 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2787 wgmc->wgmc_salt); 2788 if (error != 0) { 2789 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2790 "%s: peer %s: wg_algo_aead_dec for cookie failed: " 2791 "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error); 2792 goto out; 2793 } 2794 /* 2795 * [W] 6.6: Interaction with Cookie Reply System 2796 * "it should simply store the decrypted cookie value from the cookie 2797 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2798 * timer for retrying a handshake initiation message." 2799 */ 2800 wgp->wgp_latest_cookie_time = time_uptime; 2801 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2802 out: 2803 mutex_exit(wgp->wgp_lock); 2804 wg_put_session(wgs, &psref); 2805 } 2806 2807 static struct mbuf * 2808 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2809 { 2810 struct wg_msg wgm; 2811 size_t mbuflen; 2812 size_t msglen; 2813 2814 /* 2815 * Get the mbuf chain length. It is already guaranteed, by 2816 * wg_overudp_cb, to be large enough for a struct wg_msg. 2817 */ 2818 mbuflen = m_length(m); 2819 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2820 2821 /* 2822 * Copy the message header (32-bit message type) out -- we'll 2823 * worry about contiguity and alignment later. 2824 */ 2825 m_copydata(m, 0, sizeof(wgm), &wgm); 2826 switch (le32toh(wgm.wgm_type)) { 2827 case WG_MSG_TYPE_INIT: 2828 msglen = sizeof(struct wg_msg_init); 2829 break; 2830 case WG_MSG_TYPE_RESP: 2831 msglen = sizeof(struct wg_msg_resp); 2832 break; 2833 case WG_MSG_TYPE_COOKIE: 2834 msglen = sizeof(struct wg_msg_cookie); 2835 break; 2836 case WG_MSG_TYPE_DATA: 2837 msglen = sizeof(struct wg_msg_data); 2838 break; 2839 default: 2840 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2841 "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if), 2842 le32toh(wgm.wgm_type)); 2843 goto error; 2844 } 2845 2846 /* Verify the mbuf chain is long enough for this type of message. */ 2847 if (__predict_false(mbuflen < msglen)) { 2848 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2849 le32toh(wgm.wgm_type)); 2850 goto error; 2851 } 2852 2853 /* Make the message header contiguous if necessary. */ 2854 if (__predict_false(m->m_len < msglen)) { 2855 m = m_pullup(m, msglen); 2856 if (m == NULL) 2857 return NULL; 2858 } 2859 2860 return m; 2861 2862 error: 2863 m_freem(m); 2864 return NULL; 2865 } 2866 2867 static void 2868 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2869 const struct sockaddr *src) 2870 { 2871 struct wg_msg *wgm; 2872 2873 KASSERT(curlwp->l_pflag & LP_BOUND); 2874 2875 m = wg_validate_msg_header(wg, m); 2876 if (__predict_false(m == NULL)) 2877 return; 2878 2879 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2880 wgm = mtod(m, struct wg_msg *); 2881 switch (le32toh(wgm->wgm_type)) { 2882 case WG_MSG_TYPE_INIT: 2883 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2884 break; 2885 case WG_MSG_TYPE_RESP: 2886 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2887 break; 2888 case WG_MSG_TYPE_COOKIE: 2889 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2890 break; 2891 case WG_MSG_TYPE_DATA: 2892 wg_handle_msg_data(wg, m, src); 2893 /* wg_handle_msg_data frees m for us */ 2894 return; 2895 default: 2896 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2897 } 2898 2899 m_freem(m); 2900 } 2901 2902 static void 2903 wg_receive_packets(struct wg_softc *wg, const int af) 2904 { 2905 2906 for (;;) { 2907 int error, flags; 2908 struct socket *so; 2909 struct mbuf *m = NULL; 2910 struct uio dummy_uio; 2911 struct mbuf *paddr = NULL; 2912 struct sockaddr *src; 2913 2914 so = wg_get_so_by_af(wg, af); 2915 flags = MSG_DONTWAIT; 2916 dummy_uio.uio_resid = 1000000000; 2917 2918 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2919 &flags); 2920 if (error || m == NULL) { 2921 //if (error == EWOULDBLOCK) 2922 return; 2923 } 2924 2925 KASSERT(paddr != NULL); 2926 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2927 src = mtod(paddr, struct sockaddr *); 2928 2929 wg_handle_packet(wg, m, src); 2930 } 2931 } 2932 2933 static void 2934 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2935 { 2936 2937 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2938 } 2939 2940 static void 2941 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2942 { 2943 2944 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 2945 } 2946 2947 static void 2948 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 2949 { 2950 struct wg_session *wgs; 2951 2952 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 2953 2954 KASSERT(mutex_owned(wgp->wgp_lock)); 2955 2956 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 2957 WGLOG(LOG_DEBUG, "%s: No endpoint available\n", 2958 if_name(&wg->wg_if)); 2959 /* XXX should do something? */ 2960 return; 2961 } 2962 2963 wgs = wgp->wgp_session_stable; 2964 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 2965 /* XXX What if the unstable session is already INIT_ACTIVE? */ 2966 wg_send_handshake_msg_init(wg, wgp); 2967 } else { 2968 /* rekey */ 2969 wgs = wgp->wgp_session_unstable; 2970 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2971 wg_send_handshake_msg_init(wg, wgp); 2972 } 2973 } 2974 2975 static void 2976 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 2977 { 2978 struct wg_session *wgs; 2979 2980 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 2981 2982 KASSERT(mutex_owned(wgp->wgp_lock)); 2983 KASSERT(wgp->wgp_handshake_start_time != 0); 2984 2985 wgs = wgp->wgp_session_unstable; 2986 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2987 return; 2988 2989 /* 2990 * XXX no real need to assign a new index here, but we do need 2991 * to transition to UNKNOWN temporarily 2992 */ 2993 wg_put_session_index(wg, wgs); 2994 2995 /* [W] 6.4 Handshake Initiation Retransmission */ 2996 if ((time_uptime - wgp->wgp_handshake_start_time) > 2997 wg_rekey_attempt_time) { 2998 /* Give up handshaking */ 2999 wgp->wgp_handshake_start_time = 0; 3000 WG_TRACE("give up"); 3001 3002 /* 3003 * If a new data packet comes, handshaking will be retried 3004 * and a new session would be established at that time, 3005 * however we don't want to send pending packets then. 3006 */ 3007 wg_purge_pending_packets(wgp); 3008 return; 3009 } 3010 3011 wg_task_send_init_message(wg, wgp); 3012 } 3013 3014 static void 3015 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 3016 { 3017 struct wg_session *wgs, *wgs_prev; 3018 struct mbuf *m; 3019 3020 KASSERT(mutex_owned(wgp->wgp_lock)); 3021 3022 wgs = wgp->wgp_session_unstable; 3023 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 3024 /* XXX Can this happen? */ 3025 return; 3026 3027 wgs->wgs_state = WGS_STATE_ESTABLISHED; 3028 wgs->wgs_time_established = time_uptime; 3029 wgs->wgs_time_last_data_sent = 0; 3030 wgs->wgs_is_initiator = false; 3031 WG_TRACE("WGS_STATE_ESTABLISHED"); 3032 3033 wg_swap_sessions(wgp); 3034 KASSERT(wgs == wgp->wgp_session_stable); 3035 wgs_prev = wgp->wgp_session_unstable; 3036 getnanotime(&wgp->wgp_last_handshake_time); 3037 wgp->wgp_handshake_start_time = 0; 3038 wgp->wgp_last_sent_mac1_valid = false; 3039 wgp->wgp_last_sent_cookie_valid = false; 3040 3041 /* If we had a data packet queued up, send it. */ 3042 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) { 3043 kpreempt_disable(); 3044 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3045 M_SETCTX(m, wgp); 3046 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3047 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3048 if_name(&wg->wg_if)); 3049 m_freem(m); 3050 } 3051 kpreempt_enable(); 3052 } 3053 3054 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 3055 /* Wait for wg_get_stable_session to drain. */ 3056 pserialize_perform(wgp->wgp_psz); 3057 3058 /* Transition ESTABLISHED->DESTROYING. */ 3059 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 3060 3061 /* We can't destroy the old session immediately */ 3062 wg_schedule_session_dtor_timer(wgp); 3063 } else { 3064 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 3065 "state=%d", wgs_prev->wgs_state); 3066 wg_clear_states(wgs_prev); 3067 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 3068 } 3069 } 3070 3071 static void 3072 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 3073 { 3074 3075 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 3076 3077 KASSERT(mutex_owned(wgp->wgp_lock)); 3078 3079 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 3080 pserialize_perform(wgp->wgp_psz); 3081 mutex_exit(wgp->wgp_lock); 3082 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 3083 wg_psref_class); 3084 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 3085 wg_psref_class); 3086 mutex_enter(wgp->wgp_lock); 3087 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 3088 } 3089 } 3090 3091 static void 3092 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 3093 { 3094 struct wg_session *wgs; 3095 3096 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 3097 3098 KASSERT(mutex_owned(wgp->wgp_lock)); 3099 3100 wgs = wgp->wgp_session_stable; 3101 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 3102 return; 3103 3104 wg_send_keepalive_msg(wgp, wgs); 3105 } 3106 3107 static void 3108 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 3109 { 3110 struct wg_session *wgs; 3111 3112 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 3113 3114 KASSERT(mutex_owned(wgp->wgp_lock)); 3115 3116 wgs = wgp->wgp_session_unstable; 3117 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 3118 wg_put_session_index(wg, wgs); 3119 } 3120 } 3121 3122 static void 3123 wg_peer_work(struct work *wk, void *cookie) 3124 { 3125 struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work); 3126 struct wg_softc *wg = wgp->wgp_sc; 3127 unsigned int tasks; 3128 3129 mutex_enter(wgp->wgp_intr_lock); 3130 while ((tasks = wgp->wgp_tasks) != 0) { 3131 wgp->wgp_tasks = 0; 3132 mutex_exit(wgp->wgp_intr_lock); 3133 3134 mutex_enter(wgp->wgp_lock); 3135 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 3136 wg_task_send_init_message(wg, wgp); 3137 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 3138 wg_task_retry_handshake(wg, wgp); 3139 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 3140 wg_task_establish_session(wg, wgp); 3141 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 3142 wg_task_endpoint_changed(wg, wgp); 3143 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 3144 wg_task_send_keepalive_message(wg, wgp); 3145 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 3146 wg_task_destroy_prev_session(wg, wgp); 3147 mutex_exit(wgp->wgp_lock); 3148 3149 mutex_enter(wgp->wgp_intr_lock); 3150 } 3151 mutex_exit(wgp->wgp_intr_lock); 3152 } 3153 3154 static void 3155 wg_job(struct threadpool_job *job) 3156 { 3157 struct wg_softc *wg = container_of(job, struct wg_softc, wg_job); 3158 int bound, upcalls; 3159 3160 mutex_enter(wg->wg_intr_lock); 3161 while ((upcalls = wg->wg_upcalls) != 0) { 3162 wg->wg_upcalls = 0; 3163 mutex_exit(wg->wg_intr_lock); 3164 bound = curlwp_bind(); 3165 if (ISSET(upcalls, WG_UPCALL_INET)) 3166 wg_receive_packets(wg, AF_INET); 3167 if (ISSET(upcalls, WG_UPCALL_INET6)) 3168 wg_receive_packets(wg, AF_INET6); 3169 curlwp_bindx(bound); 3170 mutex_enter(wg->wg_intr_lock); 3171 } 3172 threadpool_job_done(job); 3173 mutex_exit(wg->wg_intr_lock); 3174 } 3175 3176 static int 3177 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3178 { 3179 int error; 3180 uint16_t old_port = wg->wg_listen_port; 3181 3182 if (port != 0 && old_port == port) 3183 return 0; 3184 3185 struct sockaddr_in _sin, *sin = &_sin; 3186 sin->sin_len = sizeof(*sin); 3187 sin->sin_family = AF_INET; 3188 sin->sin_addr.s_addr = INADDR_ANY; 3189 sin->sin_port = htons(port); 3190 3191 error = sobind(wg->wg_so4, sintosa(sin), curlwp); 3192 if (error != 0) 3193 return error; 3194 3195 #ifdef INET6 3196 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3197 sin6->sin6_len = sizeof(*sin6); 3198 sin6->sin6_family = AF_INET6; 3199 sin6->sin6_addr = in6addr_any; 3200 sin6->sin6_port = htons(port); 3201 3202 error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp); 3203 if (error != 0) 3204 return error; 3205 #endif 3206 3207 wg->wg_listen_port = port; 3208 3209 return 0; 3210 } 3211 3212 static void 3213 wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag) 3214 { 3215 struct wg_softc *wg = cookie; 3216 int reason; 3217 3218 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3219 WG_UPCALL_INET : 3220 WG_UPCALL_INET6; 3221 3222 mutex_enter(wg->wg_intr_lock); 3223 wg->wg_upcalls |= reason; 3224 threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job); 3225 mutex_exit(wg->wg_intr_lock); 3226 } 3227 3228 static int 3229 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3230 struct sockaddr *src, void *arg) 3231 { 3232 struct wg_softc *wg = arg; 3233 struct wg_msg wgm; 3234 struct mbuf *m = *mp; 3235 3236 WG_TRACE("enter"); 3237 3238 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3239 KASSERT(offset <= m_length(m)); 3240 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3241 /* drop on the floor */ 3242 m_freem(m); 3243 return -1; 3244 } 3245 3246 /* 3247 * Copy the message header (32-bit message type) out -- we'll 3248 * worry about contiguity and alignment later. 3249 */ 3250 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3251 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3252 3253 /* 3254 * Handle DATA packets promptly as they arrive. Other packets 3255 * may require expensive public-key crypto and are not as 3256 * sensitive to latency, so defer them to the worker thread. 3257 */ 3258 switch (le32toh(wgm.wgm_type)) { 3259 case WG_MSG_TYPE_DATA: 3260 /* handle immediately */ 3261 m_adj(m, offset); 3262 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3263 m = m_pullup(m, sizeof(struct wg_msg_data)); 3264 if (m == NULL) 3265 return -1; 3266 } 3267 wg_handle_msg_data(wg, m, src); 3268 *mp = NULL; 3269 return 1; 3270 case WG_MSG_TYPE_INIT: 3271 case WG_MSG_TYPE_RESP: 3272 case WG_MSG_TYPE_COOKIE: 3273 /* pass through to so_receive in wg_receive_packets */ 3274 return 0; 3275 default: 3276 /* drop on the floor */ 3277 m_freem(m); 3278 return -1; 3279 } 3280 } 3281 3282 static int 3283 wg_socreate(struct wg_softc *wg, int af, struct socket **sop) 3284 { 3285 int error; 3286 struct socket *so; 3287 3288 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3289 if (error != 0) 3290 return error; 3291 3292 solock(so); 3293 so->so_upcallarg = wg; 3294 so->so_upcall = wg_so_upcall; 3295 so->so_rcv.sb_flags |= SB_UPCALL; 3296 inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3297 sounlock(so); 3298 3299 *sop = so; 3300 3301 return 0; 3302 } 3303 3304 static bool 3305 wg_session_hit_limits(struct wg_session *wgs) 3306 { 3307 3308 /* 3309 * [W] 6.2: Transport Message Limits 3310 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3311 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3312 * comes first, WireGuard will refuse to send any more transport data 3313 * messages using the current secure session, ..." 3314 */ 3315 KASSERT(wgs->wgs_time_established != 0); 3316 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3317 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3318 return true; 3319 } else if (wg_session_get_send_counter(wgs) > 3320 wg_reject_after_messages) { 3321 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3322 return true; 3323 } 3324 3325 return false; 3326 } 3327 3328 static void 3329 wgintr(void *cookie) 3330 { 3331 struct wg_peer *wgp; 3332 struct wg_session *wgs; 3333 struct mbuf *m; 3334 struct psref psref; 3335 3336 while ((m = pktq_dequeue(wg_pktq)) != NULL) { 3337 wgp = M_GETCTX(m, struct wg_peer *); 3338 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3339 WG_TRACE("no stable session"); 3340 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3341 goto next0; 3342 } 3343 if (__predict_false(wg_session_hit_limits(wgs))) { 3344 WG_TRACE("stable session hit limits"); 3345 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3346 goto next1; 3347 } 3348 wg_send_data_msg(wgp, wgs, m); 3349 m = NULL; /* consumed */ 3350 next1: wg_put_session(wgs, &psref); 3351 next0: if (m) 3352 m_freem(m); 3353 /* XXX Yield to avoid userland starvation? */ 3354 } 3355 } 3356 3357 static void 3358 wg_rekey_timer(void *arg) 3359 { 3360 struct wg_peer *wgp = arg; 3361 3362 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3363 } 3364 3365 static void 3366 wg_purge_pending_packets(struct wg_peer *wgp) 3367 { 3368 struct mbuf *m; 3369 3370 if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) 3371 m_freem(m); 3372 pktq_barrier(wg_pktq); 3373 } 3374 3375 static void 3376 wg_handshake_timeout_timer(void *arg) 3377 { 3378 struct wg_peer *wgp = arg; 3379 3380 WG_TRACE("enter"); 3381 3382 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3383 } 3384 3385 static struct wg_peer * 3386 wg_alloc_peer(struct wg_softc *wg) 3387 { 3388 struct wg_peer *wgp; 3389 3390 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3391 3392 wgp->wgp_sc = wg; 3393 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3394 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3395 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3396 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3397 wg_handshake_timeout_timer, wgp); 3398 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3399 callout_setfunc(&wgp->wgp_session_dtor_timer, 3400 wg_session_dtor_timer, wgp); 3401 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3402 wgp->wgp_endpoint_changing = false; 3403 wgp->wgp_endpoint_available = false; 3404 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3405 wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3406 wgp->wgp_psz = pserialize_create(); 3407 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3408 3409 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3410 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3411 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3412 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3413 3414 struct wg_session *wgs; 3415 wgp->wgp_session_stable = 3416 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3417 wgp->wgp_session_unstable = 3418 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3419 wgs = wgp->wgp_session_stable; 3420 wgs->wgs_peer = wgp; 3421 wgs->wgs_state = WGS_STATE_UNKNOWN; 3422 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3423 #ifndef __HAVE_ATOMIC64_LOADSTORE 3424 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3425 #endif 3426 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3427 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3428 3429 wgs = wgp->wgp_session_unstable; 3430 wgs->wgs_peer = wgp; 3431 wgs->wgs_state = WGS_STATE_UNKNOWN; 3432 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3433 #ifndef __HAVE_ATOMIC64_LOADSTORE 3434 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3435 #endif 3436 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3437 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3438 3439 return wgp; 3440 } 3441 3442 static void 3443 wg_destroy_peer(struct wg_peer *wgp) 3444 { 3445 struct wg_session *wgs; 3446 struct wg_softc *wg = wgp->wgp_sc; 3447 3448 /* Prevent new packets from this peer on any source address. */ 3449 rw_enter(wg->wg_rwlock, RW_WRITER); 3450 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3451 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3452 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3453 struct radix_node *rn; 3454 3455 KASSERT(rnh != NULL); 3456 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3457 &wga->wga_sa_mask, rnh); 3458 if (rn == NULL) { 3459 char addrstr[128]; 3460 sockaddr_format(&wga->wga_sa_addr, addrstr, 3461 sizeof(addrstr)); 3462 WGLOG(LOG_WARNING, "%s: Couldn't delete %s", 3463 if_name(&wg->wg_if), addrstr); 3464 } 3465 } 3466 rw_exit(wg->wg_rwlock); 3467 3468 /* Purge pending packets. */ 3469 wg_purge_pending_packets(wgp); 3470 3471 /* Halt all packet processing and timeouts. */ 3472 callout_halt(&wgp->wgp_rekey_timer, NULL); 3473 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3474 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3475 3476 /* Wait for any queued work to complete. */ 3477 workqueue_wait(wg_wq, &wgp->wgp_work); 3478 3479 wgs = wgp->wgp_session_unstable; 3480 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3481 mutex_enter(wgp->wgp_lock); 3482 wg_destroy_session(wg, wgs); 3483 mutex_exit(wgp->wgp_lock); 3484 } 3485 mutex_destroy(&wgs->wgs_recvwin->lock); 3486 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3487 #ifndef __HAVE_ATOMIC64_LOADSTORE 3488 mutex_destroy(&wgs->wgs_send_counter_lock); 3489 #endif 3490 kmem_free(wgs, sizeof(*wgs)); 3491 3492 wgs = wgp->wgp_session_stable; 3493 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3494 mutex_enter(wgp->wgp_lock); 3495 wg_destroy_session(wg, wgs); 3496 mutex_exit(wgp->wgp_lock); 3497 } 3498 mutex_destroy(&wgs->wgs_recvwin->lock); 3499 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3500 #ifndef __HAVE_ATOMIC64_LOADSTORE 3501 mutex_destroy(&wgs->wgs_send_counter_lock); 3502 #endif 3503 kmem_free(wgs, sizeof(*wgs)); 3504 3505 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3506 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3507 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3508 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3509 3510 pserialize_destroy(wgp->wgp_psz); 3511 mutex_obj_free(wgp->wgp_intr_lock); 3512 mutex_obj_free(wgp->wgp_lock); 3513 3514 kmem_free(wgp, sizeof(*wgp)); 3515 } 3516 3517 static void 3518 wg_destroy_all_peers(struct wg_softc *wg) 3519 { 3520 struct wg_peer *wgp, *wgp0 __diagused; 3521 void *garbage_byname, *garbage_bypubkey; 3522 3523 restart: 3524 garbage_byname = garbage_bypubkey = NULL; 3525 mutex_enter(wg->wg_lock); 3526 WG_PEER_WRITER_FOREACH(wgp, wg) { 3527 if (wgp->wgp_name[0]) { 3528 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3529 strlen(wgp->wgp_name)); 3530 KASSERT(wgp0 == wgp); 3531 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3532 } 3533 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3534 sizeof(wgp->wgp_pubkey)); 3535 KASSERT(wgp0 == wgp); 3536 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3537 WG_PEER_WRITER_REMOVE(wgp); 3538 wg->wg_npeers--; 3539 mutex_enter(wgp->wgp_lock); 3540 pserialize_perform(wgp->wgp_psz); 3541 mutex_exit(wgp->wgp_lock); 3542 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3543 break; 3544 } 3545 mutex_exit(wg->wg_lock); 3546 3547 if (wgp == NULL) 3548 return; 3549 3550 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3551 3552 wg_destroy_peer(wgp); 3553 thmap_gc(wg->wg_peers_byname, garbage_byname); 3554 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3555 3556 goto restart; 3557 } 3558 3559 static int 3560 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3561 { 3562 struct wg_peer *wgp, *wgp0 __diagused; 3563 void *garbage_byname, *garbage_bypubkey; 3564 3565 mutex_enter(wg->wg_lock); 3566 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3567 if (wgp != NULL) { 3568 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3569 sizeof(wgp->wgp_pubkey)); 3570 KASSERT(wgp0 == wgp); 3571 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3572 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3573 WG_PEER_WRITER_REMOVE(wgp); 3574 wg->wg_npeers--; 3575 if (wg->wg_npeers == 0) 3576 if_link_state_change(&wg->wg_if, LINK_STATE_DOWN); 3577 mutex_enter(wgp->wgp_lock); 3578 pserialize_perform(wgp->wgp_psz); 3579 mutex_exit(wgp->wgp_lock); 3580 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3581 } 3582 mutex_exit(wg->wg_lock); 3583 3584 if (wgp == NULL) 3585 return ENOENT; 3586 3587 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3588 3589 wg_destroy_peer(wgp); 3590 thmap_gc(wg->wg_peers_byname, garbage_byname); 3591 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3592 3593 return 0; 3594 } 3595 3596 static int 3597 wg_if_attach(struct wg_softc *wg) 3598 { 3599 3600 wg->wg_if.if_addrlen = 0; 3601 wg->wg_if.if_mtu = WG_MTU; 3602 wg->wg_if.if_flags = IFF_MULTICAST; 3603 wg->wg_if.if_extflags = IFEF_MPSAFE; 3604 wg->wg_if.if_ioctl = wg_ioctl; 3605 wg->wg_if.if_output = wg_output; 3606 wg->wg_if.if_init = wg_init; 3607 #ifdef ALTQ 3608 wg->wg_if.if_start = wg_start; 3609 #endif 3610 wg->wg_if.if_stop = wg_stop; 3611 wg->wg_if.if_type = IFT_OTHER; 3612 wg->wg_if.if_dlt = DLT_NULL; 3613 wg->wg_if.if_softc = wg; 3614 #ifdef ALTQ 3615 IFQ_SET_READY(&wg->wg_if.if_snd); 3616 #endif 3617 if_initialize(&wg->wg_if); 3618 3619 wg->wg_if.if_link_state = LINK_STATE_DOWN; 3620 if_alloc_sadl(&wg->wg_if); 3621 if_register(&wg->wg_if); 3622 3623 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3624 3625 return 0; 3626 } 3627 3628 static void 3629 wg_if_detach(struct wg_softc *wg) 3630 { 3631 struct ifnet *ifp = &wg->wg_if; 3632 3633 bpf_detach(ifp); 3634 if_detach(ifp); 3635 } 3636 3637 static int 3638 wg_clone_create(struct if_clone *ifc, int unit) 3639 { 3640 struct wg_softc *wg; 3641 int error; 3642 3643 wg_guarantee_initialized(); 3644 3645 error = wg_count_inc(); 3646 if (error) 3647 return error; 3648 3649 wg = kmem_zalloc(sizeof(*wg), KM_SLEEP); 3650 3651 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3652 3653 PSLIST_INIT(&wg->wg_peers); 3654 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3655 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3656 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3657 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3658 wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 3659 wg->wg_rwlock = rw_obj_alloc(); 3660 threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock, 3661 "%s", if_name(&wg->wg_if)); 3662 wg->wg_ops = &wg_ops_rumpkernel; 3663 3664 error = threadpool_get(&wg->wg_threadpool, PRI_NONE); 3665 if (error) 3666 goto fail0; 3667 3668 #ifdef INET 3669 error = wg_socreate(wg, AF_INET, &wg->wg_so4); 3670 if (error) 3671 goto fail1; 3672 rn_inithead((void **)&wg->wg_rtable_ipv4, 3673 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3674 #endif 3675 #ifdef INET6 3676 error = wg_socreate(wg, AF_INET6, &wg->wg_so6); 3677 if (error) 3678 goto fail2; 3679 rn_inithead((void **)&wg->wg_rtable_ipv6, 3680 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3681 #endif 3682 3683 error = wg_if_attach(wg); 3684 if (error) 3685 goto fail3; 3686 3687 return 0; 3688 3689 fail4: __unused 3690 wg_if_detach(wg); 3691 fail3: wg_destroy_all_peers(wg); 3692 #ifdef INET6 3693 solock(wg->wg_so6); 3694 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3695 sounlock(wg->wg_so6); 3696 #endif 3697 #ifdef INET 3698 solock(wg->wg_so4); 3699 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3700 sounlock(wg->wg_so4); 3701 #endif 3702 mutex_enter(wg->wg_intr_lock); 3703 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3704 mutex_exit(wg->wg_intr_lock); 3705 #ifdef INET6 3706 if (wg->wg_rtable_ipv6 != NULL) 3707 free(wg->wg_rtable_ipv6, M_RTABLE); 3708 soclose(wg->wg_so6); 3709 fail2: 3710 #endif 3711 #ifdef INET 3712 if (wg->wg_rtable_ipv4 != NULL) 3713 free(wg->wg_rtable_ipv4, M_RTABLE); 3714 soclose(wg->wg_so4); 3715 fail1: 3716 #endif 3717 threadpool_put(wg->wg_threadpool, PRI_NONE); 3718 fail0: threadpool_job_destroy(&wg->wg_job); 3719 rw_obj_free(wg->wg_rwlock); 3720 mutex_obj_free(wg->wg_intr_lock); 3721 mutex_obj_free(wg->wg_lock); 3722 thmap_destroy(wg->wg_sessions_byindex); 3723 thmap_destroy(wg->wg_peers_byname); 3724 thmap_destroy(wg->wg_peers_bypubkey); 3725 PSLIST_DESTROY(&wg->wg_peers); 3726 kmem_free(wg, sizeof(*wg)); 3727 wg_count_dec(); 3728 return error; 3729 } 3730 3731 static int 3732 wg_clone_destroy(struct ifnet *ifp) 3733 { 3734 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3735 3736 #ifdef WG_RUMPKERNEL 3737 if (wg_user_mode(wg)) { 3738 rumpuser_wg_destroy(wg->wg_user); 3739 wg->wg_user = NULL; 3740 } 3741 #endif 3742 3743 wg_if_detach(wg); 3744 wg_destroy_all_peers(wg); 3745 #ifdef INET6 3746 solock(wg->wg_so6); 3747 wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL; 3748 sounlock(wg->wg_so6); 3749 #endif 3750 #ifdef INET 3751 solock(wg->wg_so4); 3752 wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL; 3753 sounlock(wg->wg_so4); 3754 #endif 3755 mutex_enter(wg->wg_intr_lock); 3756 threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job); 3757 mutex_exit(wg->wg_intr_lock); 3758 #ifdef INET6 3759 if (wg->wg_rtable_ipv6 != NULL) 3760 free(wg->wg_rtable_ipv6, M_RTABLE); 3761 soclose(wg->wg_so6); 3762 #endif 3763 #ifdef INET 3764 if (wg->wg_rtable_ipv4 != NULL) 3765 free(wg->wg_rtable_ipv4, M_RTABLE); 3766 soclose(wg->wg_so4); 3767 #endif 3768 threadpool_put(wg->wg_threadpool, PRI_NONE); 3769 threadpool_job_destroy(&wg->wg_job); 3770 rw_obj_free(wg->wg_rwlock); 3771 mutex_obj_free(wg->wg_intr_lock); 3772 mutex_obj_free(wg->wg_lock); 3773 thmap_destroy(wg->wg_sessions_byindex); 3774 thmap_destroy(wg->wg_peers_byname); 3775 thmap_destroy(wg->wg_peers_bypubkey); 3776 PSLIST_DESTROY(&wg->wg_peers); 3777 kmem_free(wg, sizeof(*wg)); 3778 wg_count_dec(); 3779 3780 return 0; 3781 } 3782 3783 static struct wg_peer * 3784 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3785 struct psref *psref) 3786 { 3787 struct radix_node_head *rnh; 3788 struct radix_node *rn; 3789 struct wg_peer *wgp = NULL; 3790 struct wg_allowedip *wga; 3791 3792 #ifdef WG_DEBUG_LOG 3793 char addrstr[128]; 3794 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3795 WG_DLOG("sa=%s\n", addrstr); 3796 #endif 3797 3798 rw_enter(wg->wg_rwlock, RW_READER); 3799 3800 rnh = wg_rnh(wg, sa->sa_family); 3801 if (rnh == NULL) 3802 goto out; 3803 3804 rn = rnh->rnh_matchaddr(sa, rnh); 3805 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3806 goto out; 3807 3808 WG_TRACE("success"); 3809 3810 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3811 wgp = wga->wga_peer; 3812 wg_get_peer(wgp, psref); 3813 3814 out: 3815 rw_exit(wg->wg_rwlock); 3816 return wgp; 3817 } 3818 3819 static void 3820 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3821 struct wg_session *wgs, struct wg_msg_data *wgmd) 3822 { 3823 3824 memset(wgmd, 0, sizeof(*wgmd)); 3825 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3826 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3827 /* [W] 5.4.6: msg.counter := Nm^send */ 3828 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3829 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3830 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3831 } 3832 3833 static int 3834 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3835 const struct rtentry *rt) 3836 { 3837 struct wg_softc *wg = ifp->if_softc; 3838 struct wg_peer *wgp = NULL; 3839 struct wg_session *wgs = NULL; 3840 struct psref wgp_psref, wgs_psref; 3841 int bound; 3842 int error; 3843 3844 bound = curlwp_bind(); 3845 3846 /* TODO make the nest limit configurable via sysctl */ 3847 error = if_tunnel_check_nesting(ifp, m, 1); 3848 if (error) { 3849 WGLOG(LOG_ERR, 3850 "%s: tunneling loop detected and packet dropped\n", 3851 if_name(&wg->wg_if)); 3852 goto out0; 3853 } 3854 3855 #ifdef ALTQ 3856 bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags) 3857 & ALTQF_ENABLED; 3858 if (altq) 3859 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3860 #endif 3861 3862 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3863 3864 m->m_flags &= ~(M_BCAST|M_MCAST); 3865 3866 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3867 if (wgp == NULL) { 3868 WG_TRACE("peer not found"); 3869 error = EHOSTUNREACH; 3870 goto out0; 3871 } 3872 3873 /* Clear checksum-offload flags. */ 3874 m->m_pkthdr.csum_flags = 0; 3875 m->m_pkthdr.csum_data = 0; 3876 3877 /* Check whether there's an established session. */ 3878 wgs = wg_get_stable_session(wgp, &wgs_psref); 3879 if (wgs == NULL) { 3880 /* 3881 * No established session. If we're the first to try 3882 * sending data, schedule a handshake and queue the 3883 * packet for when the handshake is done; otherwise 3884 * just drop the packet and let the ongoing handshake 3885 * attempt continue. We could queue more data packets 3886 * but it's not clear that's worthwhile. 3887 */ 3888 if (atomic_cas_ptr(&wgp->wgp_pending, NULL, m) == NULL) { 3889 m = NULL; /* consume */ 3890 WG_TRACE("queued first packet; init handshake"); 3891 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3892 } else { 3893 WG_TRACE("first packet already queued, dropping"); 3894 } 3895 goto out1; 3896 } 3897 3898 /* There's an established session. Toss it in the queue. */ 3899 #ifdef ALTQ 3900 if (altq) { 3901 mutex_enter(ifp->if_snd.ifq_lock); 3902 if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 3903 M_SETCTX(m, wgp); 3904 ALTQ_ENQUEUE(&ifp->if_snd, m, error); 3905 m = NULL; /* consume */ 3906 } 3907 mutex_exit(ifp->if_snd.ifq_lock); 3908 if (m == NULL) { 3909 wg_start(ifp); 3910 goto out2; 3911 } 3912 } 3913 #endif 3914 kpreempt_disable(); 3915 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 3916 M_SETCTX(m, wgp); 3917 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 3918 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 3919 if_name(&wg->wg_if)); 3920 error = ENOBUFS; 3921 goto out3; 3922 } 3923 m = NULL; /* consumed */ 3924 error = 0; 3925 out3: kpreempt_enable(); 3926 3927 #ifdef ALTQ 3928 out2: 3929 #endif 3930 wg_put_session(wgs, &wgs_psref); 3931 out1: wg_put_peer(wgp, &wgp_psref); 3932 out0: if (m) 3933 m_freem(m); 3934 curlwp_bindx(bound); 3935 return error; 3936 } 3937 3938 static int 3939 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3940 { 3941 struct psref psref; 3942 struct wg_sockaddr *wgsa; 3943 int error; 3944 struct socket *so; 3945 3946 wgsa = wg_get_endpoint_sa(wgp, &psref); 3947 so = wg_get_so_by_peer(wgp, wgsa); 3948 solock(so); 3949 if (wgsatosa(wgsa)->sa_family == AF_INET) { 3950 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 3951 } else { 3952 #ifdef INET6 3953 error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa), 3954 NULL, curlwp); 3955 #else 3956 m_freem(m); 3957 error = EPFNOSUPPORT; 3958 #endif 3959 } 3960 sounlock(so); 3961 wg_put_sa(wgp, wgsa, &psref); 3962 3963 return error; 3964 } 3965 3966 /* Inspired by pppoe_get_mbuf */ 3967 static struct mbuf * 3968 wg_get_mbuf(size_t leading_len, size_t len) 3969 { 3970 struct mbuf *m; 3971 3972 KASSERT(leading_len <= MCLBYTES); 3973 KASSERT(len <= MCLBYTES - leading_len); 3974 3975 m = m_gethdr(M_DONTWAIT, MT_DATA); 3976 if (m == NULL) 3977 return NULL; 3978 if (len + leading_len > MHLEN) { 3979 m_clget(m, M_DONTWAIT); 3980 if ((m->m_flags & M_EXT) == 0) { 3981 m_free(m); 3982 return NULL; 3983 } 3984 } 3985 m->m_data += leading_len; 3986 m->m_pkthdr.len = m->m_len = len; 3987 3988 return m; 3989 } 3990 3991 static int 3992 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 3993 struct mbuf *m) 3994 { 3995 struct wg_softc *wg = wgp->wgp_sc; 3996 int error; 3997 size_t inner_len, padded_len, encrypted_len; 3998 char *padded_buf = NULL; 3999 size_t mlen; 4000 struct wg_msg_data *wgmd; 4001 bool free_padded_buf = false; 4002 struct mbuf *n; 4003 size_t leading_len = max_hdr + sizeof(struct udphdr); 4004 4005 mlen = m_length(m); 4006 inner_len = mlen; 4007 padded_len = roundup(mlen, 16); 4008 encrypted_len = padded_len + WG_AUTHTAG_LEN; 4009 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 4010 inner_len, padded_len, encrypted_len); 4011 if (mlen != 0) { 4012 bool success; 4013 success = m_ensure_contig(&m, padded_len); 4014 if (success) { 4015 padded_buf = mtod(m, char *); 4016 } else { 4017 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 4018 if (padded_buf == NULL) { 4019 error = ENOBUFS; 4020 goto end; 4021 } 4022 free_padded_buf = true; 4023 m_copydata(m, 0, mlen, padded_buf); 4024 } 4025 memset(padded_buf + mlen, 0, padded_len - inner_len); 4026 } 4027 4028 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 4029 if (n == NULL) { 4030 error = ENOBUFS; 4031 goto end; 4032 } 4033 KASSERT(n->m_len >= sizeof(*wgmd)); 4034 wgmd = mtod(n, struct wg_msg_data *); 4035 wg_fill_msg_data(wg, wgp, wgs, wgmd); 4036 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 4037 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 4038 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 4039 padded_buf, padded_len, 4040 NULL, 0); 4041 4042 error = wg->wg_ops->send_data_msg(wgp, n); 4043 if (error == 0) { 4044 struct ifnet *ifp = &wg->wg_if; 4045 if_statadd(ifp, if_obytes, mlen); 4046 if_statinc(ifp, if_opackets); 4047 if (wgs->wgs_is_initiator && 4048 wgs->wgs_time_last_data_sent == 0) { 4049 /* 4050 * [W] 6.2 Transport Message Limits 4051 * "if a peer is the initiator of a current secure 4052 * session, WireGuard will send a handshake initiation 4053 * message to begin a new secure session if, after 4054 * transmitting a transport data message, the current 4055 * secure session is REKEY-AFTER-TIME seconds old," 4056 */ 4057 wg_schedule_rekey_timer(wgp); 4058 } 4059 wgs->wgs_time_last_data_sent = time_uptime; 4060 if (wg_session_get_send_counter(wgs) >= 4061 wg_rekey_after_messages) { 4062 /* 4063 * [W] 6.2 Transport Message Limits 4064 * "WireGuard will try to create a new session, by 4065 * sending a handshake initiation message (section 4066 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 4067 * transport data messages..." 4068 */ 4069 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 4070 } 4071 } 4072 end: 4073 m_freem(m); 4074 if (free_padded_buf) 4075 kmem_intr_free(padded_buf, padded_len); 4076 return error; 4077 } 4078 4079 static void 4080 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 4081 { 4082 pktqueue_t *pktq; 4083 size_t pktlen; 4084 4085 KASSERT(af == AF_INET || af == AF_INET6); 4086 4087 WG_TRACE(""); 4088 4089 m_set_rcvif(m, ifp); 4090 pktlen = m->m_pkthdr.len; 4091 4092 bpf_mtap_af(ifp, af, m, BPF_D_IN); 4093 4094 switch (af) { 4095 case AF_INET: 4096 pktq = ip_pktq; 4097 break; 4098 #ifdef INET6 4099 case AF_INET6: 4100 pktq = ip6_pktq; 4101 break; 4102 #endif 4103 default: 4104 panic("invalid af=%d", af); 4105 } 4106 4107 kpreempt_disable(); 4108 const u_int h = curcpu()->ci_index; 4109 if (__predict_true(pktq_enqueue(pktq, m, h))) { 4110 if_statadd(ifp, if_ibytes, pktlen); 4111 if_statinc(ifp, if_ipackets); 4112 } else { 4113 m_freem(m); 4114 } 4115 kpreempt_enable(); 4116 } 4117 4118 static void 4119 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 4120 const uint8_t privkey[WG_STATIC_KEY_LEN]) 4121 { 4122 4123 crypto_scalarmult_base(pubkey, privkey); 4124 } 4125 4126 static int 4127 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 4128 { 4129 struct radix_node_head *rnh; 4130 struct radix_node *rn; 4131 int error = 0; 4132 4133 rw_enter(wg->wg_rwlock, RW_WRITER); 4134 rnh = wg_rnh(wg, wga->wga_family); 4135 KASSERT(rnh != NULL); 4136 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 4137 wga->wga_nodes); 4138 rw_exit(wg->wg_rwlock); 4139 4140 if (rn == NULL) 4141 error = EEXIST; 4142 4143 return error; 4144 } 4145 4146 static int 4147 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 4148 struct wg_peer **wgpp) 4149 { 4150 int error = 0; 4151 const void *pubkey; 4152 size_t pubkey_len; 4153 const void *psk; 4154 size_t psk_len; 4155 const char *name = NULL; 4156 4157 if (prop_dictionary_get_string(peer, "name", &name)) { 4158 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 4159 error = EINVAL; 4160 goto out; 4161 } 4162 } 4163 4164 if (!prop_dictionary_get_data(peer, "public_key", 4165 &pubkey, &pubkey_len)) { 4166 error = EINVAL; 4167 goto out; 4168 } 4169 #ifdef WG_DEBUG_DUMP 4170 { 4171 char *hex = gethexdump(pubkey, pubkey_len); 4172 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n%s\n", 4173 pubkey, pubkey_len, hex); 4174 puthexdump(hex, pubkey, pubkey_len); 4175 } 4176 #endif 4177 4178 struct wg_peer *wgp = wg_alloc_peer(wg); 4179 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4180 if (name != NULL) 4181 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4182 4183 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4184 if (psk_len != sizeof(wgp->wgp_psk)) { 4185 error = EINVAL; 4186 goto out; 4187 } 4188 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4189 } 4190 4191 const void *addr; 4192 size_t addr_len; 4193 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4194 4195 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4196 goto skip_endpoint; 4197 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4198 addr_len > sizeof(*wgsatoss(wgsa))) { 4199 error = EINVAL; 4200 goto out; 4201 } 4202 memcpy(wgsatoss(wgsa), addr, addr_len); 4203 switch (wgsa_family(wgsa)) { 4204 case AF_INET: 4205 #ifdef INET6 4206 case AF_INET6: 4207 #endif 4208 break; 4209 default: 4210 error = EPFNOSUPPORT; 4211 goto out; 4212 } 4213 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4214 error = EINVAL; 4215 goto out; 4216 } 4217 { 4218 char addrstr[128]; 4219 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4220 WG_DLOG("addr=%s\n", addrstr); 4221 } 4222 wgp->wgp_endpoint_available = true; 4223 4224 prop_array_t allowedips; 4225 skip_endpoint: 4226 allowedips = prop_dictionary_get(peer, "allowedips"); 4227 if (allowedips == NULL) 4228 goto skip; 4229 4230 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4231 prop_dictionary_t prop_allowedip; 4232 int j = 0; 4233 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4234 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4235 4236 if (!prop_dictionary_get_int(prop_allowedip, "family", 4237 &wga->wga_family)) 4238 continue; 4239 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4240 &addr, &addr_len)) 4241 continue; 4242 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4243 &wga->wga_cidr)) 4244 continue; 4245 4246 switch (wga->wga_family) { 4247 case AF_INET: { 4248 struct sockaddr_in sin; 4249 char addrstr[128]; 4250 struct in_addr mask; 4251 struct sockaddr_in sin_mask; 4252 4253 if (addr_len != sizeof(struct in_addr)) 4254 return EINVAL; 4255 memcpy(&wga->wga_addr4, addr, addr_len); 4256 4257 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4258 0); 4259 sockaddr_copy(&wga->wga_sa_addr, 4260 sizeof(sin), sintosa(&sin)); 4261 4262 sockaddr_format(sintosa(&sin), 4263 addrstr, sizeof(addrstr)); 4264 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4265 4266 in_len2mask(&mask, wga->wga_cidr); 4267 sockaddr_in_init(&sin_mask, &mask, 0); 4268 sockaddr_copy(&wga->wga_sa_mask, 4269 sizeof(sin_mask), sintosa(&sin_mask)); 4270 4271 break; 4272 } 4273 #ifdef INET6 4274 case AF_INET6: { 4275 struct sockaddr_in6 sin6; 4276 char addrstr[128]; 4277 struct in6_addr mask; 4278 struct sockaddr_in6 sin6_mask; 4279 4280 if (addr_len != sizeof(struct in6_addr)) 4281 return EINVAL; 4282 memcpy(&wga->wga_addr6, addr, addr_len); 4283 4284 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4285 0, 0, 0); 4286 sockaddr_copy(&wga->wga_sa_addr, 4287 sizeof(sin6), sin6tosa(&sin6)); 4288 4289 sockaddr_format(sin6tosa(&sin6), 4290 addrstr, sizeof(addrstr)); 4291 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4292 4293 in6_prefixlen2mask(&mask, wga->wga_cidr); 4294 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4295 sockaddr_copy(&wga->wga_sa_mask, 4296 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4297 4298 break; 4299 } 4300 #endif 4301 default: 4302 error = EINVAL; 4303 goto out; 4304 } 4305 wga->wga_peer = wgp; 4306 4307 error = wg_rtable_add_route(wg, wga); 4308 if (error != 0) 4309 goto out; 4310 4311 j++; 4312 } 4313 wgp->wgp_n_allowedips = j; 4314 skip: 4315 *wgpp = wgp; 4316 out: 4317 return error; 4318 } 4319 4320 static int 4321 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4322 { 4323 int error; 4324 char *buf; 4325 4326 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4327 if (ifd->ifd_len >= WG_MAX_PROPLEN) 4328 return E2BIG; 4329 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4330 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4331 if (error != 0) 4332 return error; 4333 buf[ifd->ifd_len] = '\0'; 4334 #ifdef WG_DEBUG_DUMP 4335 log(LOG_DEBUG, "%.*s\n", 4336 (int)MIN(INT_MAX, ifd->ifd_len), 4337 (const char *)buf); 4338 #endif 4339 *_buf = buf; 4340 return 0; 4341 } 4342 4343 static int 4344 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4345 { 4346 int error; 4347 prop_dictionary_t prop_dict; 4348 char *buf = NULL; 4349 const void *privkey; 4350 size_t privkey_len; 4351 4352 error = wg_alloc_prop_buf(&buf, ifd); 4353 if (error != 0) 4354 return error; 4355 error = EINVAL; 4356 prop_dict = prop_dictionary_internalize(buf); 4357 if (prop_dict == NULL) 4358 goto out; 4359 if (!prop_dictionary_get_data(prop_dict, "private_key", 4360 &privkey, &privkey_len)) 4361 goto out; 4362 #ifdef WG_DEBUG_DUMP 4363 { 4364 char *hex = gethexdump(privkey, privkey_len); 4365 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n%s\n", 4366 privkey, privkey_len, hex); 4367 puthexdump(hex, privkey, privkey_len); 4368 } 4369 #endif 4370 if (privkey_len != WG_STATIC_KEY_LEN) 4371 goto out; 4372 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4373 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4374 error = 0; 4375 4376 out: 4377 kmem_free(buf, ifd->ifd_len + 1); 4378 return error; 4379 } 4380 4381 static int 4382 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4383 { 4384 int error; 4385 prop_dictionary_t prop_dict; 4386 char *buf = NULL; 4387 uint16_t port; 4388 4389 error = wg_alloc_prop_buf(&buf, ifd); 4390 if (error != 0) 4391 return error; 4392 error = EINVAL; 4393 prop_dict = prop_dictionary_internalize(buf); 4394 if (prop_dict == NULL) 4395 goto out; 4396 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4397 goto out; 4398 4399 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4400 4401 out: 4402 kmem_free(buf, ifd->ifd_len + 1); 4403 return error; 4404 } 4405 4406 static int 4407 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4408 { 4409 int error; 4410 prop_dictionary_t prop_dict; 4411 char *buf = NULL; 4412 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4413 4414 error = wg_alloc_prop_buf(&buf, ifd); 4415 if (error != 0) 4416 return error; 4417 error = EINVAL; 4418 prop_dict = prop_dictionary_internalize(buf); 4419 if (prop_dict == NULL) 4420 goto out; 4421 4422 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4423 if (error != 0) 4424 goto out; 4425 4426 mutex_enter(wg->wg_lock); 4427 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4428 sizeof(wgp->wgp_pubkey)) != NULL || 4429 (wgp->wgp_name[0] && 4430 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4431 strlen(wgp->wgp_name)) != NULL)) { 4432 mutex_exit(wg->wg_lock); 4433 wg_destroy_peer(wgp); 4434 error = EEXIST; 4435 goto out; 4436 } 4437 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4438 sizeof(wgp->wgp_pubkey), wgp); 4439 KASSERT(wgp0 == wgp); 4440 if (wgp->wgp_name[0]) { 4441 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4442 strlen(wgp->wgp_name), wgp); 4443 KASSERT(wgp0 == wgp); 4444 } 4445 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4446 wg->wg_npeers++; 4447 mutex_exit(wg->wg_lock); 4448 4449 if_link_state_change(&wg->wg_if, LINK_STATE_UP); 4450 4451 out: 4452 kmem_free(buf, ifd->ifd_len + 1); 4453 return error; 4454 } 4455 4456 static int 4457 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4458 { 4459 int error; 4460 prop_dictionary_t prop_dict; 4461 char *buf = NULL; 4462 const char *name; 4463 4464 error = wg_alloc_prop_buf(&buf, ifd); 4465 if (error != 0) 4466 return error; 4467 error = EINVAL; 4468 prop_dict = prop_dictionary_internalize(buf); 4469 if (prop_dict == NULL) 4470 goto out; 4471 4472 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4473 goto out; 4474 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4475 goto out; 4476 4477 error = wg_destroy_peer_name(wg, name); 4478 out: 4479 kmem_free(buf, ifd->ifd_len + 1); 4480 return error; 4481 } 4482 4483 static bool 4484 wg_is_authorized(struct wg_softc *wg, u_long cmd) 4485 { 4486 int au = cmd == SIOCGDRVSPEC ? 4487 KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV : 4488 KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV; 4489 return kauth_authorize_network(kauth_cred_get(), 4490 KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if, 4491 (void *)cmd, NULL) == 0; 4492 } 4493 4494 static int 4495 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4496 { 4497 int error = ENOMEM; 4498 prop_dictionary_t prop_dict; 4499 prop_array_t peers = NULL; 4500 char *buf; 4501 struct wg_peer *wgp; 4502 int s, i; 4503 4504 prop_dict = prop_dictionary_create(); 4505 if (prop_dict == NULL) 4506 goto error; 4507 4508 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4509 if (!prop_dictionary_set_data(prop_dict, "private_key", 4510 wg->wg_privkey, WG_STATIC_KEY_LEN)) 4511 goto error; 4512 } 4513 4514 if (wg->wg_listen_port != 0) { 4515 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4516 wg->wg_listen_port)) 4517 goto error; 4518 } 4519 4520 if (wg->wg_npeers == 0) 4521 goto skip_peers; 4522 4523 peers = prop_array_create(); 4524 if (peers == NULL) 4525 goto error; 4526 4527 s = pserialize_read_enter(); 4528 i = 0; 4529 WG_PEER_READER_FOREACH(wgp, wg) { 4530 struct wg_sockaddr *wgsa; 4531 struct psref wgp_psref, wgsa_psref; 4532 prop_dictionary_t prop_peer; 4533 4534 wg_get_peer(wgp, &wgp_psref); 4535 pserialize_read_exit(s); 4536 4537 prop_peer = prop_dictionary_create(); 4538 if (prop_peer == NULL) 4539 goto next; 4540 4541 if (strlen(wgp->wgp_name) > 0) { 4542 if (!prop_dictionary_set_string(prop_peer, "name", 4543 wgp->wgp_name)) 4544 goto next; 4545 } 4546 4547 if (!prop_dictionary_set_data(prop_peer, "public_key", 4548 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4549 goto next; 4550 4551 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4552 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4553 sizeof(wgp->wgp_psk))) { 4554 if (wg_is_authorized(wg, SIOCGDRVSPEC)) { 4555 if (!prop_dictionary_set_data(prop_peer, 4556 "preshared_key", 4557 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4558 goto next; 4559 } 4560 } 4561 4562 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4563 CTASSERT(AF_UNSPEC == 0); 4564 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4565 !prop_dictionary_set_data(prop_peer, "endpoint", 4566 wgsatoss(wgsa), 4567 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4568 wg_put_sa(wgp, wgsa, &wgsa_psref); 4569 goto next; 4570 } 4571 wg_put_sa(wgp, wgsa, &wgsa_psref); 4572 4573 const struct timespec *t = &wgp->wgp_last_handshake_time; 4574 4575 if (!prop_dictionary_set_uint64(prop_peer, 4576 "last_handshake_time_sec", (uint64_t)t->tv_sec)) 4577 goto next; 4578 if (!prop_dictionary_set_uint32(prop_peer, 4579 "last_handshake_time_nsec", (uint32_t)t->tv_nsec)) 4580 goto next; 4581 4582 if (wgp->wgp_n_allowedips == 0) 4583 goto skip_allowedips; 4584 4585 prop_array_t allowedips = prop_array_create(); 4586 if (allowedips == NULL) 4587 goto next; 4588 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4589 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4590 prop_dictionary_t prop_allowedip; 4591 4592 prop_allowedip = prop_dictionary_create(); 4593 if (prop_allowedip == NULL) 4594 break; 4595 4596 if (!prop_dictionary_set_int(prop_allowedip, "family", 4597 wga->wga_family)) 4598 goto _next; 4599 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4600 wga->wga_cidr)) 4601 goto _next; 4602 4603 switch (wga->wga_family) { 4604 case AF_INET: 4605 if (!prop_dictionary_set_data(prop_allowedip, 4606 "ip", &wga->wga_addr4, 4607 sizeof(wga->wga_addr4))) 4608 goto _next; 4609 break; 4610 #ifdef INET6 4611 case AF_INET6: 4612 if (!prop_dictionary_set_data(prop_allowedip, 4613 "ip", &wga->wga_addr6, 4614 sizeof(wga->wga_addr6))) 4615 goto _next; 4616 break; 4617 #endif 4618 default: 4619 break; 4620 } 4621 prop_array_set(allowedips, j, prop_allowedip); 4622 _next: 4623 prop_object_release(prop_allowedip); 4624 } 4625 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4626 prop_object_release(allowedips); 4627 4628 skip_allowedips: 4629 4630 prop_array_set(peers, i, prop_peer); 4631 next: 4632 if (prop_peer) 4633 prop_object_release(prop_peer); 4634 i++; 4635 4636 s = pserialize_read_enter(); 4637 wg_put_peer(wgp, &wgp_psref); 4638 } 4639 pserialize_read_exit(s); 4640 4641 prop_dictionary_set(prop_dict, "peers", peers); 4642 prop_object_release(peers); 4643 peers = NULL; 4644 4645 skip_peers: 4646 buf = prop_dictionary_externalize(prop_dict); 4647 if (buf == NULL) 4648 goto error; 4649 if (ifd->ifd_len < (strlen(buf) + 1)) { 4650 error = EINVAL; 4651 goto error; 4652 } 4653 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4654 4655 free(buf, 0); 4656 error: 4657 if (peers != NULL) 4658 prop_object_release(peers); 4659 if (prop_dict != NULL) 4660 prop_object_release(prop_dict); 4661 4662 return error; 4663 } 4664 4665 static int 4666 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4667 { 4668 struct wg_softc *wg = ifp->if_softc; 4669 struct ifreq *ifr = data; 4670 struct ifaddr *ifa = data; 4671 struct ifdrv *ifd = data; 4672 int error = 0; 4673 4674 switch (cmd) { 4675 case SIOCINITIFADDR: 4676 if (ifa->ifa_addr->sa_family != AF_LINK && 4677 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4678 (IFF_UP | IFF_RUNNING)) { 4679 ifp->if_flags |= IFF_UP; 4680 error = if_init(ifp); 4681 } 4682 return error; 4683 case SIOCADDMULTI: 4684 case SIOCDELMULTI: 4685 switch (ifr->ifr_addr.sa_family) { 4686 case AF_INET: /* IP supports Multicast */ 4687 break; 4688 #ifdef INET6 4689 case AF_INET6: /* IP6 supports Multicast */ 4690 break; 4691 #endif 4692 default: /* Other protocols doesn't support Multicast */ 4693 error = EAFNOSUPPORT; 4694 break; 4695 } 4696 return error; 4697 case SIOCSDRVSPEC: 4698 if (!wg_is_authorized(wg, cmd)) { 4699 return EPERM; 4700 } 4701 switch (ifd->ifd_cmd) { 4702 case WG_IOCTL_SET_PRIVATE_KEY: 4703 error = wg_ioctl_set_private_key(wg, ifd); 4704 break; 4705 case WG_IOCTL_SET_LISTEN_PORT: 4706 error = wg_ioctl_set_listen_port(wg, ifd); 4707 break; 4708 case WG_IOCTL_ADD_PEER: 4709 error = wg_ioctl_add_peer(wg, ifd); 4710 break; 4711 case WG_IOCTL_DELETE_PEER: 4712 error = wg_ioctl_delete_peer(wg, ifd); 4713 break; 4714 default: 4715 error = EINVAL; 4716 break; 4717 } 4718 return error; 4719 case SIOCGDRVSPEC: 4720 return wg_ioctl_get(wg, ifd); 4721 case SIOCSIFFLAGS: 4722 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4723 break; 4724 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4725 case IFF_RUNNING: 4726 /* 4727 * If interface is marked down and it is running, 4728 * then stop and disable it. 4729 */ 4730 if_stop(ifp, 1); 4731 break; 4732 case IFF_UP: 4733 /* 4734 * If interface is marked up and it is stopped, then 4735 * start it. 4736 */ 4737 error = if_init(ifp); 4738 break; 4739 default: 4740 break; 4741 } 4742 return error; 4743 #ifdef WG_RUMPKERNEL 4744 case SIOCSLINKSTR: 4745 error = wg_ioctl_linkstr(wg, ifd); 4746 if (error == 0) 4747 wg->wg_ops = &wg_ops_rumpuser; 4748 return error; 4749 #endif 4750 default: 4751 break; 4752 } 4753 4754 error = ifioctl_common(ifp, cmd, data); 4755 4756 #ifdef WG_RUMPKERNEL 4757 if (!wg_user_mode(wg)) 4758 return error; 4759 4760 /* Do the same to the corresponding tun device on the host */ 4761 /* 4762 * XXX Actually the command has not been handled yet. It 4763 * will be handled via pr_ioctl form doifioctl later. 4764 */ 4765 switch (cmd) { 4766 case SIOCAIFADDR: 4767 case SIOCDIFADDR: { 4768 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4769 struct in_aliasreq *ifra = &_ifra; 4770 KASSERT(error == ENOTTY); 4771 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4772 IFNAMSIZ); 4773 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4774 if (error == 0) 4775 error = ENOTTY; 4776 break; 4777 } 4778 #ifdef INET6 4779 case SIOCAIFADDR_IN6: 4780 case SIOCDIFADDR_IN6: { 4781 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4782 struct in6_aliasreq *ifra = &_ifra; 4783 KASSERT(error == ENOTTY); 4784 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4785 IFNAMSIZ); 4786 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4787 if (error == 0) 4788 error = ENOTTY; 4789 break; 4790 } 4791 #endif 4792 } 4793 #endif /* WG_RUMPKERNEL */ 4794 4795 return error; 4796 } 4797 4798 static int 4799 wg_init(struct ifnet *ifp) 4800 { 4801 4802 ifp->if_flags |= IFF_RUNNING; 4803 4804 /* TODO flush pending packets. */ 4805 return 0; 4806 } 4807 4808 #ifdef ALTQ 4809 static void 4810 wg_start(struct ifnet *ifp) 4811 { 4812 struct mbuf *m; 4813 4814 for (;;) { 4815 IFQ_DEQUEUE(&ifp->if_snd, m); 4816 if (m == NULL) 4817 break; 4818 4819 kpreempt_disable(); 4820 const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m) 4821 if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) { 4822 WGLOG(LOG_ERR, "%s: pktq full, dropping\n", 4823 if_name(ifp)); 4824 m_freem(m); 4825 } 4826 kpreempt_enable(); 4827 } 4828 } 4829 #endif 4830 4831 static void 4832 wg_stop(struct ifnet *ifp, int disable) 4833 { 4834 4835 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4836 ifp->if_flags &= ~IFF_RUNNING; 4837 4838 /* Need to do something? */ 4839 } 4840 4841 #ifdef WG_DEBUG_PARAMS 4842 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4843 { 4844 const struct sysctlnode *node = NULL; 4845 4846 sysctl_createv(clog, 0, NULL, &node, 4847 CTLFLAG_PERMANENT, 4848 CTLTYPE_NODE, "wg", 4849 SYSCTL_DESCR("wg(4)"), 4850 NULL, 0, NULL, 0, 4851 CTL_NET, CTL_CREATE, CTL_EOL); 4852 sysctl_createv(clog, 0, &node, NULL, 4853 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4854 CTLTYPE_QUAD, "rekey_after_messages", 4855 SYSCTL_DESCR("session liftime by messages"), 4856 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4857 sysctl_createv(clog, 0, &node, NULL, 4858 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4859 CTLTYPE_INT, "rekey_after_time", 4860 SYSCTL_DESCR("session liftime"), 4861 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4862 sysctl_createv(clog, 0, &node, NULL, 4863 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4864 CTLTYPE_INT, "rekey_timeout", 4865 SYSCTL_DESCR("session handshake retry time"), 4866 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4867 sysctl_createv(clog, 0, &node, NULL, 4868 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4869 CTLTYPE_INT, "rekey_attempt_time", 4870 SYSCTL_DESCR("session handshake timeout"), 4871 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4872 sysctl_createv(clog, 0, &node, NULL, 4873 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4874 CTLTYPE_INT, "keepalive_timeout", 4875 SYSCTL_DESCR("keepalive timeout"), 4876 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4877 sysctl_createv(clog, 0, &node, NULL, 4878 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4879 CTLTYPE_BOOL, "force_underload", 4880 SYSCTL_DESCR("force to detemine under load"), 4881 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4882 } 4883 #endif 4884 4885 #ifdef WG_RUMPKERNEL 4886 static bool 4887 wg_user_mode(struct wg_softc *wg) 4888 { 4889 4890 return wg->wg_user != NULL; 4891 } 4892 4893 static int 4894 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4895 { 4896 struct ifnet *ifp = &wg->wg_if; 4897 int error; 4898 4899 if (ifp->if_flags & IFF_UP) 4900 return EBUSY; 4901 4902 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4903 /* XXX do nothing */ 4904 return 0; 4905 } else if (ifd->ifd_cmd != 0) { 4906 return EINVAL; 4907 } else if (wg->wg_user != NULL) { 4908 return EBUSY; 4909 } 4910 4911 /* Assume \0 included */ 4912 if (ifd->ifd_len > IFNAMSIZ) { 4913 return E2BIG; 4914 } else if (ifd->ifd_len < 1) { 4915 return EINVAL; 4916 } 4917 4918 char tun_name[IFNAMSIZ]; 4919 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4920 if (error != 0) 4921 return error; 4922 4923 if (strncmp(tun_name, "tun", 3) != 0) 4924 return EINVAL; 4925 4926 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 4927 4928 return error; 4929 } 4930 4931 static int 4932 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 4933 { 4934 int error; 4935 struct psref psref; 4936 struct wg_sockaddr *wgsa; 4937 struct wg_softc *wg = wgp->wgp_sc; 4938 struct iovec iov[1]; 4939 4940 wgsa = wg_get_endpoint_sa(wgp, &psref); 4941 4942 iov[0].iov_base = mtod(m, void *); 4943 iov[0].iov_len = m->m_len; 4944 4945 /* Send messages to a peer via an ordinary socket. */ 4946 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 4947 4948 wg_put_sa(wgp, wgsa, &psref); 4949 4950 m_freem(m); 4951 4952 return error; 4953 } 4954 4955 static void 4956 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 4957 { 4958 struct wg_softc *wg = ifp->if_softc; 4959 struct iovec iov[2]; 4960 struct sockaddr_storage ss; 4961 4962 KASSERT(af == AF_INET || af == AF_INET6); 4963 4964 WG_TRACE(""); 4965 4966 if (af == AF_INET) { 4967 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 4968 struct ip *ip; 4969 4970 KASSERT(m->m_len >= sizeof(struct ip)); 4971 ip = mtod(m, struct ip *); 4972 sockaddr_in_init(sin, &ip->ip_dst, 0); 4973 } else { 4974 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 4975 struct ip6_hdr *ip6; 4976 4977 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 4978 ip6 = mtod(m, struct ip6_hdr *); 4979 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 4980 } 4981 4982 iov[0].iov_base = &ss; 4983 iov[0].iov_len = ss.ss_len; 4984 iov[1].iov_base = mtod(m, void *); 4985 iov[1].iov_len = m->m_len; 4986 4987 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4988 4989 /* Send decrypted packets to users via a tun. */ 4990 rumpuser_wg_send_user(wg->wg_user, iov, 2); 4991 4992 m_freem(m); 4993 } 4994 4995 static int 4996 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 4997 { 4998 int error; 4999 uint16_t old_port = wg->wg_listen_port; 5000 5001 if (port != 0 && old_port == port) 5002 return 0; 5003 5004 error = rumpuser_wg_sock_bind(wg->wg_user, port); 5005 if (error == 0) 5006 wg->wg_listen_port = port; 5007 return error; 5008 } 5009 5010 /* 5011 * Receive user packets. 5012 */ 5013 void 5014 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5015 { 5016 struct ifnet *ifp = &wg->wg_if; 5017 struct mbuf *m; 5018 const struct sockaddr *dst; 5019 5020 WG_TRACE(""); 5021 5022 dst = iov[0].iov_base; 5023 5024 m = m_gethdr(M_DONTWAIT, MT_DATA); 5025 if (m == NULL) 5026 return; 5027 m->m_len = m->m_pkthdr.len = 0; 5028 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5029 5030 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5031 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5032 5033 (void)wg_output(ifp, m, dst, NULL); 5034 } 5035 5036 /* 5037 * Receive packets from a peer. 5038 */ 5039 void 5040 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 5041 { 5042 struct mbuf *m; 5043 const struct sockaddr *src; 5044 int bound; 5045 5046 WG_TRACE(""); 5047 5048 src = iov[0].iov_base; 5049 5050 m = m_gethdr(M_DONTWAIT, MT_DATA); 5051 if (m == NULL) 5052 return; 5053 m->m_len = m->m_pkthdr.len = 0; 5054 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 5055 5056 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 5057 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 5058 5059 bound = curlwp_bind(); 5060 wg_handle_packet(wg, m, src); 5061 curlwp_bindx(bound); 5062 } 5063 #endif /* WG_RUMPKERNEL */ 5064 5065 /* 5066 * Module infrastructure 5067 */ 5068 #include "if_module.h" 5069 5070 IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s") 5071
Indexes created Wed Oct 15 07:09:58 GMT 2025