if_wg.c revision 1.50
1 /* $NetBSD: if_wg.c,v 1.50 2020/08/31 20:34:18 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.50 2020/08/31 20:34:18 riastradh Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_inet.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/types.h> 52 53 #include <sys/atomic.h> 54 #include <sys/callout.h> 55 #include <sys/cprng.h> 56 #include <sys/cpu.h> 57 #include <sys/device.h> 58 #include <sys/domain.h> 59 #include <sys/errno.h> 60 #include <sys/intr.h> 61 #include <sys/ioctl.h> 62 #include <sys/kernel.h> 63 #include <sys/kmem.h> 64 #include <sys/kthread.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/pcq.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/time.h> 82 #include <sys/timespec.h> 83 84 #include <net/bpf.h> 85 #include <net/if.h> 86 #include <net/if_types.h> 87 #include <net/if_wg.h> 88 #include <net/route.h> 89 90 #include <netinet/in.h> 91 #include <netinet/in_pcb.h> 92 #include <netinet/in_var.h> 93 #include <netinet/ip.h> 94 #include <netinet/ip_var.h> 95 #include <netinet/udp.h> 96 #include <netinet/udp_var.h> 97 98 #ifdef INET6 99 #include <netinet/ip6.h> 100 #include <netinet6/in6_pcb.h> 101 #include <netinet6/in6_var.h> 102 #include <netinet6/ip6_var.h> 103 #include <netinet6/udp6_var.h> 104 #endif /* INET6 */ 105 106 #include <prop/proplib.h> 107 108 #include <crypto/blake2/blake2s.h> 109 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 110 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 111 #include <crypto/sodium/crypto_scalarmult.h> 112 113 #include "ioconf.h" 114 115 #ifdef WG_RUMPKERNEL 116 #include "wg_user.h" 117 #endif 118 119 /* 120 * Data structures 121 * - struct wg_softc is an instance of wg interfaces 122 * - It has a list of peers (struct wg_peer) 123 * - It has a kthread that sends/receives handshake messages and 124 * runs event handlers 125 * - It has its own two routing tables: one is for IPv4 and the other IPv6 126 * - struct wg_peer is a representative of a peer 127 * - It has a softint that is used to send packets over an wg interface 128 * to a peer 129 * - It has a pair of session instances (struct wg_session) 130 * - It has a pair of endpoint instances (struct wg_sockaddr) 131 * - Normally one endpoint is used and the second one is used only on 132 * a peer migration (a change of peer's IP address) 133 * - It has a list of IP addresses and sub networks called allowedips 134 * (struct wg_allowedip) 135 * - A packets sent over a session is allowed if its destination matches 136 * any IP addresses or sub networks of the list 137 * - struct wg_session represents a session of a secure tunnel with a peer 138 * - Two instances of sessions belong to a peer; a stable session and a 139 * unstable session 140 * - A handshake process of a session always starts with a unstable instance 141 * - Once a session is established, its instance becomes stable and the 142 * other becomes unstable instead 143 * - Data messages are always sent via a stable session 144 * 145 * Locking notes: 146 * - wg interfaces (struct wg_softc, wg) is listed in wg_softcs.list and 147 * protected by wg_softcs.lock 148 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 149 * - Changes to the peer list are serialized by wg_lock 150 * - The peer list may be read with pserialize(9) and psref(9) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 152 * => XXX replace by pserialize when routing table is psz-safe 153 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 154 * only in thread context and serializes: 155 * - the stable and unstable session pointers 156 * - all unstable session state 157 * - Packet processing may be done in softint context: 158 * - The stable session can be read under pserialize(9) or psref(9) 159 * - The stable session is always ESTABLISHED 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 * - Lock order: wg_lock -> wgp_lock 164 */ 165 166 167 #define WGLOG(level, fmt, args...) \ 168 log(level, "%s: " fmt, __func__, ##args) 169 170 /* Debug options */ 171 #ifdef WG_DEBUG 172 /* Output debug logs */ 173 #ifndef WG_DEBUG_LOG 174 #define WG_DEBUG_LOG 175 #endif 176 /* Output trace logs */ 177 #ifndef WG_DEBUG_TRACE 178 #define WG_DEBUG_TRACE 179 #endif 180 /* Output hash values, etc. */ 181 #ifndef WG_DEBUG_DUMP 182 #define WG_DEBUG_DUMP 183 #endif 184 /* Make some internal parameters configurable for testing and debugging */ 185 #ifndef WG_DEBUG_PARAMS 186 #define WG_DEBUG_PARAMS 187 #endif 188 #endif 189 190 #ifdef WG_DEBUG_TRACE 191 #define WG_TRACE(msg) \ 192 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)) 193 #else 194 #define WG_TRACE(msg) __nothing 195 #endif 196 197 #ifdef WG_DEBUG_LOG 198 #define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args) 199 #else 200 #define WG_DLOG(fmt, args...) __nothing 201 #endif 202 203 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 204 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 205 &(wgprc)->wgprc_curpps, 1)) { \ 206 log(level, fmt, ##args); \ 207 } \ 208 } while (0) 209 210 #ifdef WG_DEBUG_PARAMS 211 static bool wg_force_underload = false; 212 #endif 213 214 #ifdef WG_DEBUG_DUMP 215 216 #ifdef WG_RUMPKERNEL 217 static void 218 wg_dump_buf(const char *func, const char *buf, const size_t size) 219 { 220 221 log(LOG_DEBUG, "%s: ", func); 222 for (int i = 0; i < size; i++) 223 log(LOG_DEBUG, "%02x ", (int)(0xff & buf[i])); 224 log(LOG_DEBUG, "\n"); 225 } 226 #endif 227 228 static void 229 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 230 const size_t size) 231 { 232 233 log(LOG_DEBUG, "%s: %s: ", func, name); 234 for (int i = 0; i < size; i++) 235 log(LOG_DEBUG, "%02x ", (int)(0xff & hash[i])); 236 log(LOG_DEBUG, "\n"); 237 } 238 239 #define WG_DUMP_HASH(name, hash) \ 240 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 241 #define WG_DUMP_HASH48(name, hash) \ 242 wg_dump_hash(__func__, name, hash, 48) 243 #define WG_DUMP_BUF(buf, size) \ 244 wg_dump_buf(__func__, buf, size) 245 #else 246 #define WG_DUMP_HASH(name, hash) __nothing 247 #define WG_DUMP_HASH48(name, hash) __nothing 248 #define WG_DUMP_BUF(buf, size) __nothing 249 #endif /* WG_DEBUG_DUMP */ 250 251 #define WG_MTU 1420 252 #define WG_ALLOWEDIPS 16 253 254 #define CURVE25519_KEY_LEN 32 255 #define TAI64N_LEN sizeof(uint32_t) * 3 256 #define POLY1305_AUTHTAG_LEN 16 257 #define HMAC_BLOCK_LEN 64 258 259 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 260 /* [N] 4.3: Hash functions */ 261 #define NOISE_DHLEN 32 262 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 263 #define NOISE_HASHLEN 32 264 #define NOISE_BLOCKLEN 64 265 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 266 /* [N] 5.1: "k" */ 267 #define NOISE_CIPHER_KEY_LEN 32 268 /* 269 * [N] 9.2: "psk" 270 * "... psk is a 32-byte secret value provided by the application." 271 */ 272 #define NOISE_PRESHARED_KEY_LEN 32 273 274 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 275 #define WG_TIMESTAMP_LEN TAI64N_LEN 276 277 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 278 279 #define WG_COOKIE_LEN 16 280 #define WG_MAC_LEN 16 281 #define WG_RANDVAL_LEN 24 282 283 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 284 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 285 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 286 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 287 #define WG_HASH_LEN NOISE_HASHLEN 288 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 289 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 290 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 291 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 292 #define WG_DATA_KEY_LEN 32 293 #define WG_SALT_LEN 24 294 295 /* 296 * The protocol messages 297 */ 298 struct wg_msg { 299 uint32_t wgm_type; 300 } __packed; 301 302 /* [W] 5.4.2 First Message: Initiator to Responder */ 303 struct wg_msg_init { 304 uint32_t wgmi_type; 305 uint32_t wgmi_sender; 306 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 307 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 308 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 309 uint8_t wgmi_mac1[WG_MAC_LEN]; 310 uint8_t wgmi_mac2[WG_MAC_LEN]; 311 } __packed; 312 313 /* [W] 5.4.3 Second Message: Responder to Initiator */ 314 struct wg_msg_resp { 315 uint32_t wgmr_type; 316 uint32_t wgmr_sender; 317 uint32_t wgmr_receiver; 318 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 319 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 320 uint8_t wgmr_mac1[WG_MAC_LEN]; 321 uint8_t wgmr_mac2[WG_MAC_LEN]; 322 } __packed; 323 324 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 325 struct wg_msg_data { 326 uint32_t wgmd_type; 327 uint32_t wgmd_receiver; 328 uint64_t wgmd_counter; 329 uint32_t wgmd_packet[0]; 330 } __packed; 331 332 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 333 struct wg_msg_cookie { 334 uint32_t wgmc_type; 335 uint32_t wgmc_receiver; 336 uint8_t wgmc_salt[WG_SALT_LEN]; 337 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 338 } __packed; 339 340 #define WG_MSG_TYPE_INIT 1 341 #define WG_MSG_TYPE_RESP 2 342 #define WG_MSG_TYPE_COOKIE 3 343 #define WG_MSG_TYPE_DATA 4 344 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 345 346 /* Sliding windows */ 347 348 #define SLIWIN_BITS 2048u 349 #define SLIWIN_TYPE uint32_t 350 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 351 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 352 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 353 354 struct sliwin { 355 SLIWIN_TYPE B[SLIWIN_WORDS]; 356 uint64_t T; 357 }; 358 359 static void 360 sliwin_reset(struct sliwin *W) 361 { 362 363 memset(W, 0, sizeof(*W)); 364 } 365 366 static int 367 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 368 { 369 370 /* 371 * If it's more than one window older than the highest sequence 372 * number we've seen, reject. 373 */ 374 #ifdef __HAVE_ATOMIC64_LOADSTORE 375 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 376 return EAUTH; 377 #endif 378 379 /* 380 * Otherwise, we need to take the lock to decide, so don't 381 * reject just yet. Caller must serialize a call to 382 * sliwin_update in this case. 383 */ 384 return 0; 385 } 386 387 static int 388 sliwin_update(struct sliwin *W, uint64_t S) 389 { 390 unsigned word, bit; 391 392 /* 393 * If it's more than one window older than the highest sequence 394 * number we've seen, reject. 395 */ 396 if (S + SLIWIN_NPKT < W->T) 397 return EAUTH; 398 399 /* 400 * If it's higher than the highest sequence number we've seen, 401 * advance the window. 402 */ 403 if (S > W->T) { 404 uint64_t i = W->T / SLIWIN_BPW; 405 uint64_t j = S / SLIWIN_BPW; 406 unsigned k; 407 408 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 409 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 410 #ifdef __HAVE_ATOMIC64_LOADSTORE 411 atomic_store_relaxed(&W->T, S); 412 #else 413 W->T = S; 414 #endif 415 } 416 417 /* Test and set the bit -- if already set, reject. */ 418 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 419 bit = S % SLIWIN_BPW; 420 if (W->B[word] & (1UL << bit)) 421 return EAUTH; 422 W->B[word] |= 1UL << bit; 423 424 /* Accept! */ 425 return 0; 426 } 427 428 struct wg_worker { 429 kmutex_t wgw_lock; 430 kcondvar_t wgw_cv; 431 bool wgw_todie; 432 struct socket *wgw_so4; 433 struct socket *wgw_so6; 434 int wgw_wakeup_reasons; 435 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 __BIT(0) 436 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6 __BIT(1) 437 #define WG_WAKEUP_REASON_PEER __BIT(2) 438 }; 439 440 struct wg_session { 441 struct wg_peer *wgs_peer; 442 struct psref_target 443 wgs_psref; 444 445 int wgs_state; 446 #define WGS_STATE_UNKNOWN 0 447 #define WGS_STATE_INIT_ACTIVE 1 448 #define WGS_STATE_INIT_PASSIVE 2 449 #define WGS_STATE_ESTABLISHED 3 450 #define WGS_STATE_DESTROYING 4 451 452 time_t wgs_time_established; 453 time_t wgs_time_last_data_sent; 454 bool wgs_is_initiator; 455 456 uint32_t wgs_local_index; 457 uint32_t wgs_remote_index; 458 #ifdef __HAVE_ATOMIC64_LOADSTORE 459 volatile uint64_t 460 wgs_send_counter; 461 #else 462 kmutex_t wgs_send_counter_lock; 463 uint64_t wgs_send_counter; 464 #endif 465 466 struct { 467 kmutex_t lock; 468 struct sliwin window; 469 } *wgs_recvwin; 470 471 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 472 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 473 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 474 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 475 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 476 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 477 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 478 }; 479 480 struct wg_sockaddr { 481 union { 482 struct sockaddr_storage _ss; 483 struct sockaddr _sa; 484 struct sockaddr_in _sin; 485 struct sockaddr_in6 _sin6; 486 }; 487 struct psref_target wgsa_psref; 488 }; 489 490 #define wgsatoss(wgsa) (&(wgsa)->_ss) 491 #define wgsatosa(wgsa) (&(wgsa)->_sa) 492 #define wgsatosin(wgsa) (&(wgsa)->_sin) 493 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 494 495 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 496 497 struct wg_peer; 498 struct wg_allowedip { 499 struct radix_node wga_nodes[2]; 500 struct wg_sockaddr _wga_sa_addr; 501 struct wg_sockaddr _wga_sa_mask; 502 #define wga_sa_addr _wga_sa_addr._sa 503 #define wga_sa_mask _wga_sa_mask._sa 504 505 int wga_family; 506 uint8_t wga_cidr; 507 union { 508 struct in_addr _ip4; 509 struct in6_addr _ip6; 510 } wga_addr; 511 #define wga_addr4 wga_addr._ip4 512 #define wga_addr6 wga_addr._ip6 513 514 struct wg_peer *wga_peer; 515 }; 516 517 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 518 519 struct wg_ppsratecheck { 520 struct timeval wgprc_lasttime; 521 int wgprc_curpps; 522 }; 523 524 struct wg_softc; 525 struct wg_peer { 526 struct wg_softc *wgp_sc; 527 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 528 struct pslist_entry wgp_peerlist_entry; 529 pserialize_t wgp_psz; 530 struct psref_target wgp_psref; 531 kmutex_t *wgp_lock; 532 533 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 534 struct wg_sockaddr *wgp_endpoint; 535 struct wg_sockaddr *wgp_endpoint0; 536 volatile unsigned wgp_endpoint_changing; 537 bool wgp_endpoint_available; 538 539 /* The preshared key (optional) */ 540 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 541 542 void *wgp_si; 543 pcq_t *wgp_q; 544 545 struct wg_session *wgp_session_stable; 546 struct wg_session *wgp_session_unstable; 547 548 /* timestamp in big-endian */ 549 wg_timestamp_t wgp_timestamp_latest_init; 550 551 struct timespec wgp_last_handshake_time; 552 553 callout_t wgp_rekey_timer; 554 callout_t wgp_handshake_timeout_timer; 555 callout_t wgp_session_dtor_timer; 556 557 time_t wgp_handshake_start_time; 558 559 int wgp_n_allowedips; 560 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 561 562 time_t wgp_latest_cookie_time; 563 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 564 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 565 bool wgp_last_sent_mac1_valid; 566 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 567 bool wgp_last_sent_cookie_valid; 568 569 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 570 571 time_t wgp_last_genrandval_time; 572 uint32_t wgp_randval; 573 574 struct wg_ppsratecheck wgp_ppsratecheck; 575 576 volatile unsigned int wgp_tasks; 577 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 578 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 579 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 580 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 581 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 582 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 583 }; 584 585 struct wg_ops; 586 587 struct wg_softc { 588 struct ifnet wg_if; 589 LIST_ENTRY(wg_softc) wg_list; 590 kmutex_t *wg_lock; 591 krwlock_t *wg_rwlock; 592 593 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 594 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 595 596 int wg_npeers; 597 struct pslist_head wg_peers; 598 struct thmap *wg_peers_bypubkey; 599 struct thmap *wg_peers_byname; 600 struct thmap *wg_sessions_byindex; 601 uint16_t wg_listen_port; 602 603 struct wg_worker *wg_worker; 604 lwp_t *wg_worker_lwp; 605 606 struct radix_node_head *wg_rtable_ipv4; 607 struct radix_node_head *wg_rtable_ipv6; 608 609 struct wg_ppsratecheck wg_ppsratecheck; 610 611 struct wg_ops *wg_ops; 612 613 #ifdef WG_RUMPKERNEL 614 struct wg_user *wg_user; 615 #endif 616 }; 617 618 /* [W] 6.1 Preliminaries */ 619 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 620 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 621 #define WG_REKEY_AFTER_TIME 120 622 #define WG_REJECT_AFTER_TIME 180 623 #define WG_REKEY_ATTEMPT_TIME 90 624 #define WG_REKEY_TIMEOUT 5 625 #define WG_KEEPALIVE_TIMEOUT 10 626 627 #define WG_COOKIE_TIME 120 628 #define WG_RANDVAL_TIME (2 * 60) 629 630 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 631 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 632 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 633 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 634 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 635 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 636 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 637 638 static struct mbuf * 639 wg_get_mbuf(size_t, size_t); 640 641 static void wg_wakeup_worker(struct wg_worker *, int); 642 643 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 644 struct mbuf *); 645 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 646 const uint32_t, const uint8_t [], const struct sockaddr *); 647 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 648 struct wg_session *, const struct wg_msg_init *); 649 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 650 651 static struct wg_peer * 652 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 653 struct psref *); 654 static struct wg_peer * 655 wg_lookup_peer_by_pubkey(struct wg_softc *, 656 const uint8_t [], struct psref *); 657 658 static struct wg_session * 659 wg_lookup_session_by_index(struct wg_softc *, 660 const uint32_t, struct psref *); 661 662 static void wg_update_endpoint_if_necessary(struct wg_peer *, 663 const struct sockaddr *); 664 665 static void wg_schedule_rekey_timer(struct wg_peer *); 666 static void wg_schedule_session_dtor_timer(struct wg_peer *); 667 668 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 669 static void wg_calculate_keys(struct wg_session *, const bool); 670 671 static void wg_clear_states(struct wg_session *); 672 673 static void wg_get_peer(struct wg_peer *, struct psref *); 674 static void wg_put_peer(struct wg_peer *, struct psref *); 675 676 static int wg_send_so(struct wg_peer *, struct mbuf *); 677 static int wg_send_udp(struct wg_peer *, struct mbuf *); 678 static int wg_output(struct ifnet *, struct mbuf *, 679 const struct sockaddr *, const struct rtentry *); 680 static void wg_input(struct ifnet *, struct mbuf *, const int); 681 static int wg_ioctl(struct ifnet *, u_long, void *); 682 static int wg_bind_port(struct wg_softc *, const uint16_t); 683 static int wg_init(struct ifnet *); 684 static void wg_stop(struct ifnet *, int); 685 686 static void wg_purge_pending_packets(struct wg_peer *); 687 688 static int wg_clone_create(struct if_clone *, int); 689 static int wg_clone_destroy(struct ifnet *); 690 691 struct wg_ops { 692 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 693 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 694 void (*input)(struct ifnet *, struct mbuf *, const int); 695 int (*bind_port)(struct wg_softc *, const uint16_t); 696 }; 697 698 struct wg_ops wg_ops_rumpkernel = { 699 .send_hs_msg = wg_send_so, 700 .send_data_msg = wg_send_udp, 701 .input = wg_input, 702 .bind_port = wg_bind_port, 703 }; 704 705 #ifdef WG_RUMPKERNEL 706 static bool wg_user_mode(struct wg_softc *); 707 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 708 709 static int wg_send_user(struct wg_peer *, struct mbuf *); 710 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 711 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 712 713 struct wg_ops wg_ops_rumpuser = { 714 .send_hs_msg = wg_send_user, 715 .send_data_msg = wg_send_user, 716 .input = wg_input_user, 717 .bind_port = wg_bind_port_user, 718 }; 719 #endif 720 721 #define WG_PEER_READER_FOREACH(wgp, wg) \ 722 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 723 wgp_peerlist_entry) 724 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 725 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 726 wgp_peerlist_entry) 727 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 728 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 729 #define WG_PEER_WRITER_REMOVE(wgp) \ 730 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 731 732 struct wg_route { 733 struct radix_node wgr_nodes[2]; 734 struct wg_peer *wgr_peer; 735 }; 736 737 static struct radix_node_head * 738 wg_rnh(struct wg_softc *wg, const int family) 739 { 740 741 switch (family) { 742 case AF_INET: 743 return wg->wg_rtable_ipv4; 744 #ifdef INET6 745 case AF_INET6: 746 return wg->wg_rtable_ipv6; 747 #endif 748 default: 749 return NULL; 750 } 751 } 752 753 754 /* 755 * Global variables 756 */ 757 LIST_HEAD(wg_sclist, wg_softc); 758 static struct { 759 struct wg_sclist list; 760 kmutex_t lock; 761 } wg_softcs __cacheline_aligned; 762 763 struct psref_class *wg_psref_class __read_mostly; 764 765 static struct if_clone wg_cloner = 766 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 767 768 769 void wgattach(int); 770 /* ARGSUSED */ 771 void 772 wgattach(int count) 773 { 774 /* 775 * Nothing to do here, initialization is handled by the 776 * module initialization code in wginit() below). 777 */ 778 } 779 780 static void 781 wginit(void) 782 { 783 784 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 785 786 mutex_init(&wg_softcs.lock, MUTEX_DEFAULT, IPL_NONE); 787 LIST_INIT(&wg_softcs.list); 788 if_clone_attach(&wg_cloner); 789 } 790 791 static int 792 wgdetach(void) 793 { 794 int error = 0; 795 796 mutex_enter(&wg_softcs.lock); 797 if (!LIST_EMPTY(&wg_softcs.list)) { 798 mutex_exit(&wg_softcs.lock); 799 error = EBUSY; 800 } 801 802 if (error == 0) { 803 psref_class_destroy(wg_psref_class); 804 805 if_clone_detach(&wg_cloner); 806 } 807 808 return error; 809 } 810 811 static void 812 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 813 uint8_t hash[WG_HASH_LEN]) 814 { 815 /* [W] 5.4: CONSTRUCTION */ 816 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 817 /* [W] 5.4: IDENTIFIER */ 818 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 819 struct blake2s state; 820 821 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 822 signature, strlen(signature)); 823 824 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 825 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 826 827 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 828 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 829 blake2s_update(&state, id, strlen(id)); 830 blake2s_final(&state, hash); 831 832 WG_DUMP_HASH("ckey", ckey); 833 WG_DUMP_HASH("hash", hash); 834 } 835 836 static void 837 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 838 const size_t inputsize) 839 { 840 struct blake2s state; 841 842 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 843 blake2s_update(&state, hash, WG_HASH_LEN); 844 blake2s_update(&state, input, inputsize); 845 blake2s_final(&state, hash); 846 } 847 848 static void 849 wg_algo_mac(uint8_t out[], const size_t outsize, 850 const uint8_t key[], const size_t keylen, 851 const uint8_t input1[], const size_t input1len, 852 const uint8_t input2[], const size_t input2len) 853 { 854 struct blake2s state; 855 856 blake2s_init(&state, outsize, key, keylen); 857 858 blake2s_update(&state, input1, input1len); 859 if (input2 != NULL) 860 blake2s_update(&state, input2, input2len); 861 blake2s_final(&state, out); 862 } 863 864 static void 865 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 866 const uint8_t input1[], const size_t input1len, 867 const uint8_t input2[], const size_t input2len) 868 { 869 struct blake2s state; 870 /* [W] 5.4: LABEL-MAC1 */ 871 const char *label = "mac1----"; 872 uint8_t key[WG_HASH_LEN]; 873 874 blake2s_init(&state, sizeof(key), NULL, 0); 875 blake2s_update(&state, label, strlen(label)); 876 blake2s_update(&state, input1, input1len); 877 blake2s_final(&state, key); 878 879 blake2s_init(&state, outsize, key, sizeof(key)); 880 if (input2 != NULL) 881 blake2s_update(&state, input2, input2len); 882 blake2s_final(&state, out); 883 } 884 885 static void 886 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 887 const uint8_t input1[], const size_t input1len) 888 { 889 struct blake2s state; 890 /* [W] 5.4: LABEL-COOKIE */ 891 const char *label = "cookie--"; 892 893 blake2s_init(&state, outsize, NULL, 0); 894 blake2s_update(&state, label, strlen(label)); 895 blake2s_update(&state, input1, input1len); 896 blake2s_final(&state, out); 897 } 898 899 static void 900 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 901 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 902 { 903 904 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 905 906 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 907 crypto_scalarmult_base(pubkey, privkey); 908 } 909 910 static void 911 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 912 const uint8_t privkey[WG_STATIC_KEY_LEN], 913 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 914 { 915 916 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 917 918 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 919 KASSERT(ret == 0); 920 } 921 922 static void 923 wg_algo_hmac(uint8_t out[], const size_t outlen, 924 const uint8_t key[], const size_t keylen, 925 const uint8_t in[], const size_t inlen) 926 { 927 #define IPAD 0x36 928 #define OPAD 0x5c 929 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 930 uint8_t ipad[HMAC_BLOCK_LEN]; 931 uint8_t opad[HMAC_BLOCK_LEN]; 932 int i; 933 struct blake2s state; 934 935 KASSERT(outlen == WG_HASH_LEN); 936 KASSERT(keylen <= HMAC_BLOCK_LEN); 937 938 memcpy(hmackey, key, keylen); 939 940 for (i = 0; i < sizeof(hmackey); i++) { 941 ipad[i] = hmackey[i] ^ IPAD; 942 opad[i] = hmackey[i] ^ OPAD; 943 } 944 945 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 946 blake2s_update(&state, ipad, sizeof(ipad)); 947 blake2s_update(&state, in, inlen); 948 blake2s_final(&state, out); 949 950 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 951 blake2s_update(&state, opad, sizeof(opad)); 952 blake2s_update(&state, out, WG_HASH_LEN); 953 blake2s_final(&state, out); 954 #undef IPAD 955 #undef OPAD 956 } 957 958 static void 959 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 960 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 961 const uint8_t input[], const size_t inputlen) 962 { 963 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 964 uint8_t one[1]; 965 966 /* 967 * [N] 4.3: "an input_key_material byte sequence with length 968 * either zero bytes, 32 bytes, or DHLEN bytes." 969 */ 970 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 971 972 WG_DUMP_HASH("ckey", ckey); 973 if (input != NULL) 974 WG_DUMP_HASH("input", input); 975 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 976 input, inputlen); 977 WG_DUMP_HASH("tmp1", tmp1); 978 one[0] = 1; 979 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 980 one, sizeof(one)); 981 WG_DUMP_HASH("out1", out1); 982 if (out2 == NULL) 983 return; 984 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 985 tmp2[WG_KDF_OUTPUT_LEN] = 2; 986 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 987 tmp2, sizeof(tmp2)); 988 WG_DUMP_HASH("out2", out2); 989 if (out3 == NULL) 990 return; 991 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 992 tmp2[WG_KDF_OUTPUT_LEN] = 3; 993 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 994 tmp2, sizeof(tmp2)); 995 WG_DUMP_HASH("out3", out3); 996 } 997 998 static void 999 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1000 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1001 const uint8_t local_key[WG_STATIC_KEY_LEN], 1002 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1003 { 1004 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1005 1006 wg_algo_dh(dhout, local_key, remote_key); 1007 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1008 1009 WG_DUMP_HASH("dhout", dhout); 1010 WG_DUMP_HASH("ckey", ckey); 1011 if (cipher_key != NULL) 1012 WG_DUMP_HASH("cipher_key", cipher_key); 1013 } 1014 1015 static void 1016 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1017 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1018 const uint8_t auth[], size_t authlen) 1019 { 1020 uint8_t nonce[(32 + 64) / 8] = {0}; 1021 long long unsigned int outsize; 1022 int error __diagused; 1023 1024 le64enc(&nonce[4], counter); 1025 1026 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1027 plainsize, auth, authlen, NULL, nonce, key); 1028 KASSERT(error == 0); 1029 KASSERT(outsize == expected_outsize); 1030 } 1031 1032 static int 1033 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1034 const uint64_t counter, const uint8_t encrypted[], 1035 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1036 { 1037 uint8_t nonce[(32 + 64) / 8] = {0}; 1038 long long unsigned int outsize; 1039 int error; 1040 1041 le64enc(&nonce[4], counter); 1042 1043 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1044 encrypted, encryptedsize, auth, authlen, nonce, key); 1045 if (error == 0) 1046 KASSERT(outsize == expected_outsize); 1047 return error; 1048 } 1049 1050 static void 1051 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1052 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1053 const uint8_t auth[], size_t authlen, 1054 const uint8_t nonce[WG_SALT_LEN]) 1055 { 1056 long long unsigned int outsize; 1057 int error __diagused; 1058 1059 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1060 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1061 plain, plainsize, auth, authlen, NULL, nonce, key); 1062 KASSERT(error == 0); 1063 KASSERT(outsize == expected_outsize); 1064 } 1065 1066 static int 1067 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1068 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1069 const uint8_t auth[], size_t authlen, 1070 const uint8_t nonce[WG_SALT_LEN]) 1071 { 1072 long long unsigned int outsize; 1073 int error; 1074 1075 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1076 encrypted, encryptedsize, auth, authlen, nonce, key); 1077 if (error == 0) 1078 KASSERT(outsize == expected_outsize); 1079 return error; 1080 } 1081 1082 static void 1083 wg_algo_tai64n(wg_timestamp_t timestamp) 1084 { 1085 struct timespec ts; 1086 1087 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1088 getnanotime(&ts); 1089 /* TAI64 label in external TAI64 format */ 1090 be32enc(timestamp, 0x40000000UL + (ts.tv_sec >> 32)); 1091 /* second beginning from 1970 TAI */ 1092 be32enc(timestamp + 4, ts.tv_sec & 0xffffffffU); 1093 /* nanosecond in big-endian format */ 1094 be32enc(timestamp + 8, ts.tv_nsec); 1095 } 1096 1097 /* 1098 * wg_get_stable_session(wgp, psref) 1099 * 1100 * Get a passive reference to the current stable session, or 1101 * return NULL if there is no current stable session. 1102 * 1103 * The pointer is always there but the session is not necessarily 1104 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1105 * the session may transition from ESTABLISHED to DESTROYING while 1106 * holding the passive reference. 1107 */ 1108 static struct wg_session * 1109 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1110 { 1111 int s; 1112 struct wg_session *wgs; 1113 1114 s = pserialize_read_enter(); 1115 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1116 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1117 wgs = NULL; 1118 else 1119 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1120 pserialize_read_exit(s); 1121 1122 return wgs; 1123 } 1124 1125 static void 1126 wg_put_session(struct wg_session *wgs, struct psref *psref) 1127 { 1128 1129 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1130 } 1131 1132 static void 1133 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1134 { 1135 struct wg_peer *wgp = wgs->wgs_peer; 1136 struct wg_session *wgs0 __diagused; 1137 void *garbage; 1138 1139 KASSERT(mutex_owned(wgp->wgp_lock)); 1140 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1141 1142 /* Remove the session from the table. */ 1143 wgs0 = thmap_del(wg->wg_sessions_byindex, 1144 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1145 KASSERT(wgs0 == wgs); 1146 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1147 1148 /* Wait for passive references to drain. */ 1149 pserialize_perform(wgp->wgp_psz); 1150 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1151 1152 /* Free memory, zero state, and transition to UNKNOWN. */ 1153 thmap_gc(wg->wg_sessions_byindex, garbage); 1154 wg_clear_states(wgs); 1155 wgs->wgs_state = WGS_STATE_UNKNOWN; 1156 } 1157 1158 /* 1159 * wg_get_session_index(wg, wgs) 1160 * 1161 * Choose a session index for wgs->wgs_local_index, and store it 1162 * in wg's table of sessions by index. 1163 * 1164 * wgs must be the unstable session of its peer, and must be 1165 * transitioning out of the UNKNOWN state. 1166 */ 1167 static void 1168 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1169 { 1170 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1171 struct wg_session *wgs0; 1172 uint32_t index; 1173 1174 KASSERT(mutex_owned(wgp->wgp_lock)); 1175 KASSERT(wgs == wgp->wgp_session_unstable); 1176 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1177 1178 do { 1179 /* Pick a uniform random index. */ 1180 index = cprng_strong32(); 1181 1182 /* Try to take it. */ 1183 wgs->wgs_local_index = index; 1184 wgs0 = thmap_put(wg->wg_sessions_byindex, 1185 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1186 1187 /* If someone else beat us, start over. */ 1188 } while (__predict_false(wgs0 != wgs)); 1189 } 1190 1191 /* 1192 * wg_put_session_index(wg, wgs) 1193 * 1194 * Remove wgs from the table of sessions by index, wait for any 1195 * passive references to drain, and transition the session to the 1196 * UNKNOWN state. 1197 * 1198 * wgs must be the unstable session of its peer, and must not be 1199 * UNKNOWN or ESTABLISHED. 1200 */ 1201 static void 1202 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1203 { 1204 struct wg_peer *wgp = wgs->wgs_peer; 1205 1206 KASSERT(mutex_owned(wgp->wgp_lock)); 1207 KASSERT(wgs == wgp->wgp_session_unstable); 1208 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1209 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1210 1211 wg_destroy_session(wg, wgs); 1212 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1213 } 1214 1215 /* 1216 * Handshake patterns 1217 * 1218 * [W] 5: "These messages use the "IK" pattern from Noise" 1219 * [N] 7.5. Interactive handshake patterns (fundamental) 1220 * "The first character refers to the initiators static key:" 1221 * "I = Static key for initiator Immediately transmitted to responder, 1222 * despite reduced or absent identity hiding" 1223 * "The second character refers to the responders static key:" 1224 * "K = Static key for responder Known to initiator" 1225 * "IK: 1226 * <- s 1227 * ... 1228 * -> e, es, s, ss 1229 * <- e, ee, se" 1230 * [N] 9.4. Pattern modifiers 1231 * "IKpsk2: 1232 * <- s 1233 * ... 1234 * -> e, es, s, ss 1235 * <- e, ee, se, psk" 1236 */ 1237 static void 1238 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1239 struct wg_session *wgs, struct wg_msg_init *wgmi) 1240 { 1241 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1242 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1243 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1244 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1245 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1246 1247 KASSERT(mutex_owned(wgp->wgp_lock)); 1248 KASSERT(wgs == wgp->wgp_session_unstable); 1249 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1250 1251 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1252 wgmi->wgmi_sender = wgs->wgs_local_index; 1253 1254 /* [W] 5.4.2: First Message: Initiator to Responder */ 1255 1256 /* Ci := HASH(CONSTRUCTION) */ 1257 /* Hi := HASH(Ci || IDENTIFIER) */ 1258 wg_init_key_and_hash(ckey, hash); 1259 /* Hi := HASH(Hi || Sr^pub) */ 1260 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1261 1262 WG_DUMP_HASH("hash", hash); 1263 1264 /* [N] 2.2: "e" */ 1265 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1266 wg_algo_generate_keypair(pubkey, privkey); 1267 /* Ci := KDF1(Ci, Ei^pub) */ 1268 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1269 /* msg.ephemeral := Ei^pub */ 1270 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1271 /* Hi := HASH(Hi || msg.ephemeral) */ 1272 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1273 1274 WG_DUMP_HASH("ckey", ckey); 1275 WG_DUMP_HASH("hash", hash); 1276 1277 /* [N] 2.2: "es" */ 1278 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1279 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1280 1281 /* [N] 2.2: "s" */ 1282 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1283 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1284 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1285 hash, sizeof(hash)); 1286 /* Hi := HASH(Hi || msg.static) */ 1287 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1288 1289 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1290 1291 /* [N] 2.2: "ss" */ 1292 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1293 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1294 1295 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1296 wg_timestamp_t timestamp; 1297 wg_algo_tai64n(timestamp); 1298 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1299 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1300 /* Hi := HASH(Hi || msg.timestamp) */ 1301 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1302 1303 /* [W] 5.4.4 Cookie MACs */ 1304 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1305 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1306 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1307 /* Need mac1 to decrypt a cookie from a cookie message */ 1308 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1309 sizeof(wgp->wgp_last_sent_mac1)); 1310 wgp->wgp_last_sent_mac1_valid = true; 1311 1312 if (wgp->wgp_latest_cookie_time == 0 || 1313 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1314 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1315 else { 1316 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1317 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1318 (const uint8_t *)wgmi, 1319 offsetof(struct wg_msg_init, wgmi_mac2), 1320 NULL, 0); 1321 } 1322 1323 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1324 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1325 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1326 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1327 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1328 } 1329 1330 static void 1331 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1332 const struct sockaddr *src) 1333 { 1334 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1335 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1336 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1337 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1338 struct wg_peer *wgp; 1339 struct wg_session *wgs; 1340 int error, ret; 1341 struct psref psref_peer; 1342 uint8_t mac1[WG_MAC_LEN]; 1343 1344 WG_TRACE("init msg received"); 1345 1346 wg_algo_mac_mac1(mac1, sizeof(mac1), 1347 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1348 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1349 1350 /* 1351 * [W] 5.3: Denial of Service Mitigation & Cookies 1352 * "the responder, ..., must always reject messages with an invalid 1353 * msg.mac1" 1354 */ 1355 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1356 WG_DLOG("mac1 is invalid\n"); 1357 return; 1358 } 1359 1360 /* 1361 * [W] 5.4.2: First Message: Initiator to Responder 1362 * "When the responder receives this message, it does the same 1363 * operations so that its final state variables are identical, 1364 * replacing the operands of the DH function to produce equivalent 1365 * values." 1366 * Note that the following comments of operations are just copies of 1367 * the initiator's ones. 1368 */ 1369 1370 /* Ci := HASH(CONSTRUCTION) */ 1371 /* Hi := HASH(Ci || IDENTIFIER) */ 1372 wg_init_key_and_hash(ckey, hash); 1373 /* Hi := HASH(Hi || Sr^pub) */ 1374 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1375 1376 /* [N] 2.2: "e" */ 1377 /* Ci := KDF1(Ci, Ei^pub) */ 1378 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1379 sizeof(wgmi->wgmi_ephemeral)); 1380 /* Hi := HASH(Hi || msg.ephemeral) */ 1381 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1382 1383 WG_DUMP_HASH("ckey", ckey); 1384 1385 /* [N] 2.2: "es" */ 1386 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1387 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1388 1389 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1390 1391 /* [N] 2.2: "s" */ 1392 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1393 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1394 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1395 if (error != 0) { 1396 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1397 "wg_algo_aead_dec for secret key failed\n"); 1398 return; 1399 } 1400 /* Hi := HASH(Hi || msg.static) */ 1401 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1402 1403 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1404 if (wgp == NULL) { 1405 WG_DLOG("peer not found\n"); 1406 return; 1407 } 1408 1409 /* 1410 * Lock the peer to serialize access to cookie state. 1411 * 1412 * XXX Can we safely avoid holding the lock across DH? Take it 1413 * just to verify mac2 and then unlock/DH/lock? 1414 */ 1415 mutex_enter(wgp->wgp_lock); 1416 1417 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1418 WG_TRACE("under load"); 1419 /* 1420 * [W] 5.3: Denial of Service Mitigation & Cookies 1421 * "the responder, ..., and when under load may reject messages 1422 * with an invalid msg.mac2. If the responder receives a 1423 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1424 * and is under load, it may respond with a cookie reply 1425 * message" 1426 */ 1427 uint8_t zero[WG_MAC_LEN] = {0}; 1428 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1429 WG_TRACE("sending a cookie message: no cookie included"); 1430 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1431 wgmi->wgmi_mac1, src); 1432 goto out; 1433 } 1434 if (!wgp->wgp_last_sent_cookie_valid) { 1435 WG_TRACE("sending a cookie message: no cookie sent ever"); 1436 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1437 wgmi->wgmi_mac1, src); 1438 goto out; 1439 } 1440 uint8_t mac2[WG_MAC_LEN]; 1441 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1442 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1443 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1444 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1445 WG_DLOG("mac2 is invalid\n"); 1446 goto out; 1447 } 1448 WG_TRACE("under load, but continue to sending"); 1449 } 1450 1451 /* [N] 2.2: "ss" */ 1452 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1453 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1454 1455 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1456 wg_timestamp_t timestamp; 1457 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1458 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1459 hash, sizeof(hash)); 1460 if (error != 0) { 1461 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1462 "wg_algo_aead_dec for timestamp failed\n"); 1463 goto out; 1464 } 1465 /* Hi := HASH(Hi || msg.timestamp) */ 1466 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1467 1468 /* 1469 * [W] 5.1 "The responder keeps track of the greatest timestamp 1470 * received per peer and discards packets containing 1471 * timestamps less than or equal to it." 1472 */ 1473 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1474 sizeof(timestamp)); 1475 if (ret <= 0) { 1476 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1477 "invalid init msg: timestamp is old\n"); 1478 goto out; 1479 } 1480 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1481 1482 /* 1483 * Message is good -- we're committing to handle it now, unless 1484 * we were already initiating a session. 1485 */ 1486 wgs = wgp->wgp_session_unstable; 1487 switch (wgs->wgs_state) { 1488 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1489 wg_get_session_index(wg, wgs); 1490 break; 1491 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1492 WG_TRACE("Session already initializing, ignoring the message"); 1493 goto out; 1494 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1495 WG_TRACE("Session already initializing, destroying old states"); 1496 wg_clear_states(wgs); 1497 /* keep session index */ 1498 break; 1499 case WGS_STATE_ESTABLISHED: /* can't happen */ 1500 panic("unstable session can't be established"); 1501 break; 1502 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1503 WG_TRACE("Session destroying, but force to clear"); 1504 callout_stop(&wgp->wgp_session_dtor_timer); 1505 wg_clear_states(wgs); 1506 /* keep session index */ 1507 break; 1508 default: 1509 panic("invalid session state: %d", wgs->wgs_state); 1510 } 1511 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1512 1513 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1514 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1515 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1516 sizeof(wgmi->wgmi_ephemeral)); 1517 1518 wg_update_endpoint_if_necessary(wgp, src); 1519 1520 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1521 1522 wg_calculate_keys(wgs, false); 1523 wg_clear_states(wgs); 1524 1525 out: 1526 mutex_exit(wgp->wgp_lock); 1527 wg_put_peer(wgp, &psref_peer); 1528 } 1529 1530 static struct socket * 1531 wg_get_so_by_af(struct wg_worker *wgw, const int af) 1532 { 1533 1534 return (af == AF_INET) ? wgw->wgw_so4 : wgw->wgw_so6; 1535 } 1536 1537 static struct socket * 1538 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1539 { 1540 1541 return wg_get_so_by_af(wgp->wgp_sc->wg_worker, wgsa_family(wgsa)); 1542 } 1543 1544 static struct wg_sockaddr * 1545 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1546 { 1547 struct wg_sockaddr *wgsa; 1548 int s; 1549 1550 s = pserialize_read_enter(); 1551 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1552 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1553 pserialize_read_exit(s); 1554 1555 return wgsa; 1556 } 1557 1558 static void 1559 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1560 { 1561 1562 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1563 } 1564 1565 static int 1566 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1567 { 1568 int error; 1569 struct socket *so; 1570 struct psref psref; 1571 struct wg_sockaddr *wgsa; 1572 1573 wgsa = wg_get_endpoint_sa(wgp, &psref); 1574 so = wg_get_so_by_peer(wgp, wgsa); 1575 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1576 wg_put_sa(wgp, wgsa, &psref); 1577 1578 return error; 1579 } 1580 1581 static int 1582 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1583 { 1584 int error; 1585 struct mbuf *m; 1586 struct wg_msg_init *wgmi; 1587 struct wg_session *wgs; 1588 1589 KASSERT(mutex_owned(wgp->wgp_lock)); 1590 1591 wgs = wgp->wgp_session_unstable; 1592 /* XXX pull dispatch out into wg_task_send_init_message */ 1593 switch (wgs->wgs_state) { 1594 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1595 wg_get_session_index(wg, wgs); 1596 break; 1597 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1598 WG_TRACE("Session already initializing, skip starting new one"); 1599 return EBUSY; 1600 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1601 WG_TRACE("Session already initializing, destroying old states"); 1602 wg_clear_states(wgs); 1603 /* keep session index */ 1604 break; 1605 case WGS_STATE_ESTABLISHED: /* can't happen */ 1606 panic("unstable session can't be established"); 1607 break; 1608 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1609 WG_TRACE("Session destroying"); 1610 /* XXX should wait? */ 1611 return EBUSY; 1612 } 1613 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1614 1615 m = m_gethdr(M_WAIT, MT_DATA); 1616 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1617 wgmi = mtod(m, struct wg_msg_init *); 1618 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1619 1620 error = wg->wg_ops->send_hs_msg(wgp, m); 1621 if (error == 0) { 1622 WG_TRACE("init msg sent"); 1623 1624 if (wgp->wgp_handshake_start_time == 0) 1625 wgp->wgp_handshake_start_time = time_uptime; 1626 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1627 MIN(wg_rekey_timeout, INT_MAX/hz) * hz); 1628 } else { 1629 wg_put_session_index(wg, wgs); 1630 } 1631 1632 return error; 1633 } 1634 1635 static void 1636 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1637 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1638 const struct wg_msg_init *wgmi) 1639 { 1640 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1641 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1642 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1643 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1644 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1645 1646 KASSERT(mutex_owned(wgp->wgp_lock)); 1647 KASSERT(wgs == wgp->wgp_session_unstable); 1648 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1649 1650 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1651 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1652 1653 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1654 wgmr->wgmr_sender = wgs->wgs_local_index; 1655 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1656 1657 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1658 1659 /* [N] 2.2: "e" */ 1660 /* Er^priv, Er^pub := DH-GENERATE() */ 1661 wg_algo_generate_keypair(pubkey, privkey); 1662 /* Cr := KDF1(Cr, Er^pub) */ 1663 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1664 /* msg.ephemeral := Er^pub */ 1665 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1666 /* Hr := HASH(Hr || msg.ephemeral) */ 1667 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1668 1669 WG_DUMP_HASH("ckey", ckey); 1670 WG_DUMP_HASH("hash", hash); 1671 1672 /* [N] 2.2: "ee" */ 1673 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1674 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1675 1676 /* [N] 2.2: "se" */ 1677 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1678 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1679 1680 /* [N] 9.2: "psk" */ 1681 { 1682 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1683 /* Cr, r, k := KDF3(Cr, Q) */ 1684 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1685 sizeof(wgp->wgp_psk)); 1686 /* Hr := HASH(Hr || r) */ 1687 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1688 } 1689 1690 /* msg.empty := AEAD(k, 0, e, Hr) */ 1691 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1692 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1693 /* Hr := HASH(Hr || msg.empty) */ 1694 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1695 1696 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1697 1698 /* [W] 5.4.4: Cookie MACs */ 1699 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1700 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1701 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1702 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1703 /* Need mac1 to decrypt a cookie from a cookie message */ 1704 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1705 sizeof(wgp->wgp_last_sent_mac1)); 1706 wgp->wgp_last_sent_mac1_valid = true; 1707 1708 if (wgp->wgp_latest_cookie_time == 0 || 1709 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1710 /* msg.mac2 := 0^16 */ 1711 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1712 else { 1713 /* msg.mac2 := MAC(Lm, msg_b) */ 1714 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1715 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1716 (const uint8_t *)wgmr, 1717 offsetof(struct wg_msg_resp, wgmr_mac2), 1718 NULL, 0); 1719 } 1720 1721 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1722 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1723 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1724 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1725 wgs->wgs_remote_index = wgmi->wgmi_sender; 1726 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1727 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1728 } 1729 1730 static void 1731 wg_swap_sessions(struct wg_peer *wgp) 1732 { 1733 struct wg_session *wgs, *wgs_prev; 1734 1735 KASSERT(mutex_owned(wgp->wgp_lock)); 1736 1737 wgs = wgp->wgp_session_unstable; 1738 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1739 1740 wgs_prev = wgp->wgp_session_stable; 1741 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1742 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1743 atomic_store_release(&wgp->wgp_session_stable, wgs); 1744 wgp->wgp_session_unstable = wgs_prev; 1745 } 1746 1747 static void 1748 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1749 const struct sockaddr *src) 1750 { 1751 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1752 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1753 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1754 struct wg_peer *wgp; 1755 struct wg_session *wgs; 1756 struct psref psref; 1757 int error; 1758 uint8_t mac1[WG_MAC_LEN]; 1759 struct wg_session *wgs_prev; 1760 1761 wg_algo_mac_mac1(mac1, sizeof(mac1), 1762 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1763 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1764 1765 /* 1766 * [W] 5.3: Denial of Service Mitigation & Cookies 1767 * "the responder, ..., must always reject messages with an invalid 1768 * msg.mac1" 1769 */ 1770 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1771 WG_DLOG("mac1 is invalid\n"); 1772 return; 1773 } 1774 1775 WG_TRACE("resp msg received"); 1776 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1777 if (wgs == NULL) { 1778 WG_TRACE("No session found"); 1779 return; 1780 } 1781 1782 wgp = wgs->wgs_peer; 1783 1784 mutex_enter(wgp->wgp_lock); 1785 1786 /* If we weren't waiting for a handshake response, drop it. */ 1787 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1788 WG_TRACE("peer sent spurious handshake response, ignoring"); 1789 goto out; 1790 } 1791 1792 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1793 WG_TRACE("under load"); 1794 /* 1795 * [W] 5.3: Denial of Service Mitigation & Cookies 1796 * "the responder, ..., and when under load may reject messages 1797 * with an invalid msg.mac2. If the responder receives a 1798 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1799 * and is under load, it may respond with a cookie reply 1800 * message" 1801 */ 1802 uint8_t zero[WG_MAC_LEN] = {0}; 1803 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1804 WG_TRACE("sending a cookie message: no cookie included"); 1805 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1806 wgmr->wgmr_mac1, src); 1807 goto out; 1808 } 1809 if (!wgp->wgp_last_sent_cookie_valid) { 1810 WG_TRACE("sending a cookie message: no cookie sent ever"); 1811 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1812 wgmr->wgmr_mac1, src); 1813 goto out; 1814 } 1815 uint8_t mac2[WG_MAC_LEN]; 1816 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1817 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1818 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1819 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1820 WG_DLOG("mac2 is invalid\n"); 1821 goto out; 1822 } 1823 WG_TRACE("under load, but continue to sending"); 1824 } 1825 1826 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1827 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1828 1829 /* 1830 * [W] 5.4.3 Second Message: Responder to Initiator 1831 * "When the initiator receives this message, it does the same 1832 * operations so that its final state variables are identical, 1833 * replacing the operands of the DH function to produce equivalent 1834 * values." 1835 * Note that the following comments of operations are just copies of 1836 * the initiator's ones. 1837 */ 1838 1839 /* [N] 2.2: "e" */ 1840 /* Cr := KDF1(Cr, Er^pub) */ 1841 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1842 sizeof(wgmr->wgmr_ephemeral)); 1843 /* Hr := HASH(Hr || msg.ephemeral) */ 1844 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1845 1846 WG_DUMP_HASH("ckey", ckey); 1847 WG_DUMP_HASH("hash", hash); 1848 1849 /* [N] 2.2: "ee" */ 1850 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1851 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1852 wgmr->wgmr_ephemeral); 1853 1854 /* [N] 2.2: "se" */ 1855 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1856 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1857 1858 /* [N] 9.2: "psk" */ 1859 { 1860 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1861 /* Cr, r, k := KDF3(Cr, Q) */ 1862 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1863 sizeof(wgp->wgp_psk)); 1864 /* Hr := HASH(Hr || r) */ 1865 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1866 } 1867 1868 { 1869 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 1870 /* msg.empty := AEAD(k, 0, e, Hr) */ 1871 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 1872 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 1873 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1874 if (error != 0) { 1875 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1876 "wg_algo_aead_dec for empty message failed\n"); 1877 goto out; 1878 } 1879 /* Hr := HASH(Hr || msg.empty) */ 1880 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1881 } 1882 1883 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 1884 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 1885 wgs->wgs_remote_index = wgmr->wgmr_sender; 1886 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1887 1888 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1889 wgs->wgs_state = WGS_STATE_ESTABLISHED; 1890 wgs->wgs_time_established = time_uptime; 1891 wgs->wgs_time_last_data_sent = 0; 1892 wgs->wgs_is_initiator = true; 1893 wg_calculate_keys(wgs, true); 1894 wg_clear_states(wgs); 1895 WG_TRACE("WGS_STATE_ESTABLISHED"); 1896 1897 callout_stop(&wgp->wgp_handshake_timeout_timer); 1898 1899 wg_swap_sessions(wgp); 1900 KASSERT(wgs == wgp->wgp_session_stable); 1901 wgs_prev = wgp->wgp_session_unstable; 1902 getnanotime(&wgp->wgp_last_handshake_time); 1903 wgp->wgp_handshake_start_time = 0; 1904 wgp->wgp_last_sent_mac1_valid = false; 1905 wgp->wgp_last_sent_cookie_valid = false; 1906 1907 wg_schedule_rekey_timer(wgp); 1908 1909 wg_update_endpoint_if_necessary(wgp, src); 1910 1911 /* 1912 * Send something immediately (same as the official implementation) 1913 * XXX if there are pending data packets, we don't need to send 1914 * a keepalive message. 1915 */ 1916 wg_send_keepalive_msg(wgp, wgs); 1917 1918 /* Anyway run a softint to flush pending packets */ 1919 kpreempt_disable(); 1920 softint_schedule(wgp->wgp_si); 1921 kpreempt_enable(); 1922 WG_TRACE("softint scheduled"); 1923 1924 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 1925 /* Wait for wg_get_stable_session to drain. */ 1926 pserialize_perform(wgp->wgp_psz); 1927 1928 /* Transition ESTABLISHED->DESTROYING. */ 1929 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 1930 1931 /* We can't destroy the old session immediately */ 1932 wg_schedule_session_dtor_timer(wgp); 1933 } else { 1934 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 1935 "state=%d", wgs_prev->wgs_state); 1936 } 1937 1938 out: 1939 mutex_exit(wgp->wgp_lock); 1940 wg_put_session(wgs, &psref); 1941 } 1942 1943 static int 1944 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1945 struct wg_session *wgs, const struct wg_msg_init *wgmi) 1946 { 1947 int error; 1948 struct mbuf *m; 1949 struct wg_msg_resp *wgmr; 1950 1951 KASSERT(mutex_owned(wgp->wgp_lock)); 1952 KASSERT(wgs == wgp->wgp_session_unstable); 1953 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1954 1955 m = m_gethdr(M_WAIT, MT_DATA); 1956 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 1957 wgmr = mtod(m, struct wg_msg_resp *); 1958 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 1959 1960 error = wg->wg_ops->send_hs_msg(wgp, m); 1961 if (error == 0) 1962 WG_TRACE("resp msg sent"); 1963 return error; 1964 } 1965 1966 static struct wg_peer * 1967 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 1968 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 1969 { 1970 struct wg_peer *wgp; 1971 1972 int s = pserialize_read_enter(); 1973 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 1974 if (wgp != NULL) 1975 wg_get_peer(wgp, psref); 1976 pserialize_read_exit(s); 1977 1978 return wgp; 1979 } 1980 1981 static void 1982 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 1983 struct wg_msg_cookie *wgmc, const uint32_t sender, 1984 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 1985 { 1986 uint8_t cookie[WG_COOKIE_LEN]; 1987 uint8_t key[WG_HASH_LEN]; 1988 uint8_t addr[sizeof(struct in6_addr)]; 1989 size_t addrlen; 1990 uint16_t uh_sport; /* be */ 1991 1992 KASSERT(mutex_owned(wgp->wgp_lock)); 1993 1994 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 1995 wgmc->wgmc_receiver = sender; 1996 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 1997 1998 /* 1999 * [W] 5.4.7: Under Load: Cookie Reply Message 2000 * "The secret variable, Rm, changes every two minutes to a 2001 * random value" 2002 */ 2003 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2004 wgp->wgp_randval = cprng_strong32(); 2005 wgp->wgp_last_genrandval_time = time_uptime; 2006 } 2007 2008 switch (src->sa_family) { 2009 case AF_INET: { 2010 const struct sockaddr_in *sin = satocsin(src); 2011 addrlen = sizeof(sin->sin_addr); 2012 memcpy(addr, &sin->sin_addr, addrlen); 2013 uh_sport = sin->sin_port; 2014 break; 2015 } 2016 #ifdef INET6 2017 case AF_INET6: { 2018 const struct sockaddr_in6 *sin6 = satocsin6(src); 2019 addrlen = sizeof(sin6->sin6_addr); 2020 memcpy(addr, &sin6->sin6_addr, addrlen); 2021 uh_sport = sin6->sin6_port; 2022 break; 2023 } 2024 #endif 2025 default: 2026 panic("invalid af=%d", src->sa_family); 2027 } 2028 2029 wg_algo_mac(cookie, sizeof(cookie), 2030 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2031 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2032 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2033 sizeof(wg->wg_pubkey)); 2034 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2035 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2036 2037 /* Need to store to calculate mac2 */ 2038 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2039 wgp->wgp_last_sent_cookie_valid = true; 2040 } 2041 2042 static int 2043 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2044 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2045 const struct sockaddr *src) 2046 { 2047 int error; 2048 struct mbuf *m; 2049 struct wg_msg_cookie *wgmc; 2050 2051 KASSERT(mutex_owned(wgp->wgp_lock)); 2052 2053 m = m_gethdr(M_WAIT, MT_DATA); 2054 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2055 wgmc = mtod(m, struct wg_msg_cookie *); 2056 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2057 2058 error = wg->wg_ops->send_hs_msg(wgp, m); 2059 if (error == 0) 2060 WG_TRACE("cookie msg sent"); 2061 return error; 2062 } 2063 2064 static bool 2065 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2066 { 2067 #ifdef WG_DEBUG_PARAMS 2068 if (wg_force_underload) 2069 return true; 2070 #endif 2071 2072 /* 2073 * XXX we don't have a means of a load estimation. The purpose of 2074 * the mechanism is a DoS mitigation, so we consider frequent handshake 2075 * messages as (a kind of) load; if a message of the same type comes 2076 * to a peer within 1 second, we consider we are under load. 2077 */ 2078 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2079 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2080 return (time_uptime - last) == 0; 2081 } 2082 2083 static void 2084 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2085 { 2086 2087 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2088 2089 /* 2090 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2091 */ 2092 if (initiator) { 2093 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2094 wgs->wgs_chaining_key, NULL, 0); 2095 } else { 2096 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2097 wgs->wgs_chaining_key, NULL, 0); 2098 } 2099 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2100 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2101 } 2102 2103 static uint64_t 2104 wg_session_get_send_counter(struct wg_session *wgs) 2105 { 2106 #ifdef __HAVE_ATOMIC64_LOADSTORE 2107 return atomic_load_relaxed(&wgs->wgs_send_counter); 2108 #else 2109 uint64_t send_counter; 2110 2111 mutex_enter(&wgs->wgs_send_counter_lock); 2112 send_counter = wgs->wgs_send_counter; 2113 mutex_exit(&wgs->wgs_send_counter_lock); 2114 2115 return send_counter; 2116 #endif 2117 } 2118 2119 static uint64_t 2120 wg_session_inc_send_counter(struct wg_session *wgs) 2121 { 2122 #ifdef __HAVE_ATOMIC64_LOADSTORE 2123 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2124 #else 2125 uint64_t send_counter; 2126 2127 mutex_enter(&wgs->wgs_send_counter_lock); 2128 send_counter = wgs->wgs_send_counter++; 2129 mutex_exit(&wgs->wgs_send_counter_lock); 2130 2131 return send_counter; 2132 #endif 2133 } 2134 2135 static void 2136 wg_clear_states(struct wg_session *wgs) 2137 { 2138 2139 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2140 2141 wgs->wgs_send_counter = 0; 2142 sliwin_reset(&wgs->wgs_recvwin->window); 2143 2144 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2145 wgs_clear(handshake_hash); 2146 wgs_clear(chaining_key); 2147 wgs_clear(ephemeral_key_pub); 2148 wgs_clear(ephemeral_key_priv); 2149 wgs_clear(ephemeral_key_peer); 2150 #undef wgs_clear 2151 } 2152 2153 static struct wg_session * 2154 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2155 struct psref *psref) 2156 { 2157 struct wg_session *wgs; 2158 2159 int s = pserialize_read_enter(); 2160 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2161 if (wgs != NULL) { 2162 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2163 WGS_STATE_UNKNOWN); 2164 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2165 } 2166 pserialize_read_exit(s); 2167 2168 return wgs; 2169 } 2170 2171 static void 2172 wg_schedule_rekey_timer(struct wg_peer *wgp) 2173 { 2174 int timeout = MIN(wg_rekey_after_time, INT_MAX/hz); 2175 2176 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2177 } 2178 2179 static void 2180 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2181 { 2182 struct mbuf *m; 2183 2184 /* 2185 * [W] 6.5 Passive Keepalive 2186 * "A keepalive message is simply a transport data message with 2187 * a zero-length encapsulated encrypted inner-packet." 2188 */ 2189 m = m_gethdr(M_WAIT, MT_DATA); 2190 wg_send_data_msg(wgp, wgs, m); 2191 } 2192 2193 static bool 2194 wg_need_to_send_init_message(struct wg_session *wgs) 2195 { 2196 /* 2197 * [W] 6.2 Transport Message Limits 2198 * "if a peer is the initiator of a current secure session, 2199 * WireGuard will send a handshake initiation message to begin 2200 * a new secure session ... if after receiving a transport data 2201 * message, the current secure session is (REJECT-AFTER-TIME 2202 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2203 * not yet acted upon this event." 2204 */ 2205 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2206 (time_uptime - wgs->wgs_time_established) >= 2207 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2208 } 2209 2210 static void 2211 wg_schedule_peer_task(struct wg_peer *wgp, int task) 2212 { 2213 2214 atomic_or_uint(&wgp->wgp_tasks, task); 2215 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2216 wg_wakeup_worker(wgp->wgp_sc->wg_worker, WG_WAKEUP_REASON_PEER); 2217 } 2218 2219 static void 2220 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2221 { 2222 struct wg_sockaddr *wgsa_prev; 2223 2224 WG_TRACE("Changing endpoint"); 2225 2226 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2227 wgsa_prev = wgp->wgp_endpoint; 2228 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2229 wgp->wgp_endpoint0 = wgsa_prev; 2230 atomic_store_release(&wgp->wgp_endpoint_available, true); 2231 2232 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2233 } 2234 2235 static bool 2236 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2237 { 2238 uint16_t packet_len; 2239 const struct ip *ip; 2240 2241 if (__predict_false(decrypted_len < sizeof(struct ip))) 2242 return false; 2243 2244 ip = (const struct ip *)packet; 2245 if (ip->ip_v == 4) 2246 *af = AF_INET; 2247 else if (ip->ip_v == 6) 2248 *af = AF_INET6; 2249 else 2250 return false; 2251 2252 WG_DLOG("af=%d\n", *af); 2253 2254 if (*af == AF_INET) { 2255 packet_len = ntohs(ip->ip_len); 2256 } else { 2257 const struct ip6_hdr *ip6; 2258 2259 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr))) 2260 return false; 2261 2262 ip6 = (const struct ip6_hdr *)packet; 2263 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen); 2264 } 2265 2266 WG_DLOG("packet_len=%u\n", packet_len); 2267 if (packet_len > decrypted_len) 2268 return false; 2269 2270 return true; 2271 } 2272 2273 static bool 2274 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2275 int af, char *packet) 2276 { 2277 struct sockaddr_storage ss; 2278 struct sockaddr *sa; 2279 struct psref psref; 2280 struct wg_peer *wgp; 2281 bool ok; 2282 2283 /* 2284 * II CRYPTOKEY ROUTING 2285 * "it will only accept it if its source IP resolves in the 2286 * table to the public key used in the secure session for 2287 * decrypting it." 2288 */ 2289 2290 if (af == AF_INET) { 2291 const struct ip *ip = (const struct ip *)packet; 2292 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2293 sockaddr_in_init(sin, &ip->ip_src, 0); 2294 sa = sintosa(sin); 2295 #ifdef INET6 2296 } else { 2297 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2298 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2299 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2300 sa = sin6tosa(sin6); 2301 #endif 2302 } 2303 2304 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2305 ok = (wgp == wgp_expected); 2306 if (wgp != NULL) 2307 wg_put_peer(wgp, &psref); 2308 2309 return ok; 2310 } 2311 2312 static void 2313 wg_session_dtor_timer(void *arg) 2314 { 2315 struct wg_peer *wgp = arg; 2316 2317 WG_TRACE("enter"); 2318 2319 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2320 } 2321 2322 static void 2323 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2324 { 2325 2326 /* 1 second grace period */ 2327 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2328 } 2329 2330 static bool 2331 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2332 { 2333 if (sa1->sa_family != sa2->sa_family) 2334 return false; 2335 2336 switch (sa1->sa_family) { 2337 case AF_INET: 2338 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2339 case AF_INET6: 2340 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2341 default: 2342 return true; 2343 } 2344 } 2345 2346 static void 2347 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2348 const struct sockaddr *src) 2349 { 2350 struct wg_sockaddr *wgsa; 2351 struct psref psref; 2352 2353 wgsa = wg_get_endpoint_sa(wgp, &psref); 2354 2355 #ifdef WG_DEBUG_LOG 2356 char oldaddr[128], newaddr[128]; 2357 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2358 sockaddr_format(src, newaddr, sizeof(newaddr)); 2359 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2360 #endif 2361 2362 /* 2363 * III: "Since the packet has authenticated correctly, the source IP of 2364 * the outer UDP/IP packet is used to update the endpoint for peer..." 2365 */ 2366 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2367 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2368 /* XXX We can't change the endpoint twice in a short period */ 2369 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2370 wg_change_endpoint(wgp, src); 2371 } 2372 } 2373 2374 wg_put_sa(wgp, wgsa, &psref); 2375 } 2376 2377 static void 2378 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2379 const struct sockaddr *src) 2380 { 2381 struct wg_msg_data *wgmd; 2382 char *encrypted_buf = NULL, *decrypted_buf; 2383 size_t encrypted_len, decrypted_len; 2384 struct wg_session *wgs; 2385 struct wg_peer *wgp; 2386 int state; 2387 size_t mlen; 2388 struct psref psref; 2389 int error, af; 2390 bool success, free_encrypted_buf = false, ok; 2391 struct mbuf *n; 2392 2393 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2394 wgmd = mtod(m, struct wg_msg_data *); 2395 2396 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2397 WG_TRACE("data"); 2398 2399 /* Find the putative session, or drop. */ 2400 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2401 if (wgs == NULL) { 2402 WG_TRACE("No session found"); 2403 m_freem(m); 2404 return; 2405 } 2406 2407 /* 2408 * We are only ready to handle data when in INIT_PASSIVE, 2409 * ESTABLISHED, or DESTROYING. All transitions out of that 2410 * state dissociate the session index and drain psrefs. 2411 */ 2412 state = atomic_load_relaxed(&wgs->wgs_state); 2413 switch (state) { 2414 case WGS_STATE_UNKNOWN: 2415 panic("wg session %p in unknown state has session index %u", 2416 wgs, wgmd->wgmd_receiver); 2417 case WGS_STATE_INIT_ACTIVE: 2418 WG_TRACE("not yet ready for data"); 2419 goto out; 2420 case WGS_STATE_INIT_PASSIVE: 2421 case WGS_STATE_ESTABLISHED: 2422 case WGS_STATE_DESTROYING: 2423 break; 2424 } 2425 2426 /* 2427 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2428 * to update the endpoint if authentication succeeds. 2429 */ 2430 wgp = wgs->wgs_peer; 2431 2432 /* 2433 * Reject outrageously wrong sequence numbers before doing any 2434 * crypto work or taking any locks. 2435 */ 2436 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2437 le64toh(wgmd->wgmd_counter)); 2438 if (error) { 2439 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2440 "out-of-window packet: %"PRIu64"\n", 2441 le64toh(wgmd->wgmd_counter)); 2442 goto out; 2443 } 2444 2445 /* Ensure the payload and authenticator are contiguous. */ 2446 mlen = m_length(m); 2447 encrypted_len = mlen - sizeof(*wgmd); 2448 if (encrypted_len < WG_AUTHTAG_LEN) { 2449 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2450 goto out; 2451 } 2452 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2453 if (success) { 2454 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2455 } else { 2456 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2457 if (encrypted_buf == NULL) { 2458 WG_DLOG("failed to allocate encrypted_buf\n"); 2459 goto out; 2460 } 2461 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2462 free_encrypted_buf = true; 2463 } 2464 /* m_ensure_contig may change m regardless of its result */ 2465 KASSERT(m->m_len >= sizeof(*wgmd)); 2466 wgmd = mtod(m, struct wg_msg_data *); 2467 2468 /* 2469 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2470 * a zero-length buffer (XXX). Drop if plaintext is longer 2471 * than MCLBYTES (XXX). 2472 */ 2473 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2474 if (decrypted_len > MCLBYTES) { 2475 /* FIXME handle larger data than MCLBYTES */ 2476 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2477 goto out; 2478 } 2479 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2480 if (n == NULL) { 2481 WG_DLOG("wg_get_mbuf failed\n"); 2482 goto out; 2483 } 2484 decrypted_buf = mtod(n, char *); 2485 2486 /* Decrypt and verify the packet. */ 2487 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2488 error = wg_algo_aead_dec(decrypted_buf, 2489 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2490 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2491 encrypted_len, NULL, 0); 2492 if (error != 0) { 2493 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2494 "failed to wg_algo_aead_dec\n"); 2495 m_freem(n); 2496 goto out; 2497 } 2498 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2499 2500 /* Packet is genuine. Reject it if a replay or just too old. */ 2501 mutex_enter(&wgs->wgs_recvwin->lock); 2502 error = sliwin_update(&wgs->wgs_recvwin->window, 2503 le64toh(wgmd->wgmd_counter)); 2504 mutex_exit(&wgs->wgs_recvwin->lock); 2505 if (error) { 2506 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2507 "replay or out-of-window packet: %"PRIu64"\n", 2508 le64toh(wgmd->wgmd_counter)); 2509 m_freem(n); 2510 goto out; 2511 } 2512 2513 /* We're done with m now; free it and chuck the pointers. */ 2514 m_freem(m); 2515 m = NULL; 2516 wgmd = NULL; 2517 2518 /* 2519 * Validate the encapsulated packet header and get the address 2520 * family, or drop. 2521 */ 2522 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2523 if (!ok) { 2524 m_freem(n); 2525 goto out; 2526 } 2527 2528 /* 2529 * The packet is genuine. Update the peer's endpoint if the 2530 * source address changed. 2531 * 2532 * XXX How to prevent DoS by replaying genuine packets from the 2533 * wrong source address? 2534 */ 2535 wg_update_endpoint_if_necessary(wgp, src); 2536 2537 /* Submit it into our network stack if routable. */ 2538 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2539 if (ok) { 2540 wg->wg_ops->input(&wg->wg_if, n, af); 2541 } else { 2542 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2543 "invalid source address\n"); 2544 m_freem(n); 2545 /* 2546 * The inner address is invalid however the session is valid 2547 * so continue the session processing below. 2548 */ 2549 } 2550 n = NULL; 2551 2552 /* Update the state machine if necessary. */ 2553 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2554 /* 2555 * We were waiting for the initiator to send their 2556 * first data transport message, and that has happened. 2557 * Schedule a task to establish this session. 2558 */ 2559 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2560 } else { 2561 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2562 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2563 } 2564 /* 2565 * [W] 6.5 Passive Keepalive 2566 * "If a peer has received a validly-authenticated transport 2567 * data message (section 5.4.6), but does not have any packets 2568 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2569 * a keepalive message." 2570 */ 2571 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2572 (uintmax_t)time_uptime, 2573 (uintmax_t)wgs->wgs_time_last_data_sent); 2574 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2575 wg_keepalive_timeout) { 2576 WG_TRACE("Schedule sending keepalive message"); 2577 /* 2578 * We can't send a keepalive message here to avoid 2579 * a deadlock; we already hold the solock of a socket 2580 * that is used to send the message. 2581 */ 2582 wg_schedule_peer_task(wgp, 2583 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2584 } 2585 } 2586 out: 2587 wg_put_session(wgs, &psref); 2588 if (m != NULL) 2589 m_freem(m); 2590 if (free_encrypted_buf) 2591 kmem_intr_free(encrypted_buf, encrypted_len); 2592 } 2593 2594 static void 2595 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2596 { 2597 struct wg_session *wgs; 2598 struct wg_peer *wgp; 2599 struct psref psref; 2600 int error; 2601 uint8_t key[WG_HASH_LEN]; 2602 uint8_t cookie[WG_COOKIE_LEN]; 2603 2604 WG_TRACE("cookie msg received"); 2605 2606 /* Find the putative session. */ 2607 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2608 if (wgs == NULL) { 2609 WG_TRACE("No session found"); 2610 return; 2611 } 2612 2613 /* Lock the peer so we can update the cookie state. */ 2614 wgp = wgs->wgs_peer; 2615 mutex_enter(wgp->wgp_lock); 2616 2617 if (!wgp->wgp_last_sent_mac1_valid) { 2618 WG_TRACE("No valid mac1 sent (or expired)"); 2619 goto out; 2620 } 2621 2622 /* Decrypt the cookie and store it for later handshake retry. */ 2623 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2624 sizeof(wgp->wgp_pubkey)); 2625 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2626 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2627 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2628 wgmc->wgmc_salt); 2629 if (error != 0) { 2630 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2631 "wg_algo_aead_dec for cookie failed: error=%d\n", error); 2632 goto out; 2633 } 2634 /* 2635 * [W] 6.6: Interaction with Cookie Reply System 2636 * "it should simply store the decrypted cookie value from the cookie 2637 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2638 * timer for retrying a handshake initiation message." 2639 */ 2640 wgp->wgp_latest_cookie_time = time_uptime; 2641 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2642 out: 2643 mutex_exit(wgp->wgp_lock); 2644 wg_put_session(wgs, &psref); 2645 } 2646 2647 static struct mbuf * 2648 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2649 { 2650 struct wg_msg wgm; 2651 size_t mbuflen; 2652 size_t msglen; 2653 2654 /* 2655 * Get the mbuf chain length. It is already guaranteed, by 2656 * wg_overudp_cb, to be large enough for a struct wg_msg. 2657 */ 2658 mbuflen = m_length(m); 2659 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2660 2661 /* 2662 * Copy the message header (32-bit message type) out -- we'll 2663 * worry about contiguity and alignment later. 2664 */ 2665 m_copydata(m, 0, sizeof(wgm), &wgm); 2666 switch (le32toh(wgm.wgm_type)) { 2667 case WG_MSG_TYPE_INIT: 2668 msglen = sizeof(struct wg_msg_init); 2669 break; 2670 case WG_MSG_TYPE_RESP: 2671 msglen = sizeof(struct wg_msg_resp); 2672 break; 2673 case WG_MSG_TYPE_COOKIE: 2674 msglen = sizeof(struct wg_msg_cookie); 2675 break; 2676 case WG_MSG_TYPE_DATA: 2677 msglen = sizeof(struct wg_msg_data); 2678 break; 2679 default: 2680 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2681 "Unexpected msg type: %u\n", le32toh(wgm.wgm_type)); 2682 goto error; 2683 } 2684 2685 /* Verify the mbuf chain is long enough for this type of message. */ 2686 if (__predict_false(mbuflen < msglen)) { 2687 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2688 le32toh(wgm.wgm_type)); 2689 goto error; 2690 } 2691 2692 /* Make the message header contiguous if necessary. */ 2693 if (__predict_false(m->m_len < msglen)) { 2694 m = m_pullup(m, msglen); 2695 if (m == NULL) 2696 return NULL; 2697 } 2698 2699 return m; 2700 2701 error: 2702 m_freem(m); 2703 return NULL; 2704 } 2705 2706 static void 2707 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2708 const struct sockaddr *src) 2709 { 2710 struct wg_msg *wgm; 2711 2712 m = wg_validate_msg_header(wg, m); 2713 if (__predict_false(m == NULL)) 2714 return; 2715 2716 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2717 wgm = mtod(m, struct wg_msg *); 2718 switch (le32toh(wgm->wgm_type)) { 2719 case WG_MSG_TYPE_INIT: 2720 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2721 break; 2722 case WG_MSG_TYPE_RESP: 2723 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2724 break; 2725 case WG_MSG_TYPE_COOKIE: 2726 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2727 break; 2728 case WG_MSG_TYPE_DATA: 2729 wg_handle_msg_data(wg, m, src); 2730 /* wg_handle_msg_data frees m for us */ 2731 return; 2732 default: 2733 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2734 } 2735 2736 m_freem(m); 2737 } 2738 2739 static void 2740 wg_receive_packets(struct wg_softc *wg, const int af) 2741 { 2742 2743 for (;;) { 2744 int error, flags; 2745 struct socket *so; 2746 struct mbuf *m = NULL; 2747 struct uio dummy_uio; 2748 struct mbuf *paddr = NULL; 2749 struct sockaddr *src; 2750 2751 so = wg_get_so_by_af(wg->wg_worker, af); 2752 flags = MSG_DONTWAIT; 2753 dummy_uio.uio_resid = 1000000000; 2754 2755 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2756 &flags); 2757 if (error || m == NULL) { 2758 //if (error == EWOULDBLOCK) 2759 return; 2760 } 2761 2762 KASSERT(paddr != NULL); 2763 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2764 src = mtod(paddr, struct sockaddr *); 2765 2766 wg_handle_packet(wg, m, src); 2767 } 2768 } 2769 2770 static void 2771 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2772 { 2773 2774 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2775 } 2776 2777 static void 2778 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2779 { 2780 2781 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 2782 } 2783 2784 static void 2785 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 2786 { 2787 struct wg_session *wgs; 2788 2789 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 2790 2791 KASSERT(mutex_owned(wgp->wgp_lock)); 2792 2793 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 2794 WGLOG(LOG_DEBUG, "No endpoint available\n"); 2795 /* XXX should do something? */ 2796 return; 2797 } 2798 2799 wgs = wgp->wgp_session_stable; 2800 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 2801 /* XXX What if the unstable session is already INIT_ACTIVE? */ 2802 wg_send_handshake_msg_init(wg, wgp); 2803 } else { 2804 /* rekey */ 2805 wgs = wgp->wgp_session_unstable; 2806 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2807 wg_send_handshake_msg_init(wg, wgp); 2808 } 2809 } 2810 2811 static void 2812 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 2813 { 2814 struct wg_session *wgs; 2815 2816 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 2817 2818 KASSERT(mutex_owned(wgp->wgp_lock)); 2819 KASSERT(wgp->wgp_handshake_start_time != 0); 2820 2821 wgs = wgp->wgp_session_unstable; 2822 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2823 return; 2824 2825 /* 2826 * XXX no real need to assign a new index here, but we do need 2827 * to transition to UNKNOWN temporarily 2828 */ 2829 wg_put_session_index(wg, wgs); 2830 2831 /* [W] 6.4 Handshake Initiation Retransmission */ 2832 if ((time_uptime - wgp->wgp_handshake_start_time) > 2833 wg_rekey_attempt_time) { 2834 /* Give up handshaking */ 2835 wgp->wgp_handshake_start_time = 0; 2836 WG_TRACE("give up"); 2837 2838 /* 2839 * If a new data packet comes, handshaking will be retried 2840 * and a new session would be established at that time, 2841 * however we don't want to send pending packets then. 2842 */ 2843 wg_purge_pending_packets(wgp); 2844 return; 2845 } 2846 2847 wg_task_send_init_message(wg, wgp); 2848 } 2849 2850 static void 2851 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 2852 { 2853 struct wg_session *wgs, *wgs_prev; 2854 2855 KASSERT(mutex_owned(wgp->wgp_lock)); 2856 2857 wgs = wgp->wgp_session_unstable; 2858 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 2859 /* XXX Can this happen? */ 2860 return; 2861 2862 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2863 wgs->wgs_time_established = time_uptime; 2864 wgs->wgs_time_last_data_sent = 0; 2865 wgs->wgs_is_initiator = false; 2866 WG_TRACE("WGS_STATE_ESTABLISHED"); 2867 2868 wg_swap_sessions(wgp); 2869 KASSERT(wgs == wgp->wgp_session_stable); 2870 wgs_prev = wgp->wgp_session_unstable; 2871 getnanotime(&wgp->wgp_last_handshake_time); 2872 wgp->wgp_handshake_start_time = 0; 2873 wgp->wgp_last_sent_mac1_valid = false; 2874 wgp->wgp_last_sent_cookie_valid = false; 2875 2876 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2877 /* Wait for wg_get_stable_session to drain. */ 2878 pserialize_perform(wgp->wgp_psz); 2879 2880 /* Transition ESTABLISHED->DESTROYING. */ 2881 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2882 2883 /* We can't destroy the old session immediately */ 2884 wg_schedule_session_dtor_timer(wgp); 2885 } else { 2886 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2887 "state=%d", wgs_prev->wgs_state); 2888 wg_clear_states(wgs_prev); 2889 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 2890 } 2891 2892 /* Anyway run a softint to flush pending packets */ 2893 kpreempt_disable(); 2894 softint_schedule(wgp->wgp_si); 2895 kpreempt_enable(); 2896 } 2897 2898 static void 2899 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 2900 { 2901 2902 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 2903 2904 KASSERT(mutex_owned(wgp->wgp_lock)); 2905 2906 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 2907 pserialize_perform(wgp->wgp_psz); 2908 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 2909 wg_psref_class); 2910 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 2911 wg_psref_class); 2912 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 2913 } 2914 } 2915 2916 static void 2917 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 2918 { 2919 struct wg_session *wgs; 2920 2921 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 2922 2923 KASSERT(mutex_owned(wgp->wgp_lock)); 2924 2925 wgs = wgp->wgp_session_stable; 2926 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 2927 return; 2928 2929 wg_send_keepalive_msg(wgp, wgs); 2930 } 2931 2932 static void 2933 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 2934 { 2935 struct wg_session *wgs; 2936 2937 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 2938 2939 KASSERT(mutex_owned(wgp->wgp_lock)); 2940 2941 wgs = wgp->wgp_session_unstable; 2942 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 2943 wg_put_session_index(wg, wgs); 2944 } 2945 } 2946 2947 static void 2948 wg_process_peer_tasks(struct wg_softc *wg) 2949 { 2950 struct wg_peer *wgp; 2951 int s; 2952 2953 /* XXX should avoid checking all peers */ 2954 s = pserialize_read_enter(); 2955 WG_PEER_READER_FOREACH(wgp, wg) { 2956 struct psref psref; 2957 unsigned int tasks; 2958 2959 if (wgp->wgp_tasks == 0) 2960 continue; 2961 2962 wg_get_peer(wgp, &psref); 2963 pserialize_read_exit(s); 2964 2965 restart: 2966 tasks = atomic_swap_uint(&wgp->wgp_tasks, 0); 2967 KASSERT(tasks != 0); 2968 2969 WG_DLOG("tasks=%x\n", tasks); 2970 2971 mutex_enter(wgp->wgp_lock); 2972 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 2973 wg_task_send_init_message(wg, wgp); 2974 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 2975 wg_task_retry_handshake(wg, wgp); 2976 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 2977 wg_task_establish_session(wg, wgp); 2978 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 2979 wg_task_endpoint_changed(wg, wgp); 2980 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 2981 wg_task_send_keepalive_message(wg, wgp); 2982 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 2983 wg_task_destroy_prev_session(wg, wgp); 2984 mutex_exit(wgp->wgp_lock); 2985 2986 /* New tasks may be scheduled during processing tasks */ 2987 WG_DLOG("wgp_tasks=%d\n", wgp->wgp_tasks); 2988 if (wgp->wgp_tasks != 0) 2989 goto restart; 2990 2991 s = pserialize_read_enter(); 2992 wg_put_peer(wgp, &psref); 2993 } 2994 pserialize_read_exit(s); 2995 } 2996 2997 static void 2998 wg_worker(void *arg) 2999 { 3000 struct wg_softc *wg = arg; 3001 struct wg_worker *wgw = wg->wg_worker; 3002 bool todie = false; 3003 3004 KASSERT(wg != NULL); 3005 KASSERT(wgw != NULL); 3006 3007 while (!todie) { 3008 int reasons; 3009 int bound; 3010 3011 mutex_enter(&wgw->wgw_lock); 3012 /* New tasks may come during task handling */ 3013 while ((reasons = wgw->wgw_wakeup_reasons) == 0 && 3014 !(todie = wgw->wgw_todie)) 3015 cv_wait(&wgw->wgw_cv, &wgw->wgw_lock); 3016 wgw->wgw_wakeup_reasons = 0; 3017 mutex_exit(&wgw->wgw_lock); 3018 3019 bound = curlwp_bind(); 3020 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4)) 3021 wg_receive_packets(wg, AF_INET); 3022 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6)) 3023 wg_receive_packets(wg, AF_INET6); 3024 if (ISSET(reasons, WG_WAKEUP_REASON_PEER)) 3025 wg_process_peer_tasks(wg); 3026 curlwp_bindx(bound); 3027 } 3028 kthread_exit(0); 3029 } 3030 3031 static void 3032 wg_wakeup_worker(struct wg_worker *wgw, const int reason) 3033 { 3034 3035 mutex_enter(&wgw->wgw_lock); 3036 wgw->wgw_wakeup_reasons |= reason; 3037 cv_broadcast(&wgw->wgw_cv); 3038 mutex_exit(&wgw->wgw_lock); 3039 } 3040 3041 static int 3042 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3043 { 3044 int error; 3045 struct wg_worker *wgw = wg->wg_worker; 3046 uint16_t old_port = wg->wg_listen_port; 3047 3048 if (port != 0 && old_port == port) 3049 return 0; 3050 3051 struct sockaddr_in _sin, *sin = &_sin; 3052 sin->sin_len = sizeof(*sin); 3053 sin->sin_family = AF_INET; 3054 sin->sin_addr.s_addr = INADDR_ANY; 3055 sin->sin_port = htons(port); 3056 3057 error = sobind(wgw->wgw_so4, sintosa(sin), curlwp); 3058 if (error != 0) 3059 return error; 3060 3061 #ifdef INET6 3062 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3063 sin6->sin6_len = sizeof(*sin6); 3064 sin6->sin6_family = AF_INET6; 3065 sin6->sin6_addr = in6addr_any; 3066 sin6->sin6_port = htons(port); 3067 3068 error = sobind(wgw->wgw_so6, sin6tosa(sin6), curlwp); 3069 if (error != 0) 3070 return error; 3071 #endif 3072 3073 wg->wg_listen_port = port; 3074 3075 return 0; 3076 } 3077 3078 static void 3079 wg_so_upcall(struct socket *so, void *arg, int events, int waitflag) 3080 { 3081 struct wg_worker *wgw = arg; 3082 int reason; 3083 3084 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3085 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 : 3086 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6; 3087 wg_wakeup_worker(wgw, reason); 3088 } 3089 3090 static int 3091 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3092 struct sockaddr *src, void *arg) 3093 { 3094 struct wg_softc *wg = arg; 3095 struct wg_msg wgm; 3096 struct mbuf *m = *mp; 3097 3098 WG_TRACE("enter"); 3099 3100 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3101 KASSERT(offset <= m_length(m)); 3102 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3103 /* drop on the floor */ 3104 m_freem(m); 3105 return -1; 3106 } 3107 3108 /* 3109 * Copy the message header (32-bit message type) out -- we'll 3110 * worry about contiguity and alignment later. 3111 */ 3112 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3113 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3114 3115 /* 3116 * Handle DATA packets promptly as they arrive. Other packets 3117 * may require expensive public-key crypto and are not as 3118 * sensitive to latency, so defer them to the worker thread. 3119 */ 3120 switch (le32toh(wgm.wgm_type)) { 3121 case WG_MSG_TYPE_DATA: 3122 /* handle immediately */ 3123 m_adj(m, offset); 3124 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3125 m = m_pullup(m, sizeof(struct wg_msg_data)); 3126 if (m == NULL) 3127 return -1; 3128 } 3129 wg_handle_msg_data(wg, m, src); 3130 *mp = NULL; 3131 return 1; 3132 case WG_MSG_TYPE_INIT: 3133 case WG_MSG_TYPE_RESP: 3134 case WG_MSG_TYPE_COOKIE: 3135 /* pass through to so_receive in wg_receive_packets */ 3136 return 0; 3137 default: 3138 /* drop on the floor */ 3139 m_freem(m); 3140 return -1; 3141 } 3142 } 3143 3144 static int 3145 wg_worker_socreate(struct wg_softc *wg, struct wg_worker *wgw, const int af, 3146 struct socket **sop) 3147 { 3148 int error; 3149 struct socket *so; 3150 3151 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3152 if (error != 0) 3153 return error; 3154 3155 solock(so); 3156 so->so_upcallarg = wgw; 3157 so->so_upcall = wg_so_upcall; 3158 so->so_rcv.sb_flags |= SB_UPCALL; 3159 if (af == AF_INET) 3160 in_pcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3161 #if INET6 3162 else 3163 in6_pcb_register_overudp_cb(sotoin6pcb(so), wg_overudp_cb, wg); 3164 #endif 3165 sounlock(so); 3166 3167 *sop = so; 3168 3169 return 0; 3170 } 3171 3172 static int 3173 wg_worker_init(struct wg_softc *wg) 3174 { 3175 int error; 3176 struct wg_worker *wgw; 3177 const char *ifname = wg->wg_if.if_xname; 3178 struct socket *so; 3179 3180 wgw = kmem_zalloc(sizeof(struct wg_worker), KM_SLEEP); 3181 3182 mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3183 cv_init(&wgw->wgw_cv, ifname); 3184 wgw->wgw_todie = false; 3185 wgw->wgw_wakeup_reasons = 0; 3186 3187 error = wg_worker_socreate(wg, wgw, AF_INET, &so); 3188 if (error != 0) 3189 goto error; 3190 wgw->wgw_so4 = so; 3191 #ifdef INET6 3192 error = wg_worker_socreate(wg, wgw, AF_INET6, &so); 3193 if (error != 0) 3194 goto error; 3195 wgw->wgw_so6 = so; 3196 #endif 3197 3198 wg->wg_worker = wgw; 3199 3200 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, 3201 NULL, wg_worker, wg, &wg->wg_worker_lwp, "%s", ifname); 3202 if (error != 0) 3203 goto error; 3204 3205 return 0; 3206 3207 error: 3208 #ifdef INET6 3209 if (wgw->wgw_so6 != NULL) 3210 soclose(wgw->wgw_so6); 3211 #endif 3212 if (wgw->wgw_so4 != NULL) 3213 soclose(wgw->wgw_so4); 3214 cv_destroy(&wgw->wgw_cv); 3215 mutex_destroy(&wgw->wgw_lock); 3216 3217 return error; 3218 } 3219 3220 static void 3221 wg_worker_destroy(struct wg_softc *wg) 3222 { 3223 struct wg_worker *wgw = wg->wg_worker; 3224 3225 mutex_enter(&wgw->wgw_lock); 3226 wgw->wgw_todie = true; 3227 wgw->wgw_wakeup_reasons = 0; 3228 cv_broadcast(&wgw->wgw_cv); 3229 mutex_exit(&wgw->wgw_lock); 3230 3231 kthread_join(wg->wg_worker_lwp); 3232 3233 #ifdef INET6 3234 soclose(wgw->wgw_so6); 3235 #endif 3236 soclose(wgw->wgw_so4); 3237 cv_destroy(&wgw->wgw_cv); 3238 mutex_destroy(&wgw->wgw_lock); 3239 kmem_free(wg->wg_worker, sizeof(struct wg_worker)); 3240 wg->wg_worker = NULL; 3241 } 3242 3243 static bool 3244 wg_session_hit_limits(struct wg_session *wgs) 3245 { 3246 3247 /* 3248 * [W] 6.2: Transport Message Limits 3249 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3250 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3251 * comes first, WireGuard will refuse to send any more transport data 3252 * messages using the current secure session, ..." 3253 */ 3254 KASSERT(wgs->wgs_time_established != 0); 3255 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3256 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3257 return true; 3258 } else if (wg_session_get_send_counter(wgs) > 3259 wg_reject_after_messages) { 3260 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3261 return true; 3262 } 3263 3264 return false; 3265 } 3266 3267 static void 3268 wg_peer_softint(void *arg) 3269 { 3270 struct wg_peer *wgp = arg; 3271 struct wg_session *wgs; 3272 struct mbuf *m; 3273 struct psref psref; 3274 3275 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3276 /* XXX how to treat? */ 3277 WG_TRACE("skipped"); 3278 return; 3279 } 3280 if (wg_session_hit_limits(wgs)) { 3281 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3282 goto out; 3283 } 3284 WG_TRACE("running"); 3285 3286 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3287 wg_send_data_msg(wgp, wgs, m); 3288 } 3289 out: 3290 wg_put_session(wgs, &psref); 3291 } 3292 3293 static void 3294 wg_rekey_timer(void *arg) 3295 { 3296 struct wg_peer *wgp = arg; 3297 3298 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3299 } 3300 3301 static void 3302 wg_purge_pending_packets(struct wg_peer *wgp) 3303 { 3304 struct mbuf *m; 3305 3306 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3307 m_freem(m); 3308 } 3309 } 3310 3311 static void 3312 wg_handshake_timeout_timer(void *arg) 3313 { 3314 struct wg_peer *wgp = arg; 3315 3316 WG_TRACE("enter"); 3317 3318 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3319 } 3320 3321 static struct wg_peer * 3322 wg_alloc_peer(struct wg_softc *wg) 3323 { 3324 struct wg_peer *wgp; 3325 3326 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3327 3328 wgp->wgp_sc = wg; 3329 wgp->wgp_q = pcq_create(1024, KM_SLEEP); 3330 wgp->wgp_si = softint_establish(SOFTINT_NET, wg_peer_softint, wgp); 3331 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3332 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3333 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3334 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3335 wg_handshake_timeout_timer, wgp); 3336 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3337 callout_setfunc(&wgp->wgp_session_dtor_timer, 3338 wg_session_dtor_timer, wgp); 3339 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3340 wgp->wgp_endpoint_changing = false; 3341 wgp->wgp_endpoint_available = false; 3342 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3343 wgp->wgp_psz = pserialize_create(); 3344 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3345 3346 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3347 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3348 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3349 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3350 3351 struct wg_session *wgs; 3352 wgp->wgp_session_stable = 3353 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3354 wgp->wgp_session_unstable = 3355 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3356 wgs = wgp->wgp_session_stable; 3357 wgs->wgs_peer = wgp; 3358 wgs->wgs_state = WGS_STATE_UNKNOWN; 3359 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3360 #ifndef __HAVE_ATOMIC64_LOADSTORE 3361 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3362 #endif 3363 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3364 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3365 3366 wgs = wgp->wgp_session_unstable; 3367 wgs->wgs_peer = wgp; 3368 wgs->wgs_state = WGS_STATE_UNKNOWN; 3369 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3370 #ifndef __HAVE_ATOMIC64_LOADSTORE 3371 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3372 #endif 3373 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3374 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3375 3376 return wgp; 3377 } 3378 3379 static void 3380 wg_destroy_peer(struct wg_peer *wgp) 3381 { 3382 struct wg_session *wgs; 3383 struct wg_softc *wg = wgp->wgp_sc; 3384 3385 /* Prevent new packets from this peer on any source address. */ 3386 rw_enter(wg->wg_rwlock, RW_WRITER); 3387 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3388 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3389 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3390 struct radix_node *rn; 3391 3392 KASSERT(rnh != NULL); 3393 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3394 &wga->wga_sa_mask, rnh); 3395 if (rn == NULL) { 3396 char addrstr[128]; 3397 sockaddr_format(&wga->wga_sa_addr, addrstr, 3398 sizeof(addrstr)); 3399 WGLOG(LOG_WARNING, "Couldn't delete %s", addrstr); 3400 } 3401 } 3402 rw_exit(wg->wg_rwlock); 3403 3404 /* Purge pending packets. */ 3405 wg_purge_pending_packets(wgp); 3406 3407 /* Halt all packet processing and timeouts. */ 3408 softint_disestablish(wgp->wgp_si); 3409 callout_halt(&wgp->wgp_rekey_timer, NULL); 3410 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3411 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3412 3413 wgs = wgp->wgp_session_unstable; 3414 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3415 mutex_enter(wgp->wgp_lock); 3416 wg_destroy_session(wg, wgs); 3417 mutex_exit(wgp->wgp_lock); 3418 } 3419 mutex_destroy(&wgs->wgs_recvwin->lock); 3420 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3421 #ifndef __HAVE_ATOMIC64_LOADSTORE 3422 mutex_destroy(&wgs->wgs_send_counter_lock); 3423 #endif 3424 kmem_free(wgs, sizeof(*wgs)); 3425 3426 wgs = wgp->wgp_session_stable; 3427 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3428 mutex_enter(wgp->wgp_lock); 3429 wg_destroy_session(wg, wgs); 3430 mutex_exit(wgp->wgp_lock); 3431 } 3432 mutex_destroy(&wgs->wgs_recvwin->lock); 3433 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3434 #ifndef __HAVE_ATOMIC64_LOADSTORE 3435 mutex_destroy(&wgs->wgs_send_counter_lock); 3436 #endif 3437 kmem_free(wgs, sizeof(*wgs)); 3438 3439 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3440 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3441 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3442 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3443 3444 pserialize_destroy(wgp->wgp_psz); 3445 pcq_destroy(wgp->wgp_q); 3446 mutex_obj_free(wgp->wgp_lock); 3447 3448 kmem_free(wgp, sizeof(*wgp)); 3449 } 3450 3451 static void 3452 wg_destroy_all_peers(struct wg_softc *wg) 3453 { 3454 struct wg_peer *wgp, *wgp0 __diagused; 3455 void *garbage_byname, *garbage_bypubkey; 3456 3457 restart: 3458 garbage_byname = garbage_bypubkey = NULL; 3459 mutex_enter(wg->wg_lock); 3460 WG_PEER_WRITER_FOREACH(wgp, wg) { 3461 if (wgp->wgp_name[0]) { 3462 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3463 strlen(wgp->wgp_name)); 3464 KASSERT(wgp0 == wgp); 3465 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3466 } 3467 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3468 sizeof(wgp->wgp_pubkey)); 3469 KASSERT(wgp0 == wgp); 3470 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3471 WG_PEER_WRITER_REMOVE(wgp); 3472 wg->wg_npeers--; 3473 mutex_enter(wgp->wgp_lock); 3474 pserialize_perform(wgp->wgp_psz); 3475 mutex_exit(wgp->wgp_lock); 3476 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3477 break; 3478 } 3479 mutex_exit(wg->wg_lock); 3480 3481 if (wgp == NULL) 3482 return; 3483 3484 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3485 3486 wg_destroy_peer(wgp); 3487 thmap_gc(wg->wg_peers_byname, garbage_byname); 3488 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3489 3490 goto restart; 3491 } 3492 3493 static int 3494 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3495 { 3496 struct wg_peer *wgp, *wgp0 __diagused; 3497 void *garbage_byname, *garbage_bypubkey; 3498 3499 mutex_enter(wg->wg_lock); 3500 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3501 if (wgp != NULL) { 3502 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3503 sizeof(wgp->wgp_pubkey)); 3504 KASSERT(wgp0 == wgp); 3505 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3506 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3507 WG_PEER_WRITER_REMOVE(wgp); 3508 wg->wg_npeers--; 3509 mutex_enter(wgp->wgp_lock); 3510 pserialize_perform(wgp->wgp_psz); 3511 mutex_exit(wgp->wgp_lock); 3512 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3513 } 3514 mutex_exit(wg->wg_lock); 3515 3516 if (wgp == NULL) 3517 return ENOENT; 3518 3519 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3520 3521 wg_destroy_peer(wgp); 3522 thmap_gc(wg->wg_peers_byname, garbage_byname); 3523 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3524 3525 return 0; 3526 } 3527 3528 static int 3529 wg_if_attach(struct wg_softc *wg) 3530 { 3531 int error; 3532 3533 wg->wg_if.if_addrlen = 0; 3534 wg->wg_if.if_mtu = WG_MTU; 3535 wg->wg_if.if_flags = IFF_MULTICAST; 3536 wg->wg_if.if_extflags = IFEF_NO_LINK_STATE_CHANGE; 3537 wg->wg_if.if_extflags |= IFEF_MPSAFE; 3538 wg->wg_if.if_ioctl = wg_ioctl; 3539 wg->wg_if.if_output = wg_output; 3540 wg->wg_if.if_init = wg_init; 3541 wg->wg_if.if_stop = wg_stop; 3542 wg->wg_if.if_type = IFT_OTHER; 3543 wg->wg_if.if_dlt = DLT_NULL; 3544 wg->wg_if.if_softc = wg; 3545 IFQ_SET_READY(&wg->wg_if.if_snd); 3546 3547 error = if_initialize(&wg->wg_if); 3548 if (error != 0) 3549 return error; 3550 3551 if_alloc_sadl(&wg->wg_if); 3552 if_register(&wg->wg_if); 3553 3554 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3555 3556 return 0; 3557 } 3558 3559 static int 3560 wg_clone_create(struct if_clone *ifc, int unit) 3561 { 3562 struct wg_softc *wg; 3563 int error; 3564 3565 wg = kmem_zalloc(sizeof(struct wg_softc), KM_SLEEP); 3566 3567 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3568 3569 error = wg_worker_init(wg); 3570 if (error != 0) { 3571 kmem_free(wg, sizeof(struct wg_softc)); 3572 return error; 3573 } 3574 3575 rn_inithead((void **)&wg->wg_rtable_ipv4, 3576 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3577 #ifdef INET6 3578 rn_inithead((void **)&wg->wg_rtable_ipv6, 3579 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3580 #endif 3581 3582 PSLIST_INIT(&wg->wg_peers); 3583 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3584 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3585 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3586 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3587 wg->wg_rwlock = rw_obj_alloc(); 3588 wg->wg_ops = &wg_ops_rumpkernel; 3589 3590 error = wg_if_attach(wg); 3591 if (error != 0) { 3592 wg_worker_destroy(wg); 3593 if (wg->wg_rtable_ipv4 != NULL) 3594 free(wg->wg_rtable_ipv4, M_RTABLE); 3595 if (wg->wg_rtable_ipv6 != NULL) 3596 free(wg->wg_rtable_ipv6, M_RTABLE); 3597 PSLIST_DESTROY(&wg->wg_peers); 3598 mutex_obj_free(wg->wg_lock); 3599 thmap_destroy(wg->wg_sessions_byindex); 3600 thmap_destroy(wg->wg_peers_byname); 3601 thmap_destroy(wg->wg_peers_bypubkey); 3602 kmem_free(wg, sizeof(struct wg_softc)); 3603 return error; 3604 } 3605 3606 mutex_enter(&wg_softcs.lock); 3607 LIST_INSERT_HEAD(&wg_softcs.list, wg, wg_list); 3608 mutex_exit(&wg_softcs.lock); 3609 3610 return 0; 3611 } 3612 3613 static int 3614 wg_clone_destroy(struct ifnet *ifp) 3615 { 3616 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3617 3618 mutex_enter(&wg_softcs.lock); 3619 LIST_REMOVE(wg, wg_list); 3620 mutex_exit(&wg_softcs.lock); 3621 3622 #ifdef WG_RUMPKERNEL 3623 if (wg_user_mode(wg)) { 3624 rumpuser_wg_destroy(wg->wg_user); 3625 wg->wg_user = NULL; 3626 } 3627 #endif 3628 3629 bpf_detach(ifp); 3630 if_detach(ifp); 3631 wg_worker_destroy(wg); 3632 wg_destroy_all_peers(wg); 3633 if (wg->wg_rtable_ipv4 != NULL) 3634 free(wg->wg_rtable_ipv4, M_RTABLE); 3635 if (wg->wg_rtable_ipv6 != NULL) 3636 free(wg->wg_rtable_ipv6, M_RTABLE); 3637 3638 PSLIST_DESTROY(&wg->wg_peers); 3639 thmap_destroy(wg->wg_sessions_byindex); 3640 thmap_destroy(wg->wg_peers_byname); 3641 thmap_destroy(wg->wg_peers_bypubkey); 3642 mutex_obj_free(wg->wg_lock); 3643 rw_obj_free(wg->wg_rwlock); 3644 3645 kmem_free(wg, sizeof(struct wg_softc)); 3646 3647 return 0; 3648 } 3649 3650 static struct wg_peer * 3651 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3652 struct psref *psref) 3653 { 3654 struct radix_node_head *rnh; 3655 struct radix_node *rn; 3656 struct wg_peer *wgp = NULL; 3657 struct wg_allowedip *wga; 3658 3659 #ifdef WG_DEBUG_LOG 3660 char addrstr[128]; 3661 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3662 WG_DLOG("sa=%s\n", addrstr); 3663 #endif 3664 3665 rw_enter(wg->wg_rwlock, RW_READER); 3666 3667 rnh = wg_rnh(wg, sa->sa_family); 3668 if (rnh == NULL) 3669 goto out; 3670 3671 rn = rnh->rnh_matchaddr(sa, rnh); 3672 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3673 goto out; 3674 3675 WG_TRACE("success"); 3676 3677 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3678 wgp = wga->wga_peer; 3679 wg_get_peer(wgp, psref); 3680 3681 out: 3682 rw_exit(wg->wg_rwlock); 3683 return wgp; 3684 } 3685 3686 static void 3687 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3688 struct wg_session *wgs, struct wg_msg_data *wgmd) 3689 { 3690 3691 memset(wgmd, 0, sizeof(*wgmd)); 3692 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3693 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3694 /* [W] 5.4.6: msg.counter := Nm^send */ 3695 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3696 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3697 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3698 } 3699 3700 static int 3701 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3702 const struct rtentry *rt) 3703 { 3704 struct wg_softc *wg = ifp->if_softc; 3705 struct wg_peer *wgp = NULL; 3706 struct wg_session *wgs = NULL; 3707 struct psref wgp_psref, wgs_psref; 3708 int bound; 3709 int error; 3710 3711 bound = curlwp_bind(); 3712 3713 /* TODO make the nest limit configurable via sysctl */ 3714 error = if_tunnel_check_nesting(ifp, m, 1); 3715 if (error) { 3716 WGLOG(LOG_ERR, "tunneling loop detected and packet dropped\n"); 3717 goto out; 3718 } 3719 3720 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3721 3722 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3723 3724 m->m_flags &= ~(M_BCAST|M_MCAST); 3725 3726 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3727 if (wgp == NULL) { 3728 WG_TRACE("peer not found"); 3729 error = EHOSTUNREACH; 3730 goto out; 3731 } 3732 3733 /* Clear checksum-offload flags. */ 3734 m->m_pkthdr.csum_flags = 0; 3735 m->m_pkthdr.csum_data = 0; 3736 3737 if (!pcq_put(wgp->wgp_q, m)) { 3738 error = ENOBUFS; 3739 goto out; 3740 } 3741 m = NULL; /* consumed */ 3742 3743 wgs = wg_get_stable_session(wgp, &wgs_psref); 3744 if (wgs != NULL && !wg_session_hit_limits(wgs)) { 3745 kpreempt_disable(); 3746 softint_schedule(wgp->wgp_si); 3747 kpreempt_enable(); 3748 WG_TRACE("softint scheduled"); 3749 } else { 3750 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3751 WG_TRACE("softint NOT scheduled"); 3752 } 3753 error = 0; 3754 3755 out: 3756 if (wgs != NULL) 3757 wg_put_session(wgs, &wgs_psref); 3758 if (wgp != NULL) 3759 wg_put_peer(wgp, &wgp_psref); 3760 if (m != NULL) 3761 m_freem(m); 3762 curlwp_bindx(bound); 3763 return error; 3764 } 3765 3766 static int 3767 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3768 { 3769 struct psref psref; 3770 struct wg_sockaddr *wgsa; 3771 int error; 3772 struct socket *so; 3773 3774 wgsa = wg_get_endpoint_sa(wgp, &psref); 3775 so = wg_get_so_by_peer(wgp, wgsa); 3776 solock(so); 3777 if (wgsatosa(wgsa)->sa_family == AF_INET) { 3778 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 3779 } else { 3780 #ifdef INET6 3781 error = udp6_output(sotoin6pcb(so), m, wgsatosin6(wgsa), 3782 NULL, curlwp); 3783 #else 3784 m_freem(m); 3785 error = EPFNOSUPPORT; 3786 #endif 3787 } 3788 sounlock(so); 3789 wg_put_sa(wgp, wgsa, &psref); 3790 3791 return error; 3792 } 3793 3794 /* Inspired by pppoe_get_mbuf */ 3795 static struct mbuf * 3796 wg_get_mbuf(size_t leading_len, size_t len) 3797 { 3798 struct mbuf *m; 3799 3800 KASSERT(leading_len <= MCLBYTES); 3801 KASSERT(len <= MCLBYTES - leading_len); 3802 3803 m = m_gethdr(M_DONTWAIT, MT_DATA); 3804 if (m == NULL) 3805 return NULL; 3806 if (len + leading_len > MHLEN) { 3807 m_clget(m, M_DONTWAIT); 3808 if ((m->m_flags & M_EXT) == 0) { 3809 m_free(m); 3810 return NULL; 3811 } 3812 } 3813 m->m_data += leading_len; 3814 m->m_pkthdr.len = m->m_len = len; 3815 3816 return m; 3817 } 3818 3819 static int 3820 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 3821 struct mbuf *m) 3822 { 3823 struct wg_softc *wg = wgp->wgp_sc; 3824 int error; 3825 size_t inner_len, padded_len, encrypted_len; 3826 char *padded_buf = NULL; 3827 size_t mlen; 3828 struct wg_msg_data *wgmd; 3829 bool free_padded_buf = false; 3830 struct mbuf *n; 3831 size_t leading_len = max_linkhdr + sizeof(struct ip6_hdr) + 3832 sizeof(struct udphdr); 3833 3834 mlen = m_length(m); 3835 inner_len = mlen; 3836 padded_len = roundup(mlen, 16); 3837 encrypted_len = padded_len + WG_AUTHTAG_LEN; 3838 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 3839 inner_len, padded_len, encrypted_len); 3840 if (mlen != 0) { 3841 bool success; 3842 success = m_ensure_contig(&m, padded_len); 3843 if (success) { 3844 padded_buf = mtod(m, char *); 3845 } else { 3846 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 3847 if (padded_buf == NULL) { 3848 error = ENOBUFS; 3849 goto end; 3850 } 3851 free_padded_buf = true; 3852 m_copydata(m, 0, mlen, padded_buf); 3853 } 3854 memset(padded_buf + mlen, 0, padded_len - inner_len); 3855 } 3856 3857 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 3858 if (n == NULL) { 3859 error = ENOBUFS; 3860 goto end; 3861 } 3862 KASSERT(n->m_len >= sizeof(*wgmd)); 3863 wgmd = mtod(n, struct wg_msg_data *); 3864 wg_fill_msg_data(wg, wgp, wgs, wgmd); 3865 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 3866 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 3867 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 3868 padded_buf, padded_len, 3869 NULL, 0); 3870 3871 error = wg->wg_ops->send_data_msg(wgp, n); 3872 if (error == 0) { 3873 struct ifnet *ifp = &wg->wg_if; 3874 if_statadd(ifp, if_obytes, mlen); 3875 if_statinc(ifp, if_opackets); 3876 if (wgs->wgs_is_initiator && 3877 wgs->wgs_time_last_data_sent == 0) { 3878 /* 3879 * [W] 6.2 Transport Message Limits 3880 * "if a peer is the initiator of a current secure 3881 * session, WireGuard will send a handshake initiation 3882 * message to begin a new secure session if, after 3883 * transmitting a transport data message, the current 3884 * secure session is REKEY-AFTER-TIME seconds old," 3885 */ 3886 wg_schedule_rekey_timer(wgp); 3887 } 3888 wgs->wgs_time_last_data_sent = time_uptime; 3889 if (wg_session_get_send_counter(wgs) >= 3890 wg_rekey_after_messages) { 3891 /* 3892 * [W] 6.2 Transport Message Limits 3893 * "WireGuard will try to create a new session, by 3894 * sending a handshake initiation message (section 3895 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 3896 * transport data messages..." 3897 */ 3898 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3899 } 3900 } 3901 end: 3902 m_freem(m); 3903 if (free_padded_buf) 3904 kmem_intr_free(padded_buf, padded_len); 3905 return error; 3906 } 3907 3908 static void 3909 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 3910 { 3911 pktqueue_t *pktq; 3912 size_t pktlen; 3913 3914 KASSERT(af == AF_INET || af == AF_INET6); 3915 3916 WG_TRACE(""); 3917 3918 m_set_rcvif(m, ifp); 3919 pktlen = m->m_pkthdr.len; 3920 3921 bpf_mtap_af(ifp, af, m, BPF_D_IN); 3922 3923 switch (af) { 3924 case AF_INET: 3925 pktq = ip_pktq; 3926 break; 3927 #ifdef INET6 3928 case AF_INET6: 3929 pktq = ip6_pktq; 3930 break; 3931 #endif 3932 default: 3933 panic("invalid af=%d", af); 3934 } 3935 3936 const u_int h = curcpu()->ci_index; 3937 if (__predict_true(pktq_enqueue(pktq, m, h))) { 3938 if_statadd(ifp, if_ibytes, pktlen); 3939 if_statinc(ifp, if_ipackets); 3940 } else { 3941 m_freem(m); 3942 } 3943 } 3944 3945 static void 3946 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 3947 const uint8_t privkey[WG_STATIC_KEY_LEN]) 3948 { 3949 3950 crypto_scalarmult_base(pubkey, privkey); 3951 } 3952 3953 static int 3954 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 3955 { 3956 struct radix_node_head *rnh; 3957 struct radix_node *rn; 3958 int error = 0; 3959 3960 rw_enter(wg->wg_rwlock, RW_WRITER); 3961 rnh = wg_rnh(wg, wga->wga_family); 3962 KASSERT(rnh != NULL); 3963 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 3964 wga->wga_nodes); 3965 rw_exit(wg->wg_rwlock); 3966 3967 if (rn == NULL) 3968 error = EEXIST; 3969 3970 return error; 3971 } 3972 3973 static int 3974 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 3975 struct wg_peer **wgpp) 3976 { 3977 int error = 0; 3978 const void *pubkey; 3979 size_t pubkey_len; 3980 const void *psk; 3981 size_t psk_len; 3982 const char *name = NULL; 3983 3984 if (prop_dictionary_get_string(peer, "name", &name)) { 3985 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 3986 error = EINVAL; 3987 goto out; 3988 } 3989 } 3990 3991 if (!prop_dictionary_get_data(peer, "public_key", 3992 &pubkey, &pubkey_len)) { 3993 error = EINVAL; 3994 goto out; 3995 } 3996 #ifdef WG_DEBUG_DUMP 3997 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n", pubkey, pubkey_len); 3998 for (int _i = 0; _i < pubkey_len; _i++) 3999 log(LOG_DEBUG, "%c", ((const char *)pubkey)[_i]); 4000 log(LOG_DEBUG, "\n"); 4001 #endif 4002 4003 struct wg_peer *wgp = wg_alloc_peer(wg); 4004 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4005 if (name != NULL) 4006 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4007 4008 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4009 if (psk_len != sizeof(wgp->wgp_psk)) { 4010 error = EINVAL; 4011 goto out; 4012 } 4013 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4014 } 4015 4016 const void *addr; 4017 size_t addr_len; 4018 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4019 4020 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4021 goto skip_endpoint; 4022 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4023 addr_len > sizeof(*wgsatoss(wgsa))) { 4024 error = EINVAL; 4025 goto out; 4026 } 4027 memcpy(wgsatoss(wgsa), addr, addr_len); 4028 switch (wgsa_family(wgsa)) { 4029 case AF_INET: 4030 #ifdef INET6 4031 case AF_INET6: 4032 #endif 4033 break; 4034 default: 4035 error = EPFNOSUPPORT; 4036 goto out; 4037 } 4038 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4039 error = EINVAL; 4040 goto out; 4041 } 4042 { 4043 char addrstr[128]; 4044 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4045 WG_DLOG("addr=%s\n", addrstr); 4046 } 4047 wgp->wgp_endpoint_available = true; 4048 4049 prop_array_t allowedips; 4050 skip_endpoint: 4051 allowedips = prop_dictionary_get(peer, "allowedips"); 4052 if (allowedips == NULL) 4053 goto skip; 4054 4055 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4056 prop_dictionary_t prop_allowedip; 4057 int j = 0; 4058 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4059 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4060 4061 if (!prop_dictionary_get_int(prop_allowedip, "family", 4062 &wga->wga_family)) 4063 continue; 4064 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4065 &addr, &addr_len)) 4066 continue; 4067 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4068 &wga->wga_cidr)) 4069 continue; 4070 4071 switch (wga->wga_family) { 4072 case AF_INET: { 4073 struct sockaddr_in sin; 4074 char addrstr[128]; 4075 struct in_addr mask; 4076 struct sockaddr_in sin_mask; 4077 4078 if (addr_len != sizeof(struct in_addr)) 4079 return EINVAL; 4080 memcpy(&wga->wga_addr4, addr, addr_len); 4081 4082 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4083 0); 4084 sockaddr_copy(&wga->wga_sa_addr, 4085 sizeof(sin), sintosa(&sin)); 4086 4087 sockaddr_format(sintosa(&sin), 4088 addrstr, sizeof(addrstr)); 4089 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4090 4091 in_len2mask(&mask, wga->wga_cidr); 4092 sockaddr_in_init(&sin_mask, &mask, 0); 4093 sockaddr_copy(&wga->wga_sa_mask, 4094 sizeof(sin_mask), sintosa(&sin_mask)); 4095 4096 break; 4097 } 4098 #ifdef INET6 4099 case AF_INET6: { 4100 struct sockaddr_in6 sin6; 4101 char addrstr[128]; 4102 struct in6_addr mask; 4103 struct sockaddr_in6 sin6_mask; 4104 4105 if (addr_len != sizeof(struct in6_addr)) 4106 return EINVAL; 4107 memcpy(&wga->wga_addr6, addr, addr_len); 4108 4109 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4110 0, 0, 0); 4111 sockaddr_copy(&wga->wga_sa_addr, 4112 sizeof(sin6), sin6tosa(&sin6)); 4113 4114 sockaddr_format(sin6tosa(&sin6), 4115 addrstr, sizeof(addrstr)); 4116 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4117 4118 in6_prefixlen2mask(&mask, wga->wga_cidr); 4119 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4120 sockaddr_copy(&wga->wga_sa_mask, 4121 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4122 4123 break; 4124 } 4125 #endif 4126 default: 4127 error = EINVAL; 4128 goto out; 4129 } 4130 wga->wga_peer = wgp; 4131 4132 error = wg_rtable_add_route(wg, wga); 4133 if (error != 0) 4134 goto out; 4135 4136 j++; 4137 } 4138 wgp->wgp_n_allowedips = j; 4139 skip: 4140 *wgpp = wgp; 4141 out: 4142 return error; 4143 } 4144 4145 static int 4146 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4147 { 4148 int error; 4149 char *buf; 4150 4151 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4152 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4153 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4154 if (error != 0) 4155 return error; 4156 buf[ifd->ifd_len] = '\0'; 4157 #ifdef WG_DEBUG_DUMP 4158 for (int i = 0; i < ifd->ifd_len; i++) 4159 log(LOG_DEBUG, "%c", buf[i]); 4160 log(LOG_DEBUG, "\n"); 4161 #endif 4162 *_buf = buf; 4163 return 0; 4164 } 4165 4166 static int 4167 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4168 { 4169 int error; 4170 prop_dictionary_t prop_dict; 4171 char *buf = NULL; 4172 const void *privkey; 4173 size_t privkey_len; 4174 4175 error = wg_alloc_prop_buf(&buf, ifd); 4176 if (error != 0) 4177 return error; 4178 error = EINVAL; 4179 prop_dict = prop_dictionary_internalize(buf); 4180 if (prop_dict == NULL) 4181 goto out; 4182 if (!prop_dictionary_get_data(prop_dict, "private_key", 4183 &privkey, &privkey_len)) 4184 goto out; 4185 #ifdef WG_DEBUG_DUMP 4186 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n", privkey, privkey_len); 4187 for (int i = 0; i < privkey_len; i++) 4188 log(LOG_DEBUG, "%c", ((const char *)privkey)[i]); 4189 log(LOG_DEBUG, "\n"); 4190 #endif 4191 if (privkey_len != WG_STATIC_KEY_LEN) 4192 goto out; 4193 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4194 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4195 error = 0; 4196 4197 out: 4198 kmem_free(buf, ifd->ifd_len + 1); 4199 return error; 4200 } 4201 4202 static int 4203 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4204 { 4205 int error; 4206 prop_dictionary_t prop_dict; 4207 char *buf = NULL; 4208 uint16_t port; 4209 4210 error = wg_alloc_prop_buf(&buf, ifd); 4211 if (error != 0) 4212 return error; 4213 error = EINVAL; 4214 prop_dict = prop_dictionary_internalize(buf); 4215 if (prop_dict == NULL) 4216 goto out; 4217 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4218 goto out; 4219 4220 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4221 4222 out: 4223 kmem_free(buf, ifd->ifd_len + 1); 4224 return error; 4225 } 4226 4227 static int 4228 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4229 { 4230 int error; 4231 prop_dictionary_t prop_dict; 4232 char *buf = NULL; 4233 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4234 4235 error = wg_alloc_prop_buf(&buf, ifd); 4236 if (error != 0) 4237 return error; 4238 error = EINVAL; 4239 prop_dict = prop_dictionary_internalize(buf); 4240 if (prop_dict == NULL) 4241 goto out; 4242 4243 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4244 if (error != 0) 4245 goto out; 4246 4247 mutex_enter(wg->wg_lock); 4248 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4249 sizeof(wgp->wgp_pubkey)) != NULL || 4250 (wgp->wgp_name[0] && 4251 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4252 strlen(wgp->wgp_name)) != NULL)) { 4253 mutex_exit(wg->wg_lock); 4254 wg_destroy_peer(wgp); 4255 error = EEXIST; 4256 goto out; 4257 } 4258 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4259 sizeof(wgp->wgp_pubkey), wgp); 4260 KASSERT(wgp0 == wgp); 4261 if (wgp->wgp_name[0]) { 4262 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4263 strlen(wgp->wgp_name), wgp); 4264 KASSERT(wgp0 == wgp); 4265 } 4266 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4267 wg->wg_npeers++; 4268 mutex_exit(wg->wg_lock); 4269 4270 out: 4271 kmem_free(buf, ifd->ifd_len + 1); 4272 return error; 4273 } 4274 4275 static int 4276 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4277 { 4278 int error; 4279 prop_dictionary_t prop_dict; 4280 char *buf = NULL; 4281 const char *name; 4282 4283 error = wg_alloc_prop_buf(&buf, ifd); 4284 if (error != 0) 4285 return error; 4286 error = EINVAL; 4287 prop_dict = prop_dictionary_internalize(buf); 4288 if (prop_dict == NULL) 4289 goto out; 4290 4291 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4292 goto out; 4293 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4294 goto out; 4295 4296 error = wg_destroy_peer_name(wg, name); 4297 out: 4298 kmem_free(buf, ifd->ifd_len + 1); 4299 return error; 4300 } 4301 4302 static int 4303 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4304 { 4305 int error = ENOMEM; 4306 prop_dictionary_t prop_dict; 4307 prop_array_t peers = NULL; 4308 char *buf; 4309 struct wg_peer *wgp; 4310 int s, i; 4311 4312 prop_dict = prop_dictionary_create(); 4313 if (prop_dict == NULL) 4314 goto error; 4315 4316 if (!prop_dictionary_set_data(prop_dict, "private_key", wg->wg_privkey, 4317 WG_STATIC_KEY_LEN)) 4318 goto error; 4319 4320 if (wg->wg_listen_port != 0) { 4321 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4322 wg->wg_listen_port)) 4323 goto error; 4324 } 4325 4326 if (wg->wg_npeers == 0) 4327 goto skip_peers; 4328 4329 peers = prop_array_create(); 4330 if (peers == NULL) 4331 goto error; 4332 4333 s = pserialize_read_enter(); 4334 i = 0; 4335 WG_PEER_READER_FOREACH(wgp, wg) { 4336 struct wg_sockaddr *wgsa; 4337 struct psref wgp_psref, wgsa_psref; 4338 prop_dictionary_t prop_peer; 4339 4340 wg_get_peer(wgp, &wgp_psref); 4341 pserialize_read_exit(s); 4342 4343 prop_peer = prop_dictionary_create(); 4344 if (prop_peer == NULL) 4345 goto next; 4346 4347 if (strlen(wgp->wgp_name) > 0) { 4348 if (!prop_dictionary_set_string(prop_peer, "name", 4349 wgp->wgp_name)) 4350 goto next; 4351 } 4352 4353 if (!prop_dictionary_set_data(prop_peer, "public_key", 4354 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4355 goto next; 4356 4357 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4358 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4359 sizeof(wgp->wgp_psk))) { 4360 if (!prop_dictionary_set_data(prop_peer, 4361 "preshared_key", 4362 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4363 goto next; 4364 } 4365 4366 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4367 CTASSERT(AF_UNSPEC == 0); 4368 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4369 !prop_dictionary_set_data(prop_peer, "endpoint", 4370 wgsatoss(wgsa), 4371 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4372 wg_put_sa(wgp, wgsa, &wgsa_psref); 4373 goto next; 4374 } 4375 wg_put_sa(wgp, wgsa, &wgsa_psref); 4376 4377 const struct timespec *t = &wgp->wgp_last_handshake_time; 4378 4379 if (!prop_dictionary_set_uint64(prop_peer, 4380 "last_handshake_time_sec", t->tv_sec)) 4381 goto next; 4382 if (!prop_dictionary_set_uint32(prop_peer, 4383 "last_handshake_time_nsec", t->tv_nsec)) 4384 goto next; 4385 4386 if (wgp->wgp_n_allowedips == 0) 4387 goto skip_allowedips; 4388 4389 prop_array_t allowedips = prop_array_create(); 4390 if (allowedips == NULL) 4391 goto next; 4392 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4393 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4394 prop_dictionary_t prop_allowedip; 4395 4396 prop_allowedip = prop_dictionary_create(); 4397 if (prop_allowedip == NULL) 4398 break; 4399 4400 if (!prop_dictionary_set_int(prop_allowedip, "family", 4401 wga->wga_family)) 4402 goto _next; 4403 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4404 wga->wga_cidr)) 4405 goto _next; 4406 4407 switch (wga->wga_family) { 4408 case AF_INET: 4409 if (!prop_dictionary_set_data(prop_allowedip, 4410 "ip", &wga->wga_addr4, 4411 sizeof(wga->wga_addr4))) 4412 goto _next; 4413 break; 4414 #ifdef INET6 4415 case AF_INET6: 4416 if (!prop_dictionary_set_data(prop_allowedip, 4417 "ip", &wga->wga_addr6, 4418 sizeof(wga->wga_addr6))) 4419 goto _next; 4420 break; 4421 #endif 4422 default: 4423 break; 4424 } 4425 prop_array_set(allowedips, j, prop_allowedip); 4426 _next: 4427 prop_object_release(prop_allowedip); 4428 } 4429 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4430 prop_object_release(allowedips); 4431 4432 skip_allowedips: 4433 4434 prop_array_set(peers, i, prop_peer); 4435 next: 4436 if (prop_peer) 4437 prop_object_release(prop_peer); 4438 i++; 4439 4440 s = pserialize_read_enter(); 4441 wg_put_peer(wgp, &wgp_psref); 4442 } 4443 pserialize_read_exit(s); 4444 4445 prop_dictionary_set(prop_dict, "peers", peers); 4446 prop_object_release(peers); 4447 peers = NULL; 4448 4449 skip_peers: 4450 buf = prop_dictionary_externalize(prop_dict); 4451 if (buf == NULL) 4452 goto error; 4453 if (ifd->ifd_len < (strlen(buf) + 1)) { 4454 error = EINVAL; 4455 goto error; 4456 } 4457 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4458 4459 free(buf, 0); 4460 error: 4461 if (peers != NULL) 4462 prop_object_release(peers); 4463 if (prop_dict != NULL) 4464 prop_object_release(prop_dict); 4465 4466 return error; 4467 } 4468 4469 static int 4470 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4471 { 4472 struct wg_softc *wg = ifp->if_softc; 4473 struct ifreq *ifr = data; 4474 struct ifaddr *ifa = data; 4475 struct ifdrv *ifd = data; 4476 int error = 0; 4477 4478 switch (cmd) { 4479 case SIOCINITIFADDR: 4480 if (ifa->ifa_addr->sa_family != AF_LINK && 4481 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4482 (IFF_UP | IFF_RUNNING)) { 4483 ifp->if_flags |= IFF_UP; 4484 error = ifp->if_init(ifp); 4485 } 4486 return error; 4487 case SIOCADDMULTI: 4488 case SIOCDELMULTI: 4489 switch (ifr->ifr_addr.sa_family) { 4490 case AF_INET: /* IP supports Multicast */ 4491 break; 4492 #ifdef INET6 4493 case AF_INET6: /* IP6 supports Multicast */ 4494 break; 4495 #endif 4496 default: /* Other protocols doesn't support Multicast */ 4497 error = EAFNOSUPPORT; 4498 break; 4499 } 4500 return error; 4501 case SIOCSDRVSPEC: 4502 switch (ifd->ifd_cmd) { 4503 case WG_IOCTL_SET_PRIVATE_KEY: 4504 error = wg_ioctl_set_private_key(wg, ifd); 4505 break; 4506 case WG_IOCTL_SET_LISTEN_PORT: 4507 error = wg_ioctl_set_listen_port(wg, ifd); 4508 break; 4509 case WG_IOCTL_ADD_PEER: 4510 error = wg_ioctl_add_peer(wg, ifd); 4511 break; 4512 case WG_IOCTL_DELETE_PEER: 4513 error = wg_ioctl_delete_peer(wg, ifd); 4514 break; 4515 default: 4516 error = EINVAL; 4517 break; 4518 } 4519 return error; 4520 case SIOCGDRVSPEC: 4521 return wg_ioctl_get(wg, ifd); 4522 case SIOCSIFFLAGS: 4523 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4524 break; 4525 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4526 case IFF_RUNNING: 4527 /* 4528 * If interface is marked down and it is running, 4529 * then stop and disable it. 4530 */ 4531 (*ifp->if_stop)(ifp, 1); 4532 break; 4533 case IFF_UP: 4534 /* 4535 * If interface is marked up and it is stopped, then 4536 * start it. 4537 */ 4538 error = (*ifp->if_init)(ifp); 4539 break; 4540 default: 4541 break; 4542 } 4543 return error; 4544 #ifdef WG_RUMPKERNEL 4545 case SIOCSLINKSTR: 4546 error = wg_ioctl_linkstr(wg, ifd); 4547 if (error == 0) 4548 wg->wg_ops = &wg_ops_rumpuser; 4549 return error; 4550 #endif 4551 default: 4552 break; 4553 } 4554 4555 error = ifioctl_common(ifp, cmd, data); 4556 4557 #ifdef WG_RUMPKERNEL 4558 if (!wg_user_mode(wg)) 4559 return error; 4560 4561 /* Do the same to the corresponding tun device on the host */ 4562 /* 4563 * XXX Actually the command has not been handled yet. It 4564 * will be handled via pr_ioctl form doifioctl later. 4565 */ 4566 switch (cmd) { 4567 case SIOCAIFADDR: 4568 case SIOCDIFADDR: { 4569 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4570 struct in_aliasreq *ifra = &_ifra; 4571 KASSERT(error == ENOTTY); 4572 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4573 IFNAMSIZ); 4574 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4575 if (error == 0) 4576 error = ENOTTY; 4577 break; 4578 } 4579 #ifdef INET6 4580 case SIOCAIFADDR_IN6: 4581 case SIOCDIFADDR_IN6: { 4582 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4583 struct in6_aliasreq *ifra = &_ifra; 4584 KASSERT(error == ENOTTY); 4585 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4586 IFNAMSIZ); 4587 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4588 if (error == 0) 4589 error = ENOTTY; 4590 break; 4591 } 4592 #endif 4593 } 4594 #endif /* WG_RUMPKERNEL */ 4595 4596 return error; 4597 } 4598 4599 static int 4600 wg_init(struct ifnet *ifp) 4601 { 4602 4603 ifp->if_flags |= IFF_RUNNING; 4604 4605 /* TODO flush pending packets. */ 4606 return 0; 4607 } 4608 4609 static void 4610 wg_stop(struct ifnet *ifp, int disable) 4611 { 4612 4613 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4614 ifp->if_flags &= ~IFF_RUNNING; 4615 4616 /* Need to do something? */ 4617 } 4618 4619 #ifdef WG_DEBUG_PARAMS 4620 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4621 { 4622 const struct sysctlnode *node = NULL; 4623 4624 sysctl_createv(clog, 0, NULL, &node, 4625 CTLFLAG_PERMANENT, 4626 CTLTYPE_NODE, "wg", 4627 SYSCTL_DESCR("wg(4)"), 4628 NULL, 0, NULL, 0, 4629 CTL_NET, CTL_CREATE, CTL_EOL); 4630 sysctl_createv(clog, 0, &node, NULL, 4631 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4632 CTLTYPE_QUAD, "rekey_after_messages", 4633 SYSCTL_DESCR("session liftime by messages"), 4634 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4635 sysctl_createv(clog, 0, &node, NULL, 4636 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4637 CTLTYPE_INT, "rekey_after_time", 4638 SYSCTL_DESCR("session liftime"), 4639 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4640 sysctl_createv(clog, 0, &node, NULL, 4641 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4642 CTLTYPE_INT, "rekey_timeout", 4643 SYSCTL_DESCR("session handshake retry time"), 4644 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4645 sysctl_createv(clog, 0, &node, NULL, 4646 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4647 CTLTYPE_INT, "rekey_attempt_time", 4648 SYSCTL_DESCR("session handshake timeout"), 4649 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4650 sysctl_createv(clog, 0, &node, NULL, 4651 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4652 CTLTYPE_INT, "keepalive_timeout", 4653 SYSCTL_DESCR("keepalive timeout"), 4654 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4655 sysctl_createv(clog, 0, &node, NULL, 4656 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4657 CTLTYPE_BOOL, "force_underload", 4658 SYSCTL_DESCR("force to detemine under load"), 4659 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4660 } 4661 #endif 4662 4663 #ifdef WG_RUMPKERNEL 4664 static bool 4665 wg_user_mode(struct wg_softc *wg) 4666 { 4667 4668 return wg->wg_user != NULL; 4669 } 4670 4671 static int 4672 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4673 { 4674 struct ifnet *ifp = &wg->wg_if; 4675 int error; 4676 4677 if (ifp->if_flags & IFF_UP) 4678 return EBUSY; 4679 4680 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4681 /* XXX do nothing */ 4682 return 0; 4683 } else if (ifd->ifd_cmd != 0) { 4684 return EINVAL; 4685 } else if (wg->wg_user != NULL) { 4686 return EBUSY; 4687 } 4688 4689 /* Assume \0 included */ 4690 if (ifd->ifd_len > IFNAMSIZ) { 4691 return E2BIG; 4692 } else if (ifd->ifd_len < 1) { 4693 return EINVAL; 4694 } 4695 4696 char tun_name[IFNAMSIZ]; 4697 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4698 if (error != 0) 4699 return error; 4700 4701 if (strncmp(tun_name, "tun", 3) != 0) 4702 return EINVAL; 4703 4704 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 4705 4706 return error; 4707 } 4708 4709 static int 4710 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 4711 { 4712 int error; 4713 struct psref psref; 4714 struct wg_sockaddr *wgsa; 4715 struct wg_softc *wg = wgp->wgp_sc; 4716 struct iovec iov[1]; 4717 4718 wgsa = wg_get_endpoint_sa(wgp, &psref); 4719 4720 iov[0].iov_base = mtod(m, void *); 4721 iov[0].iov_len = m->m_len; 4722 4723 /* Send messages to a peer via an ordinary socket. */ 4724 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 4725 4726 wg_put_sa(wgp, wgsa, &psref); 4727 4728 m_freem(m); 4729 4730 return error; 4731 } 4732 4733 static void 4734 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 4735 { 4736 struct wg_softc *wg = ifp->if_softc; 4737 struct iovec iov[2]; 4738 struct sockaddr_storage ss; 4739 4740 KASSERT(af == AF_INET || af == AF_INET6); 4741 4742 WG_TRACE(""); 4743 4744 if (af == AF_INET) { 4745 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 4746 struct ip *ip; 4747 4748 KASSERT(m->m_len >= sizeof(struct ip)); 4749 ip = mtod(m, struct ip *); 4750 sockaddr_in_init(sin, &ip->ip_dst, 0); 4751 } else { 4752 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 4753 struct ip6_hdr *ip6; 4754 4755 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 4756 ip6 = mtod(m, struct ip6_hdr *); 4757 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 4758 } 4759 4760 iov[0].iov_base = &ss; 4761 iov[0].iov_len = ss.ss_len; 4762 iov[1].iov_base = mtod(m, void *); 4763 iov[1].iov_len = m->m_len; 4764 4765 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4766 4767 /* Send decrypted packets to users via a tun. */ 4768 rumpuser_wg_send_user(wg->wg_user, iov, 2); 4769 4770 m_freem(m); 4771 } 4772 4773 static int 4774 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 4775 { 4776 int error; 4777 uint16_t old_port = wg->wg_listen_port; 4778 4779 if (port != 0 && old_port == port) 4780 return 0; 4781 4782 error = rumpuser_wg_sock_bind(wg->wg_user, port); 4783 if (error == 0) 4784 wg->wg_listen_port = port; 4785 return error; 4786 } 4787 4788 /* 4789 * Receive user packets. 4790 */ 4791 void 4792 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4793 { 4794 struct ifnet *ifp = &wg->wg_if; 4795 struct mbuf *m; 4796 const struct sockaddr *dst; 4797 4798 WG_TRACE(""); 4799 4800 dst = iov[0].iov_base; 4801 4802 m = m_gethdr(M_DONTWAIT, MT_DATA); 4803 if (m == NULL) 4804 return; 4805 m->m_len = m->m_pkthdr.len = 0; 4806 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4807 4808 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4809 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4810 4811 (void)wg_output(ifp, m, dst, NULL); 4812 } 4813 4814 /* 4815 * Receive packets from a peer. 4816 */ 4817 void 4818 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4819 { 4820 struct mbuf *m; 4821 const struct sockaddr *src; 4822 4823 WG_TRACE(""); 4824 4825 src = iov[0].iov_base; 4826 4827 m = m_gethdr(M_DONTWAIT, MT_DATA); 4828 if (m == NULL) 4829 return; 4830 m->m_len = m->m_pkthdr.len = 0; 4831 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4832 4833 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4834 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4835 4836 wg_handle_packet(wg, m, src); 4837 } 4838 #endif /* WG_RUMPKERNEL */ 4839 4840 /* 4841 * Module infrastructure 4842 */ 4843 #include "if_module.h" 4844 4845 IF_MODULE(MODULE_CLASS_DRIVER, wg, "") 4846
Indexes created Fri Sep 26 20:09:58 GMT 2025