if_wg.c revision 1.25
1 /* $NetBSD: if_wg.c,v 1.25 2020/08/27 02:52:33 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.25 2020/08/27 02:52:33 riastradh Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_inet.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/kernel.h> 53 #include <sys/mbuf.h> 54 #include <sys/socket.h> 55 #include <sys/sockio.h> 56 #include <sys/errno.h> 57 #include <sys/ioctl.h> 58 #include <sys/time.h> 59 #include <sys/timespec.h> 60 #include <sys/socketvar.h> 61 #include <sys/syslog.h> 62 #include <sys/cpu.h> 63 #include <sys/intr.h> 64 #include <sys/kmem.h> 65 #include <sys/device.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/rwlock.h> 69 #include <sys/pserialize.h> 70 #include <sys/psref.h> 71 #include <sys/kthread.h> 72 #include <sys/cprng.h> 73 #include <sys/atomic.h> 74 #include <sys/sysctl.h> 75 #include <sys/domain.h> 76 #include <sys/pcq.h> 77 #include <sys/queue.h> 78 #include <sys/percpu.h> 79 #include <sys/callout.h> 80 81 #include <net/bpf.h> 82 #include <net/if.h> 83 #include <net/if_types.h> 84 #include <net/route.h> 85 86 #include <netinet/in.h> 87 #include <netinet/ip.h> 88 #include <netinet/ip_var.h> 89 #include <netinet/udp.h> 90 #include <netinet/udp_var.h> 91 #include <netinet/in_var.h> 92 #include <netinet/in_pcb.h> 93 94 #ifdef INET6 95 #include <netinet6/in6_var.h> 96 #include <netinet/ip6.h> 97 #include <netinet6/ip6_var.h> 98 #include <netinet6/in6_pcb.h> 99 #include <netinet6/udp6_var.h> 100 #endif /* INET6 */ 101 102 #include <net/if_wg.h> 103 104 #include <prop/proplib.h> 105 106 #include <crypto/blake2/blake2s.h> 107 #include <crypto/sodium/crypto_scalarmult.h> 108 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 109 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 110 111 #include "ioconf.h" 112 113 #ifdef WG_RUMPKERNEL 114 #include "wg_user.h" 115 #endif 116 117 /* 118 * Data structures 119 * - struct wg_softc is an instance of wg interfaces 120 * - It has a list of peers (struct wg_peer) 121 * - It has a kthread that sends/receives handshake messages and 122 * runs event handlers 123 * - It has its own two routing tables: one is for IPv4 and the other IPv6 124 * - struct wg_peer is a representative of a peer 125 * - It has a softint that is used to send packets over an wg interface 126 * to a peer 127 * - It has a pair of session instances (struct wg_session) 128 * - It has a pair of endpoint instances (struct wg_sockaddr) 129 * - Normally one endpoint is used and the second one is used only on 130 * a peer migration (a change of peer's IP address) 131 * - It has a list of IP addresses and sub networks called allowedips 132 * (struct wg_allowedip) 133 * - A packets sent over a session is allowed if its destination matches 134 * any IP addresses or sub networks of the list 135 * - struct wg_session represents a session of a secure tunnel with a peer 136 * - Two instances of sessions belong to a peer; a stable session and a 137 * unstable session 138 * - A handshake process of a session always starts with a unstable instace 139 * - Once a session is established, its instance becomes stable and the 140 * other becomes unstable instead 141 * - Data messages are always sent via a stable session 142 * 143 * Locking notes: 144 * - wg interfaces (struct wg_softc, wg) is listed in wg_softcs.list and 145 * protected by wg_softcs.lock 146 * - Each wg has a mutex(9) and a rwlock(9) 147 * - The mutex (wg_lock) protects its peer list (wg_peers) 148 * - A peer on the list is also protected by pserialize(9) or psref(9) 149 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 150 * - Each peer (struct wg_peer, wgp) has a mutex 151 * - The mutex (wgp_lock) protects wgp_session_unstable and wgp_state 152 * - Each session (struct wg_session, wgs) has a mutex 153 * - The mutex (wgs_lock) protects its state (wgs_state) and its handshake 154 * states 155 * - wgs_state of a unstable session can be changed while it never be 156 * changed on a stable session, so once get a session instace via 157 * wgp_session_stable we can safely access wgs_state without 158 * holding wgs_lock 159 * - A session is protected by pserialize or psref like wgp 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 */ 164 165 166 #define WGLOG(level, fmt, args...) \ 167 log(level, "%s: " fmt, __func__, ##args) 168 169 /* Debug options */ 170 #ifdef WG_DEBUG 171 /* Output debug logs */ 172 #ifndef WG_DEBUG_LOG 173 #define WG_DEBUG_LOG 174 #endif 175 /* Output trace logs */ 176 #ifndef WG_DEBUG_TRACE 177 #define WG_DEBUG_TRACE 178 #endif 179 /* Output hash values, etc. */ 180 #ifndef WG_DEBUG_DUMP 181 #define WG_DEBUG_DUMP 182 #endif 183 /* Make some internal parameters configurable for testing and debugging */ 184 #ifndef WG_DEBUG_PARAMS 185 #define WG_DEBUG_PARAMS 186 #endif 187 #endif 188 189 #ifdef WG_DEBUG_TRACE 190 #define WG_TRACE(msg) \ 191 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)) 192 #else 193 #define WG_TRACE(msg) __nothing 194 #endif 195 196 #ifdef WG_DEBUG_LOG 197 #define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args) 198 #else 199 #define WG_DLOG(fmt, args...) __nothing 200 #endif 201 202 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 203 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 204 &(wgprc)->wgprc_curpps, 1)) { \ 205 log(level, fmt, ##args); \ 206 } \ 207 } while (0) 208 209 #ifdef WG_DEBUG_PARAMS 210 static bool wg_force_underload = false; 211 #endif 212 213 #ifdef WG_DEBUG_DUMP 214 215 #ifdef WG_RUMPKERNEL 216 static void 217 wg_dump_buf(const char *func, const char *buf, const size_t size) 218 { 219 220 log(LOG_DEBUG, "%s: ", func); 221 for (int i = 0; i < size; i++) 222 log(LOG_DEBUG, "%02x ", (int)(0xff & buf[i])); 223 log(LOG_DEBUG, "\n"); 224 } 225 #endif 226 227 static void 228 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 229 const size_t size) 230 { 231 232 log(LOG_DEBUG, "%s: %s: ", func, name); 233 for (int i = 0; i < size; i++) 234 log(LOG_DEBUG, "%02x ", (int)(0xff & hash[i])); 235 log(LOG_DEBUG, "\n"); 236 } 237 238 #define WG_DUMP_HASH(name, hash) \ 239 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 240 #define WG_DUMP_HASH48(name, hash) \ 241 wg_dump_hash(__func__, name, hash, 48) 242 #define WG_DUMP_BUF(buf, size) \ 243 wg_dump_buf(__func__, buf, size) 244 #else 245 #define WG_DUMP_HASH(name, hash) __nothing 246 #define WG_DUMP_HASH48(name, hash) __nothing 247 #define WG_DUMP_BUF(buf, size) __nothing 248 #endif /* WG_DEBUG_DUMP */ 249 250 #define WG_MTU 1420 251 #define WG_ALLOWEDIPS 16 252 253 #define CURVE25519_KEY_LEN 32 254 #define TAI64N_LEN sizeof(uint32_t) * 3 255 #define POLY1305_AUTHTAG_LEN 16 256 #define HMAC_BLOCK_LEN 64 257 258 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 259 /* [N] 4.3: Hash functions */ 260 #define NOISE_DHLEN 32 261 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 262 #define NOISE_HASHLEN 32 263 #define NOISE_BLOCKLEN 64 264 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 265 /* [N] 5.1: "k" */ 266 #define NOISE_CIPHER_KEY_LEN 32 267 /* 268 * [N] 9.2: "psk" 269 * "... psk is a 32-byte secret value provided by the application." 270 */ 271 #define NOISE_PRESHARED_KEY_LEN 32 272 273 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 274 #define WG_TIMESTAMP_LEN TAI64N_LEN 275 276 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 277 278 #define WG_COOKIE_LEN 16 279 #define WG_MAC_LEN 16 280 #define WG_RANDVAL_LEN 24 281 282 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 283 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 284 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 285 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 286 #define WG_HASH_LEN NOISE_HASHLEN 287 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 288 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 289 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 290 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 291 #define WG_DATA_KEY_LEN 32 292 #define WG_SALT_LEN 24 293 294 /* 295 * The protocol messages 296 */ 297 struct wg_msg { 298 uint32_t wgm_type; 299 } __packed; 300 301 /* [W] 5.4.2 First Message: Initiator to Responder */ 302 struct wg_msg_init { 303 uint32_t wgmi_type; 304 uint32_t wgmi_sender; 305 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 306 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 307 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 308 uint8_t wgmi_mac1[WG_MAC_LEN]; 309 uint8_t wgmi_mac2[WG_MAC_LEN]; 310 } __packed; 311 312 /* [W] 5.4.3 Second Message: Responder to Initiator */ 313 struct wg_msg_resp { 314 uint32_t wgmr_type; 315 uint32_t wgmr_sender; 316 uint32_t wgmr_receiver; 317 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 318 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 319 uint8_t wgmr_mac1[WG_MAC_LEN]; 320 uint8_t wgmr_mac2[WG_MAC_LEN]; 321 } __packed; 322 323 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 324 struct wg_msg_data { 325 uint32_t wgmd_type; 326 uint32_t wgmd_receiver; 327 uint64_t wgmd_counter; 328 uint32_t wgmd_packet[0]; 329 } __packed; 330 331 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 332 struct wg_msg_cookie { 333 uint32_t wgmc_type; 334 uint32_t wgmc_receiver; 335 uint8_t wgmc_salt[WG_SALT_LEN]; 336 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 337 } __packed; 338 339 #define WG_MSG_TYPE_INIT 1 340 #define WG_MSG_TYPE_RESP 2 341 #define WG_MSG_TYPE_COOKIE 3 342 #define WG_MSG_TYPE_DATA 4 343 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 344 345 /* Sliding windows */ 346 347 #define SLIWIN_BITS 2048u 348 #define SLIWIN_TYPE uint32_t 349 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 350 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 351 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 352 353 struct sliwin { 354 SLIWIN_TYPE B[SLIWIN_WORDS]; 355 uint64_t T; 356 }; 357 358 static void 359 sliwin_reset(struct sliwin *W) 360 { 361 362 memset(W, 0, sizeof(*W)); 363 } 364 365 static int 366 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 367 { 368 369 /* 370 * If it's more than one window older than the highest sequence 371 * number we've seen, reject. 372 */ 373 #ifdef __HAVE_ATOMIC64_LOADSTORE 374 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 375 return EAUTH; 376 #endif 377 378 /* 379 * Otherwise, we need to take the lock to decide, so don't 380 * reject just yet. Caller must serialize a call to 381 * sliwin_update in this case. 382 */ 383 return 0; 384 } 385 386 static int 387 sliwin_update(struct sliwin *W, uint64_t S) 388 { 389 unsigned word, bit; 390 391 /* 392 * If it's more than one window older than the highest sequence 393 * number we've seen, reject. 394 */ 395 if (S + SLIWIN_NPKT < W->T) 396 return EAUTH; 397 398 /* 399 * If it's higher than the highest sequence number we've seen, 400 * advance the window. 401 */ 402 if (S > W->T) { 403 uint64_t i = W->T / SLIWIN_BPW; 404 uint64_t j = S / SLIWIN_BPW; 405 unsigned k; 406 407 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 408 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 409 #ifdef __HAVE_ATOMIC64_LOADSTORE 410 atomic_store_relaxed(&W->T, S); 411 #else 412 W->T = S; 413 #endif 414 } 415 416 /* Test and set the bit -- if already set, reject. */ 417 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 418 bit = S % SLIWIN_BPW; 419 if (W->B[word] & (1UL << bit)) 420 return EAUTH; 421 W->B[word] |= 1UL << bit; 422 423 /* Accept! */ 424 return 0; 425 } 426 427 struct wg_worker { 428 kmutex_t wgw_lock; 429 kcondvar_t wgw_cv; 430 bool wgw_todie; 431 struct socket *wgw_so4; 432 struct socket *wgw_so6; 433 int wgw_wakeup_reasons; 434 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 __BIT(0) 435 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6 __BIT(1) 436 #define WG_WAKEUP_REASON_PEER __BIT(2) 437 }; 438 439 struct wg_session { 440 struct wg_peer *wgs_peer; 441 struct psref_target 442 wgs_psref; 443 kmutex_t *wgs_lock; 444 445 int wgs_state; 446 #define WGS_STATE_UNKNOWN 0 447 #define WGS_STATE_INIT_ACTIVE 1 448 #define WGS_STATE_INIT_PASSIVE 2 449 #define WGS_STATE_ESTABLISHED 3 450 #define WGS_STATE_DESTROYING 4 451 452 time_t wgs_time_established; 453 time_t wgs_time_last_data_sent; 454 bool wgs_is_initiator; 455 456 uint32_t wgs_sender_index; 457 uint32_t wgs_receiver_index; 458 #ifdef __HAVE_ATOMIC64_LOADSTORE 459 volatile uint64_t 460 wgs_send_counter; 461 #else 462 kmutex_t wgs_send_counter_lock; 463 uint64_t wgs_send_counter; 464 #endif 465 466 struct { 467 kmutex_t lock; 468 struct sliwin window; 469 } *wgs_recvwin; 470 471 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 472 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 473 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 474 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 475 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 476 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 477 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 478 }; 479 480 struct wg_sockaddr { 481 union { 482 struct sockaddr_storage _ss; 483 struct sockaddr _sa; 484 struct sockaddr_in _sin; 485 struct sockaddr_in6 _sin6; 486 }; 487 struct psref_target wgsa_psref; 488 }; 489 490 #define wgsatosa(wgsa) (&(wgsa)->_sa) 491 #define wgsatosin(wgsa) (&(wgsa)->_sin) 492 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 493 494 struct wg_peer; 495 struct wg_allowedip { 496 struct radix_node wga_nodes[2]; 497 struct wg_sockaddr _wga_sa_addr; 498 struct wg_sockaddr _wga_sa_mask; 499 #define wga_sa_addr _wga_sa_addr._sa 500 #define wga_sa_mask _wga_sa_mask._sa 501 502 int wga_family; 503 uint8_t wga_cidr; 504 union { 505 struct in_addr _ip4; 506 struct in6_addr _ip6; 507 } wga_addr; 508 #define wga_addr4 wga_addr._ip4 509 #define wga_addr6 wga_addr._ip6 510 511 struct wg_peer *wga_peer; 512 }; 513 514 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 515 516 struct wg_ppsratecheck { 517 struct timeval wgprc_lasttime; 518 int wgprc_curpps; 519 }; 520 521 struct wg_softc; 522 struct wg_peer { 523 struct wg_softc *wgp_sc; 524 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 525 struct pslist_entry wgp_peerlist_entry; 526 pserialize_t wgp_psz; 527 struct psref_target wgp_psref; 528 kmutex_t *wgp_lock; 529 530 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 531 struct wg_sockaddr *wgp_endpoint; 532 #define wgp_ss wgp_endpoint->_ss 533 #define wgp_sa wgp_endpoint->_sa 534 #define wgp_sin wgp_endpoint->_sin 535 #define wgp_sin6 wgp_endpoint->_sin6 536 struct wg_sockaddr *wgp_endpoint0; 537 bool wgp_endpoint_changing; 538 bool wgp_endpoint_available; 539 540 /* The preshared key (optional) */ 541 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 542 543 int wgp_state; 544 #define WGP_STATE_INIT 0 545 #define WGP_STATE_ESTABLISHED 1 546 #define WGP_STATE_GIVEUP 2 547 #define WGP_STATE_DESTROYING 3 548 549 void *wgp_si; 550 pcq_t *wgp_q; 551 552 struct wg_session *wgp_session_stable; 553 struct wg_session *wgp_session_unstable; 554 555 /* timestamp in big-endian */ 556 wg_timestamp_t wgp_timestamp_latest_init; 557 558 struct timespec wgp_last_handshake_time; 559 560 callout_t wgp_rekey_timer; 561 callout_t wgp_handshake_timeout_timer; 562 callout_t wgp_session_dtor_timer; 563 564 time_t wgp_handshake_start_time; 565 566 int wgp_n_allowedips; 567 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 568 569 time_t wgp_latest_cookie_time; 570 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 571 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 572 bool wgp_last_sent_mac1_valid; 573 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 574 bool wgp_last_sent_cookie_valid; 575 576 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 577 578 time_t wgp_last_genrandval_time; 579 uint32_t wgp_randval; 580 581 struct wg_ppsratecheck wgp_ppsratecheck; 582 583 volatile unsigned int wgp_tasks; 584 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 585 #define WGP_TASK_ENDPOINT_CHANGED __BIT(1) 586 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(2) 587 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(3) 588 }; 589 590 struct wg_ops; 591 592 struct wg_softc { 593 struct ifnet wg_if; 594 LIST_ENTRY(wg_softc) wg_list; 595 kmutex_t *wg_lock; 596 krwlock_t *wg_rwlock; 597 598 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 599 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 600 601 int wg_npeers; 602 struct pslist_head wg_peers; 603 uint16_t wg_listen_port; 604 605 struct wg_worker *wg_worker; 606 lwp_t *wg_worker_lwp; 607 608 struct radix_node_head *wg_rtable_ipv4; 609 struct radix_node_head *wg_rtable_ipv6; 610 611 struct wg_ppsratecheck wg_ppsratecheck; 612 613 struct wg_ops *wg_ops; 614 615 #ifdef WG_RUMPKERNEL 616 struct wg_user *wg_user; 617 #endif 618 }; 619 620 /* [W] 6.1 Preliminaries */ 621 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 622 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 623 #define WG_REKEY_AFTER_TIME 120 624 #define WG_REJECT_AFTER_TIME 180 625 #define WG_REKEY_ATTEMPT_TIME 90 626 #define WG_REKEY_TIMEOUT 5 627 #define WG_KEEPALIVE_TIMEOUT 10 628 629 #define WG_COOKIE_TIME 120 630 #define WG_RANDVAL_TIME (2 * 60) 631 632 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 633 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 634 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 635 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 636 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 637 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 638 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 639 640 static struct mbuf * 641 wg_get_mbuf(size_t, size_t); 642 643 static void wg_wakeup_worker(struct wg_worker *, int); 644 645 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 646 struct mbuf *); 647 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 648 const uint32_t, const uint8_t [], const struct sockaddr *); 649 static int wg_send_handshake_msg_resp(struct wg_softc *, 650 struct wg_peer *, const struct wg_msg_init *); 651 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 652 653 static struct wg_peer * 654 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 655 struct psref *); 656 static struct wg_peer * 657 wg_lookup_peer_by_pubkey(struct wg_softc *, 658 const uint8_t [], struct psref *); 659 660 static struct wg_session * 661 wg_lookup_session_by_index(struct wg_softc *, 662 const uint32_t, struct psref *); 663 664 static void wg_update_endpoint_if_necessary(struct wg_peer *, 665 const struct sockaddr *); 666 667 static void wg_schedule_rekey_timer(struct wg_peer *); 668 static void wg_schedule_session_dtor_timer(struct wg_peer *); 669 static void wg_stop_session_dtor_timer(struct wg_peer *); 670 671 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 672 static void wg_calculate_keys(struct wg_session *, const bool); 673 674 static void wg_clear_states(struct wg_session *); 675 676 static void wg_get_peer(struct wg_peer *, struct psref *); 677 static void wg_put_peer(struct wg_peer *, struct psref *); 678 679 static int wg_send_so(struct wg_peer *, struct mbuf *); 680 static int wg_send_udp(struct wg_peer *, struct mbuf *); 681 static int wg_output(struct ifnet *, struct mbuf *, 682 const struct sockaddr *, const struct rtentry *); 683 static void wg_input(struct ifnet *, struct mbuf *, const int); 684 static int wg_ioctl(struct ifnet *, u_long, void *); 685 static int wg_bind_port(struct wg_softc *, const uint16_t); 686 static int wg_init(struct ifnet *); 687 static void wg_stop(struct ifnet *, int); 688 689 static int wg_clone_create(struct if_clone *, int); 690 static int wg_clone_destroy(struct ifnet *); 691 692 struct wg_ops { 693 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 694 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 695 void (*input)(struct ifnet *, struct mbuf *, const int); 696 int (*bind_port)(struct wg_softc *, const uint16_t); 697 }; 698 699 struct wg_ops wg_ops_rumpkernel = { 700 .send_hs_msg = wg_send_so, 701 .send_data_msg = wg_send_udp, 702 .input = wg_input, 703 .bind_port = wg_bind_port, 704 }; 705 706 #ifdef WG_RUMPKERNEL 707 static bool wg_user_mode(struct wg_softc *); 708 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 709 710 static int wg_send_user(struct wg_peer *, struct mbuf *); 711 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 712 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 713 714 struct wg_ops wg_ops_rumpuser = { 715 .send_hs_msg = wg_send_user, 716 .send_data_msg = wg_send_user, 717 .input = wg_input_user, 718 .bind_port = wg_bind_port_user, 719 }; 720 #endif 721 722 #define WG_PEER_READER_FOREACH(wgp, wg) \ 723 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 724 wgp_peerlist_entry) 725 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 726 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 727 wgp_peerlist_entry) 728 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 729 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 730 #define WG_PEER_WRITER_REMOVE(wgp) \ 731 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 732 733 struct wg_route { 734 struct radix_node wgr_nodes[2]; 735 struct wg_peer *wgr_peer; 736 }; 737 738 static struct radix_node_head * 739 wg_rnh(struct wg_softc *wg, const int family) 740 { 741 742 switch (family) { 743 case AF_INET: 744 return wg->wg_rtable_ipv4; 745 #ifdef INET6 746 case AF_INET6: 747 return wg->wg_rtable_ipv6; 748 #endif 749 default: 750 return NULL; 751 } 752 } 753 754 755 /* 756 * Global variables 757 */ 758 LIST_HEAD(wg_sclist, wg_softc); 759 static struct { 760 struct wg_sclist list; 761 kmutex_t lock; 762 } wg_softcs __cacheline_aligned; 763 764 struct psref_class *wg_psref_class __read_mostly; 765 766 static struct if_clone wg_cloner = 767 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 768 769 770 void wgattach(int); 771 /* ARGSUSED */ 772 void 773 wgattach(int count) 774 { 775 /* 776 * Nothing to do here, initialization is handled by the 777 * module initialization code in wginit() below). 778 */ 779 } 780 781 static void 782 wginit(void) 783 { 784 785 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 786 787 mutex_init(&wg_softcs.lock, MUTEX_DEFAULT, IPL_NONE); 788 LIST_INIT(&wg_softcs.list); 789 if_clone_attach(&wg_cloner); 790 } 791 792 static int 793 wgdetach(void) 794 { 795 int error = 0; 796 797 mutex_enter(&wg_softcs.lock); 798 if (!LIST_EMPTY(&wg_softcs.list)) { 799 mutex_exit(&wg_softcs.lock); 800 error = EBUSY; 801 } 802 803 if (error == 0) { 804 psref_class_destroy(wg_psref_class); 805 806 if_clone_detach(&wg_cloner); 807 } 808 809 return error; 810 } 811 812 static void 813 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 814 uint8_t hash[WG_HASH_LEN]) 815 { 816 /* [W] 5.4: CONSTRUCTION */ 817 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 818 /* [W] 5.4: IDENTIFIER */ 819 const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com"; 820 struct blake2s state; 821 822 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 823 signature, strlen(signature)); 824 825 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 826 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 827 828 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 829 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 830 blake2s_update(&state, id, strlen(id)); 831 blake2s_final(&state, hash); 832 833 WG_DUMP_HASH("ckey", ckey); 834 WG_DUMP_HASH("hash", hash); 835 } 836 837 static void 838 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 839 const size_t inputsize) 840 { 841 struct blake2s state; 842 843 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 844 blake2s_update(&state, hash, WG_HASH_LEN); 845 blake2s_update(&state, input, inputsize); 846 blake2s_final(&state, hash); 847 } 848 849 static void 850 wg_algo_mac(uint8_t out[], const size_t outsize, 851 const uint8_t key[], const size_t keylen, 852 const uint8_t input1[], const size_t input1len, 853 const uint8_t input2[], const size_t input2len) 854 { 855 struct blake2s state; 856 857 blake2s_init(&state, outsize, key, keylen); 858 859 blake2s_update(&state, input1, input1len); 860 if (input2 != NULL) 861 blake2s_update(&state, input2, input2len); 862 blake2s_final(&state, out); 863 } 864 865 static void 866 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 867 const uint8_t input1[], const size_t input1len, 868 const uint8_t input2[], const size_t input2len) 869 { 870 struct blake2s state; 871 /* [W] 5.4: LABEL-MAC1 */ 872 const char *label = "mac1----"; 873 uint8_t key[WG_HASH_LEN]; 874 875 blake2s_init(&state, sizeof(key), NULL, 0); 876 blake2s_update(&state, label, strlen(label)); 877 blake2s_update(&state, input1, input1len); 878 blake2s_final(&state, key); 879 880 blake2s_init(&state, outsize, key, sizeof(key)); 881 if (input2 != NULL) 882 blake2s_update(&state, input2, input2len); 883 blake2s_final(&state, out); 884 } 885 886 static void 887 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 888 const uint8_t input1[], const size_t input1len) 889 { 890 struct blake2s state; 891 /* [W] 5.4: LABEL-COOKIE */ 892 const char *label = "cookie--"; 893 894 blake2s_init(&state, outsize, NULL, 0); 895 blake2s_update(&state, label, strlen(label)); 896 blake2s_update(&state, input1, input1len); 897 blake2s_final(&state, out); 898 } 899 900 static void 901 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 902 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 903 { 904 905 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 906 907 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 908 crypto_scalarmult_base(pubkey, privkey); 909 } 910 911 static void 912 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 913 const uint8_t privkey[WG_STATIC_KEY_LEN], 914 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 915 { 916 917 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 918 919 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 920 KASSERT(ret == 0); 921 } 922 923 static void 924 wg_algo_hmac(uint8_t out[], const size_t outlen, 925 const uint8_t key[], const size_t keylen, 926 const uint8_t in[], const size_t inlen) 927 { 928 #define IPAD 0x36 929 #define OPAD 0x5c 930 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 931 uint8_t ipad[HMAC_BLOCK_LEN]; 932 uint8_t opad[HMAC_BLOCK_LEN]; 933 int i; 934 struct blake2s state; 935 936 KASSERT(outlen == WG_HASH_LEN); 937 KASSERT(keylen <= HMAC_BLOCK_LEN); 938 939 memcpy(hmackey, key, keylen); 940 941 for (i = 0; i < sizeof(hmackey); i++) { 942 ipad[i] = hmackey[i] ^ IPAD; 943 opad[i] = hmackey[i] ^ OPAD; 944 } 945 946 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 947 blake2s_update(&state, ipad, sizeof(ipad)); 948 blake2s_update(&state, in, inlen); 949 blake2s_final(&state, out); 950 951 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 952 blake2s_update(&state, opad, sizeof(opad)); 953 blake2s_update(&state, out, WG_HASH_LEN); 954 blake2s_final(&state, out); 955 #undef IPAD 956 #undef OPAD 957 } 958 959 static void 960 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 961 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 962 const uint8_t input[], const size_t inputlen) 963 { 964 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 965 uint8_t one[1]; 966 967 /* 968 * [N] 4.3: "an input_key_material byte sequence with length 969 * either zero bytes, 32 bytes, or DHLEN bytes." 970 */ 971 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 972 973 WG_DUMP_HASH("ckey", ckey); 974 if (input != NULL) 975 WG_DUMP_HASH("input", input); 976 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 977 input, inputlen); 978 WG_DUMP_HASH("tmp1", tmp1); 979 one[0] = 1; 980 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 981 one, sizeof(one)); 982 WG_DUMP_HASH("out1", out1); 983 if (out2 == NULL) 984 return; 985 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 986 tmp2[WG_KDF_OUTPUT_LEN] = 2; 987 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 988 tmp2, sizeof(tmp2)); 989 WG_DUMP_HASH("out2", out2); 990 if (out3 == NULL) 991 return; 992 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 993 tmp2[WG_KDF_OUTPUT_LEN] = 3; 994 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 995 tmp2, sizeof(tmp2)); 996 WG_DUMP_HASH("out3", out3); 997 } 998 999 static void 1000 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1001 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1002 const uint8_t local_key[WG_STATIC_KEY_LEN], 1003 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1004 { 1005 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1006 1007 wg_algo_dh(dhout, local_key, remote_key); 1008 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1009 1010 WG_DUMP_HASH("dhout", dhout); 1011 WG_DUMP_HASH("ckey", ckey); 1012 if (cipher_key != NULL) 1013 WG_DUMP_HASH("cipher_key", cipher_key); 1014 } 1015 1016 static void 1017 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1018 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1019 const uint8_t auth[], size_t authlen) 1020 { 1021 uint8_t nonce[(32 + 64) / 8] = {0}; 1022 long long unsigned int outsize; 1023 int error __diagused; 1024 1025 memcpy(&nonce[4], &counter, sizeof(counter)); 1026 1027 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1028 plainsize, auth, authlen, NULL, nonce, key); 1029 KASSERT(error == 0); 1030 KASSERT(outsize == expected_outsize); 1031 } 1032 1033 static int 1034 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1035 const uint64_t counter, const uint8_t encrypted[], 1036 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1037 { 1038 uint8_t nonce[(32 + 64) / 8] = {0}; 1039 long long unsigned int outsize; 1040 int error; 1041 1042 memcpy(&nonce[4], &counter, sizeof(counter)); 1043 1044 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1045 encrypted, encryptedsize, auth, authlen, nonce, key); 1046 if (error == 0) 1047 KASSERT(outsize == expected_outsize); 1048 return error; 1049 } 1050 1051 static void 1052 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1053 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1054 const uint8_t auth[], size_t authlen, 1055 const uint8_t nonce[WG_SALT_LEN]) 1056 { 1057 long long unsigned int outsize; 1058 int error __diagused; 1059 1060 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1061 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1062 plain, plainsize, auth, authlen, NULL, nonce, key); 1063 KASSERT(error == 0); 1064 KASSERT(outsize == expected_outsize); 1065 } 1066 1067 static int 1068 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1069 const uint8_t key[], const uint64_t counter, 1070 const uint8_t encrypted[], const size_t encryptedsize, 1071 const uint8_t auth[], size_t authlen, 1072 const uint8_t nonce[WG_SALT_LEN]) 1073 { 1074 long long unsigned int outsize; 1075 int error; 1076 1077 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1078 encrypted, encryptedsize, auth, authlen, nonce, key); 1079 if (error == 0) 1080 KASSERT(outsize == expected_outsize); 1081 return error; 1082 } 1083 1084 static void 1085 wg_algo_tai64n(wg_timestamp_t timestamp) 1086 { 1087 struct timespec ts; 1088 1089 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1090 getnanotime(&ts); 1091 /* TAI64 label in external TAI64 format */ 1092 be32enc(timestamp, 0x40000000UL + (ts.tv_sec >> 32)); 1093 /* second beginning from 1970 TAI */ 1094 be32enc(timestamp + 4, ts.tv_sec & 0xffffffffU); 1095 /* nanosecond in big-endian format */ 1096 be32enc(timestamp + 8, ts.tv_nsec); 1097 } 1098 1099 static struct wg_session * 1100 wg_get_unstable_session(struct wg_peer *wgp, struct psref *psref) 1101 { 1102 int s; 1103 struct wg_session *wgs; 1104 1105 s = pserialize_read_enter(); 1106 wgs = wgp->wgp_session_unstable; 1107 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1108 pserialize_read_exit(s); 1109 return wgs; 1110 } 1111 1112 static struct wg_session * 1113 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1114 { 1115 int s; 1116 struct wg_session *wgs; 1117 1118 s = pserialize_read_enter(); 1119 wgs = wgp->wgp_session_stable; 1120 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1121 pserialize_read_exit(s); 1122 return wgs; 1123 } 1124 1125 static void 1126 wg_get_session(struct wg_session *wgs, struct psref *psref) 1127 { 1128 1129 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1130 } 1131 1132 static void 1133 wg_put_session(struct wg_session *wgs, struct psref *psref) 1134 { 1135 1136 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1137 } 1138 1139 static struct wg_session * 1140 wg_lock_unstable_session(struct wg_peer *wgp) 1141 { 1142 struct wg_session *wgs; 1143 1144 mutex_enter(wgp->wgp_lock); 1145 wgs = wgp->wgp_session_unstable; 1146 mutex_enter(wgs->wgs_lock); 1147 mutex_exit(wgp->wgp_lock); 1148 return wgs; 1149 } 1150 1151 #if 0 1152 static void 1153 wg_unlock_session(struct wg_peer *wgp, struct wg_session *wgs) 1154 { 1155 1156 mutex_exit(wgs->wgs_lock); 1157 } 1158 #endif 1159 1160 /* 1161 * Handshake patterns 1162 * 1163 * [W] 5: "These messages use the "IK" pattern from Noise" 1164 * [N] 7.5. Interactive handshake patterns (fundamental) 1165 * "The first character refers to the initiators static key:" 1166 * "I = Static key for initiator Immediately transmitted to responder, 1167 * despite reduced or absent identity hiding" 1168 * "The second character refers to the responders static key:" 1169 * "K = Static key for responder Known to initiator" 1170 * "IK: 1171 * <- s 1172 * ... 1173 * -> e, es, s, ss 1174 * <- e, ee, se" 1175 * [N] 9.4. Pattern modifiers 1176 * "IKpsk2: 1177 * <- s 1178 * ... 1179 * -> e, es, s, ss 1180 * <- e, ee, se, psk" 1181 */ 1182 static void 1183 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1184 struct wg_session *wgs, struct wg_msg_init *wgmi) 1185 { 1186 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1187 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1188 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1189 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1190 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1191 1192 wgmi->wgmi_type = WG_MSG_TYPE_INIT; 1193 wgmi->wgmi_sender = cprng_strong32(); 1194 1195 /* [W] 5.4.2: First Message: Initiator to Responder */ 1196 1197 /* Ci := HASH(CONSTRUCTION) */ 1198 /* Hi := HASH(Ci || IDENTIFIER) */ 1199 wg_init_key_and_hash(ckey, hash); 1200 /* Hi := HASH(Hi || Sr^pub) */ 1201 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1202 1203 WG_DUMP_HASH("hash", hash); 1204 1205 /* [N] 2.2: "e" */ 1206 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1207 wg_algo_generate_keypair(pubkey, privkey); 1208 /* Ci := KDF1(Ci, Ei^pub) */ 1209 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1210 /* msg.ephemeral := Ei^pub */ 1211 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1212 /* Hi := HASH(Hi || msg.ephemeral) */ 1213 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1214 1215 WG_DUMP_HASH("ckey", ckey); 1216 WG_DUMP_HASH("hash", hash); 1217 1218 /* [N] 2.2: "es" */ 1219 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1220 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1221 1222 /* [N] 2.2: "s" */ 1223 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1224 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1225 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1226 hash, sizeof(hash)); 1227 /* Hi := HASH(Hi || msg.static) */ 1228 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1229 1230 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1231 1232 /* [N] 2.2: "ss" */ 1233 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1234 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1235 1236 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1237 wg_timestamp_t timestamp; 1238 wg_algo_tai64n(timestamp); 1239 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1240 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1241 /* Hi := HASH(Hi || msg.timestamp) */ 1242 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1243 1244 /* [W] 5.4.4 Cookie MACs */ 1245 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1246 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1247 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1248 /* Need mac1 to decrypt a cookie from a cookie message */ 1249 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1250 sizeof(wgp->wgp_last_sent_mac1)); 1251 wgp->wgp_last_sent_mac1_valid = true; 1252 1253 if (wgp->wgp_latest_cookie_time == 0 || 1254 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1255 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1256 else { 1257 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1258 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1259 (const uint8_t *)wgmi, 1260 offsetof(struct wg_msg_init, wgmi_mac2), 1261 NULL, 0); 1262 } 1263 1264 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1265 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1266 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1267 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1268 wgs->wgs_sender_index = wgmi->wgmi_sender; 1269 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_sender_index); 1270 } 1271 1272 static void 1273 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1274 const struct sockaddr *src) 1275 { 1276 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1277 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1278 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1279 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1280 struct wg_peer *wgp; 1281 struct wg_session *wgs; 1282 bool reset_state_on_error = false; 1283 int error, ret; 1284 struct psref psref_peer; 1285 struct psref psref_session; 1286 uint8_t mac1[WG_MAC_LEN]; 1287 1288 WG_TRACE("init msg received"); 1289 1290 /* 1291 * [W] 5.4.2: First Message: Initiator to Responder 1292 * "When the responder receives this message, it does the same 1293 * operations so that its final state variables are identical, 1294 * replacing the operands of the DH function to produce equivalent 1295 * values." 1296 * Note that the following comments of operations are just copies of 1297 * the initiator's ones. 1298 */ 1299 1300 /* Ci := HASH(CONSTRUCTION) */ 1301 /* Hi := HASH(Ci || IDENTIFIER) */ 1302 wg_init_key_and_hash(ckey, hash); 1303 /* Hi := HASH(Hi || Sr^pub) */ 1304 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1305 1306 /* [N] 2.2: "e" */ 1307 /* Ci := KDF1(Ci, Ei^pub) */ 1308 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1309 sizeof(wgmi->wgmi_ephemeral)); 1310 /* Hi := HASH(Hi || msg.ephemeral) */ 1311 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1312 1313 WG_DUMP_HASH("ckey", ckey); 1314 1315 /* [N] 2.2: "es" */ 1316 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1317 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1318 1319 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1320 1321 /* [N] 2.2: "s" */ 1322 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1323 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1324 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1325 if (error != 0) { 1326 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1327 "wg_algo_aead_dec for secret key failed\n"); 1328 return; 1329 } 1330 /* Hi := HASH(Hi || msg.static) */ 1331 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1332 1333 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1334 if (wgp == NULL) { 1335 WG_DLOG("peer not found\n"); 1336 return; 1337 } 1338 1339 wgs = wg_lock_unstable_session(wgp); 1340 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 1341 /* 1342 * We can assume that the peer doesn't have an established 1343 * session, so clear it now. 1344 */ 1345 WG_TRACE("Session destroying, but force to clear"); 1346 wg_stop_session_dtor_timer(wgp); 1347 wg_clear_states(wgs); 1348 wgs->wgs_state = WGS_STATE_UNKNOWN; 1349 } 1350 if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) { 1351 WG_TRACE("Sesssion already initializing, ignoring the message"); 1352 mutex_exit(wgs->wgs_lock); 1353 goto out_wgp; 1354 } 1355 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) { 1356 WG_TRACE("Sesssion already initializing, destroying old states"); 1357 wg_clear_states(wgs); 1358 } 1359 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1360 reset_state_on_error = true; 1361 wg_get_session(wgs, &psref_session); 1362 mutex_exit(wgs->wgs_lock); 1363 1364 wg_algo_mac_mac1(mac1, sizeof(mac1), 1365 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1366 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1367 1368 /* 1369 * [W] 5.3: Denial of Service Mitigation & Cookies 1370 * "the responder, ..., must always reject messages with an invalid 1371 * msg.mac1" 1372 */ 1373 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1374 WG_DLOG("mac1 is invalid\n"); 1375 goto out; 1376 } 1377 1378 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1379 WG_TRACE("under load"); 1380 /* 1381 * [W] 5.3: Denial of Service Mitigation & Cookies 1382 * "the responder, ..., and when under load may reject messages 1383 * with an invalid msg.mac2. If the responder receives a 1384 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1385 * and is under load, it may respond with a cookie reply 1386 * message" 1387 */ 1388 uint8_t zero[WG_MAC_LEN] = {0}; 1389 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1390 WG_TRACE("sending a cookie message: no cookie included"); 1391 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1392 wgmi->wgmi_mac1, src); 1393 goto out; 1394 } 1395 if (!wgp->wgp_last_sent_cookie_valid) { 1396 WG_TRACE("sending a cookie message: no cookie sent ever"); 1397 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1398 wgmi->wgmi_mac1, src); 1399 goto out; 1400 } 1401 uint8_t mac2[WG_MAC_LEN]; 1402 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1403 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1404 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1405 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1406 WG_DLOG("mac2 is invalid\n"); 1407 goto out; 1408 } 1409 WG_TRACE("under load, but continue to sending"); 1410 } 1411 1412 /* [N] 2.2: "ss" */ 1413 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1414 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1415 1416 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1417 wg_timestamp_t timestamp; 1418 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1419 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1420 hash, sizeof(hash)); 1421 if (error != 0) { 1422 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1423 "wg_algo_aead_dec for timestamp failed\n"); 1424 goto out; 1425 } 1426 /* Hi := HASH(Hi || msg.timestamp) */ 1427 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1428 1429 /* 1430 * [W] 5.1 "The responder keeps track of the greatest timestamp 1431 * received per peer and discards packets containing 1432 * timestamps less than or equal to it." 1433 */ 1434 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1435 sizeof(timestamp)); 1436 if (ret <= 0) { 1437 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1438 "invalid init msg: timestamp is old\n"); 1439 goto out; 1440 } 1441 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1442 1443 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1444 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1445 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1446 sizeof(wgmi->wgmi_ephemeral)); 1447 1448 wg_update_endpoint_if_necessary(wgp, src); 1449 1450 (void)wg_send_handshake_msg_resp(wg, wgp, wgmi); 1451 1452 wg_calculate_keys(wgs, false); 1453 wg_clear_states(wgs); 1454 1455 wg_put_session(wgs, &psref_session); 1456 wg_put_peer(wgp, &psref_peer); 1457 return; 1458 1459 out: 1460 if (reset_state_on_error) { 1461 mutex_enter(wgs->wgs_lock); 1462 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1463 wgs->wgs_state = WGS_STATE_UNKNOWN; 1464 mutex_exit(wgs->wgs_lock); 1465 } 1466 wg_put_session(wgs, &psref_session); 1467 out_wgp: 1468 wg_put_peer(wgp, &psref_peer); 1469 } 1470 1471 static void 1472 wg_schedule_handshake_timeout_timer(struct wg_peer *wgp) 1473 { 1474 1475 mutex_enter(wgp->wgp_lock); 1476 if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) { 1477 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1478 MIN(wg_rekey_timeout, INT_MAX/hz) * hz); 1479 } 1480 mutex_exit(wgp->wgp_lock); 1481 } 1482 1483 static void 1484 wg_stop_handshake_timeout_timer(struct wg_peer *wgp) 1485 { 1486 1487 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 1488 } 1489 1490 static struct socket * 1491 wg_get_so_by_af(struct wg_worker *wgw, const int af) 1492 { 1493 1494 return (af == AF_INET) ? wgw->wgw_so4 : wgw->wgw_so6; 1495 } 1496 1497 static struct socket * 1498 wg_get_so_by_peer(struct wg_peer *wgp) 1499 { 1500 1501 return wg_get_so_by_af(wgp->wgp_sc->wg_worker, wgp->wgp_sa.sa_family); 1502 } 1503 1504 static struct wg_sockaddr * 1505 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1506 { 1507 struct wg_sockaddr *wgsa; 1508 int s; 1509 1510 s = pserialize_read_enter(); 1511 wgsa = wgp->wgp_endpoint; 1512 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1513 pserialize_read_exit(s); 1514 1515 return wgsa; 1516 } 1517 1518 static void 1519 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1520 { 1521 1522 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1523 } 1524 1525 static int 1526 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1527 { 1528 int error; 1529 struct socket *so; 1530 struct psref psref; 1531 struct wg_sockaddr *wgsa; 1532 1533 so = wg_get_so_by_peer(wgp); 1534 wgsa = wg_get_endpoint_sa(wgp, &psref); 1535 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1536 wg_put_sa(wgp, wgsa, &psref); 1537 1538 return error; 1539 } 1540 1541 static int 1542 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1543 { 1544 int error; 1545 struct mbuf *m; 1546 struct wg_msg_init *wgmi; 1547 struct wg_session *wgs; 1548 struct psref psref; 1549 1550 wgs = wg_lock_unstable_session(wgp); 1551 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 1552 WG_TRACE("Session destroying"); 1553 mutex_exit(wgs->wgs_lock); 1554 /* XXX should wait? */ 1555 return EBUSY; 1556 } 1557 if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) { 1558 WG_TRACE("Sesssion already initializing, skip starting a new one"); 1559 mutex_exit(wgs->wgs_lock); 1560 return EBUSY; 1561 } 1562 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) { 1563 WG_TRACE("Sesssion already initializing, destroying old states"); 1564 wg_clear_states(wgs); 1565 } 1566 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1567 wg_get_session(wgs, &psref); 1568 mutex_exit(wgs->wgs_lock); 1569 1570 m = m_gethdr(M_WAIT, MT_DATA); 1571 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1572 wgmi = mtod(m, struct wg_msg_init *); 1573 1574 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1575 1576 error = wg->wg_ops->send_hs_msg(wgp, m); 1577 if (error == 0) { 1578 WG_TRACE("init msg sent"); 1579 1580 if (wgp->wgp_handshake_start_time == 0) 1581 wgp->wgp_handshake_start_time = time_uptime; 1582 wg_schedule_handshake_timeout_timer(wgp); 1583 } else { 1584 mutex_enter(wgs->wgs_lock); 1585 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1586 wgs->wgs_state = WGS_STATE_UNKNOWN; 1587 mutex_exit(wgs->wgs_lock); 1588 } 1589 wg_put_session(wgs, &psref); 1590 1591 return error; 1592 } 1593 1594 static void 1595 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1596 struct wg_msg_resp *wgmr, const struct wg_msg_init *wgmi) 1597 { 1598 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1599 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1600 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1601 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1602 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1603 struct wg_session *wgs; 1604 struct psref psref; 1605 1606 wgs = wg_get_unstable_session(wgp, &psref); 1607 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1608 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1609 1610 wgmr->wgmr_type = WG_MSG_TYPE_RESP; 1611 wgmr->wgmr_sender = cprng_strong32(); 1612 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1613 1614 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1615 1616 /* [N] 2.2: "e" */ 1617 /* Er^priv, Er^pub := DH-GENERATE() */ 1618 wg_algo_generate_keypair(pubkey, privkey); 1619 /* Cr := KDF1(Cr, Er^pub) */ 1620 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1621 /* msg.ephemeral := Er^pub */ 1622 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1623 /* Hr := HASH(Hr || msg.ephemeral) */ 1624 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1625 1626 WG_DUMP_HASH("ckey", ckey); 1627 WG_DUMP_HASH("hash", hash); 1628 1629 /* [N] 2.2: "ee" */ 1630 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1631 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1632 1633 /* [N] 2.2: "se" */ 1634 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1635 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1636 1637 /* [N] 9.2: "psk" */ 1638 { 1639 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1640 /* Cr, r, k := KDF3(Cr, Q) */ 1641 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1642 sizeof(wgp->wgp_psk)); 1643 /* Hr := HASH(Hr || r) */ 1644 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1645 } 1646 1647 /* msg.empty := AEAD(k, 0, e, Hr) */ 1648 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1649 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1650 /* Hr := HASH(Hr || msg.empty) */ 1651 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1652 1653 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1654 1655 /* [W] 5.4.4: Cookie MACs */ 1656 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1657 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1658 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1659 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1660 /* Need mac1 to decrypt a cookie from a cookie message */ 1661 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1662 sizeof(wgp->wgp_last_sent_mac1)); 1663 wgp->wgp_last_sent_mac1_valid = true; 1664 1665 if (wgp->wgp_latest_cookie_time == 0 || 1666 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1667 /* msg.mac2 := 0^16 */ 1668 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1669 else { 1670 /* msg.mac2 := MAC(Lm, msg_b) */ 1671 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1672 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1673 (const uint8_t *)wgmr, 1674 offsetof(struct wg_msg_resp, wgmr_mac2), 1675 NULL, 0); 1676 } 1677 1678 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1679 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1680 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1681 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1682 wgs->wgs_sender_index = wgmr->wgmr_sender; 1683 wgs->wgs_receiver_index = wgmi->wgmi_sender; 1684 WG_DLOG("sender=%x\n", wgs->wgs_sender_index); 1685 WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index); 1686 wg_put_session(wgs, &psref); 1687 } 1688 1689 static void 1690 wg_swap_sessions(struct wg_peer *wgp) 1691 { 1692 1693 KASSERT(mutex_owned(wgp->wgp_lock)); 1694 1695 wgp->wgp_session_unstable = atomic_swap_ptr(&wgp->wgp_session_stable, 1696 wgp->wgp_session_unstable); 1697 KASSERT(wgp->wgp_session_stable->wgs_state == WGS_STATE_ESTABLISHED); 1698 } 1699 1700 static void 1701 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1702 const struct sockaddr *src) 1703 { 1704 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1705 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1706 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1707 struct wg_peer *wgp; 1708 struct wg_session *wgs; 1709 struct psref psref; 1710 int error; 1711 uint8_t mac1[WG_MAC_LEN]; 1712 struct wg_session *wgs_prev; 1713 1714 WG_TRACE("resp msg received"); 1715 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1716 if (wgs == NULL) { 1717 WG_TRACE("No session found"); 1718 return; 1719 } 1720 1721 wgp = wgs->wgs_peer; 1722 1723 wg_algo_mac_mac1(mac1, sizeof(mac1), 1724 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1725 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1726 1727 /* 1728 * [W] 5.3: Denial of Service Mitigation & Cookies 1729 * "the responder, ..., must always reject messages with an invalid 1730 * msg.mac1" 1731 */ 1732 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1733 WG_DLOG("mac1 is invalid\n"); 1734 goto out; 1735 } 1736 1737 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1738 WG_TRACE("under load"); 1739 /* 1740 * [W] 5.3: Denial of Service Mitigation & Cookies 1741 * "the responder, ..., and when under load may reject messages 1742 * with an invalid msg.mac2. If the responder receives a 1743 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1744 * and is under load, it may respond with a cookie reply 1745 * message" 1746 */ 1747 uint8_t zero[WG_MAC_LEN] = {0}; 1748 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1749 WG_TRACE("sending a cookie message: no cookie included"); 1750 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1751 wgmr->wgmr_mac1, src); 1752 goto out; 1753 } 1754 if (!wgp->wgp_last_sent_cookie_valid) { 1755 WG_TRACE("sending a cookie message: no cookie sent ever"); 1756 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1757 wgmr->wgmr_mac1, src); 1758 goto out; 1759 } 1760 uint8_t mac2[WG_MAC_LEN]; 1761 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1762 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1763 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1764 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1765 WG_DLOG("mac2 is invalid\n"); 1766 goto out; 1767 } 1768 WG_TRACE("under load, but continue to sending"); 1769 } 1770 1771 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1772 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1773 1774 /* 1775 * [W] 5.4.3 Second Message: Responder to Initiator 1776 * "When the initiator receives this message, it does the same 1777 * operations so that its final state variables are identical, 1778 * replacing the operands of the DH function to produce equivalent 1779 * values." 1780 * Note that the following comments of operations are just copies of 1781 * the initiator's ones. 1782 */ 1783 1784 /* [N] 2.2: "e" */ 1785 /* Cr := KDF1(Cr, Er^pub) */ 1786 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1787 sizeof(wgmr->wgmr_ephemeral)); 1788 /* Hr := HASH(Hr || msg.ephemeral) */ 1789 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1790 1791 WG_DUMP_HASH("ckey", ckey); 1792 WG_DUMP_HASH("hash", hash); 1793 1794 /* [N] 2.2: "ee" */ 1795 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1796 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1797 wgmr->wgmr_ephemeral); 1798 1799 /* [N] 2.2: "se" */ 1800 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1801 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1802 1803 /* [N] 9.2: "psk" */ 1804 { 1805 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1806 /* Cr, r, k := KDF3(Cr, Q) */ 1807 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1808 sizeof(wgp->wgp_psk)); 1809 /* Hr := HASH(Hr || r) */ 1810 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1811 } 1812 1813 { 1814 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 1815 /* msg.empty := AEAD(k, 0, e, Hr) */ 1816 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 1817 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 1818 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1819 if (error != 0) { 1820 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1821 "wg_algo_aead_dec for empty message failed\n"); 1822 goto out; 1823 } 1824 /* Hr := HASH(Hr || msg.empty) */ 1825 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1826 } 1827 1828 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 1829 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 1830 wgs->wgs_receiver_index = wgmr->wgmr_sender; 1831 WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index); 1832 1833 wgs->wgs_state = WGS_STATE_ESTABLISHED; 1834 wgs->wgs_time_established = time_uptime; 1835 wgs->wgs_time_last_data_sent = 0; 1836 wgs->wgs_is_initiator = true; 1837 wg_calculate_keys(wgs, true); 1838 wg_clear_states(wgs); 1839 WG_TRACE("WGS_STATE_ESTABLISHED"); 1840 1841 wg_stop_handshake_timeout_timer(wgp); 1842 1843 mutex_enter(wgp->wgp_lock); 1844 wg_swap_sessions(wgp); 1845 wgs_prev = wgp->wgp_session_unstable; 1846 mutex_enter(wgs_prev->wgs_lock); 1847 1848 getnanotime(&wgp->wgp_last_handshake_time); 1849 wgp->wgp_handshake_start_time = 0; 1850 wgp->wgp_last_sent_mac1_valid = false; 1851 wgp->wgp_last_sent_cookie_valid = false; 1852 mutex_exit(wgp->wgp_lock); 1853 1854 wg_schedule_rekey_timer(wgp); 1855 1856 wg_update_endpoint_if_necessary(wgp, src); 1857 1858 /* 1859 * Send something immediately (same as the official implementation) 1860 * XXX if there are pending data packets, we don't need to send 1861 * a keepalive message. 1862 */ 1863 wg_send_keepalive_msg(wgp, wgs); 1864 1865 /* Anyway run a softint to flush pending packets */ 1866 kpreempt_disable(); 1867 softint_schedule(wgp->wgp_si); 1868 kpreempt_enable(); 1869 WG_TRACE("softint scheduled"); 1870 1871 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 1872 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 1873 /* We can't destroy the old session immediately */ 1874 wg_schedule_session_dtor_timer(wgp); 1875 } 1876 mutex_exit(wgs_prev->wgs_lock); 1877 1878 out: 1879 wg_put_session(wgs, &psref); 1880 } 1881 1882 static int 1883 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1884 const struct wg_msg_init *wgmi) 1885 { 1886 int error; 1887 struct mbuf *m; 1888 struct wg_msg_resp *wgmr; 1889 1890 m = m_gethdr(M_WAIT, MT_DATA); 1891 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 1892 wgmr = mtod(m, struct wg_msg_resp *); 1893 wg_fill_msg_resp(wg, wgp, wgmr, wgmi); 1894 1895 error = wg->wg_ops->send_hs_msg(wgp, m); 1896 if (error == 0) 1897 WG_TRACE("resp msg sent"); 1898 return error; 1899 } 1900 1901 static struct wg_peer * 1902 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 1903 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 1904 { 1905 struct wg_peer *wgp; 1906 1907 int s = pserialize_read_enter(); 1908 /* XXX O(n) */ 1909 WG_PEER_READER_FOREACH(wgp, wg) { 1910 if (consttime_memequal(wgp->wgp_pubkey, pubkey, 1911 sizeof(wgp->wgp_pubkey))) 1912 break; 1913 } 1914 if (wgp != NULL) 1915 wg_get_peer(wgp, psref); 1916 pserialize_read_exit(s); 1917 1918 return wgp; 1919 } 1920 1921 static void 1922 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 1923 struct wg_msg_cookie *wgmc, const uint32_t sender, 1924 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 1925 { 1926 uint8_t cookie[WG_COOKIE_LEN]; 1927 uint8_t key[WG_HASH_LEN]; 1928 uint8_t addr[sizeof(struct in6_addr)]; 1929 size_t addrlen; 1930 uint16_t uh_sport; /* be */ 1931 1932 wgmc->wgmc_type = WG_MSG_TYPE_COOKIE; 1933 wgmc->wgmc_receiver = sender; 1934 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 1935 1936 /* 1937 * [W] 5.4.7: Under Load: Cookie Reply Message 1938 * "The secret variable, Rm, changes every two minutes to a 1939 * random value" 1940 */ 1941 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 1942 wgp->wgp_randval = cprng_strong32(); 1943 wgp->wgp_last_genrandval_time = time_uptime; 1944 } 1945 1946 switch (src->sa_family) { 1947 case AF_INET: { 1948 const struct sockaddr_in *sin = satocsin(src); 1949 addrlen = sizeof(sin->sin_addr); 1950 memcpy(addr, &sin->sin_addr, addrlen); 1951 uh_sport = sin->sin_port; 1952 break; 1953 } 1954 #ifdef INET6 1955 case AF_INET6: { 1956 const struct sockaddr_in6 *sin6 = satocsin6(src); 1957 addrlen = sizeof(sin6->sin6_addr); 1958 memcpy(addr, &sin6->sin6_addr, addrlen); 1959 uh_sport = sin6->sin6_port; 1960 break; 1961 } 1962 #endif 1963 default: 1964 panic("invalid af=%d", wgp->wgp_sa.sa_family); 1965 } 1966 1967 wg_algo_mac(cookie, sizeof(cookie), 1968 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 1969 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 1970 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 1971 sizeof(wg->wg_pubkey)); 1972 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 1973 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 1974 1975 /* Need to store to calculate mac2 */ 1976 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 1977 wgp->wgp_last_sent_cookie_valid = true; 1978 } 1979 1980 static int 1981 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 1982 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 1983 const struct sockaddr *src) 1984 { 1985 int error; 1986 struct mbuf *m; 1987 struct wg_msg_cookie *wgmc; 1988 1989 m = m_gethdr(M_WAIT, MT_DATA); 1990 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 1991 wgmc = mtod(m, struct wg_msg_cookie *); 1992 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 1993 1994 error = wg->wg_ops->send_hs_msg(wgp, m); 1995 if (error == 0) 1996 WG_TRACE("cookie msg sent"); 1997 return error; 1998 } 1999 2000 static bool 2001 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2002 { 2003 #ifdef WG_DEBUG_PARAMS 2004 if (wg_force_underload) 2005 return true; 2006 #endif 2007 2008 /* 2009 * XXX we don't have a means of a load estimation. The purpose of 2010 * the mechanism is a DoS mitigation, so we consider frequent handshake 2011 * messages as (a kind of) load; if a message of the same type comes 2012 * to a peer within 1 second, we consider we are under load. 2013 */ 2014 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2015 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2016 return (time_uptime - last) == 0; 2017 } 2018 2019 static void 2020 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2021 { 2022 2023 /* 2024 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2025 */ 2026 if (initiator) { 2027 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2028 wgs->wgs_chaining_key, NULL, 0); 2029 } else { 2030 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2031 wgs->wgs_chaining_key, NULL, 0); 2032 } 2033 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2034 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2035 } 2036 2037 static uint64_t 2038 wg_session_get_send_counter(struct wg_session *wgs) 2039 { 2040 #ifdef __HAVE_ATOMIC64_LOADSTORE 2041 return atomic_load_relaxed(&wgs->wgs_send_counter); 2042 #else 2043 uint64_t send_counter; 2044 2045 mutex_enter(&wgs->wgs_send_counter_lock); 2046 send_counter = wgs->wgs_send_counter; 2047 mutex_exit(&wgs->wgs_send_counter_lock); 2048 2049 return send_counter; 2050 #endif 2051 } 2052 2053 static uint64_t 2054 wg_session_inc_send_counter(struct wg_session *wgs) 2055 { 2056 #ifdef __HAVE_ATOMIC64_LOADSTORE 2057 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2058 #else 2059 uint64_t send_counter; 2060 2061 mutex_enter(&wgs->wgs_send_counter_lock); 2062 send_counter = wgs->wgs_send_counter++; 2063 mutex_exit(&wgs->wgs_send_counter_lock); 2064 2065 return send_counter; 2066 #endif 2067 } 2068 2069 static void 2070 wg_clear_states(struct wg_session *wgs) 2071 { 2072 2073 wgs->wgs_send_counter = 0; 2074 sliwin_reset(&wgs->wgs_recvwin->window); 2075 2076 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2077 wgs_clear(handshake_hash); 2078 wgs_clear(chaining_key); 2079 wgs_clear(ephemeral_key_pub); 2080 wgs_clear(ephemeral_key_priv); 2081 wgs_clear(ephemeral_key_peer); 2082 #undef wgs_clear 2083 } 2084 2085 static struct wg_session * 2086 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2087 struct psref *psref) 2088 { 2089 struct wg_peer *wgp; 2090 struct wg_session *wgs; 2091 2092 int s = pserialize_read_enter(); 2093 /* XXX O(n) */ 2094 WG_PEER_READER_FOREACH(wgp, wg) { 2095 wgs = wgp->wgp_session_stable; 2096 WG_DLOG("index=%x wgs_sender_index=%x\n", 2097 index, wgs->wgs_sender_index); 2098 if (wgs->wgs_sender_index == index) 2099 break; 2100 wgs = wgp->wgp_session_unstable; 2101 WG_DLOG("index=%x wgs_sender_index=%x\n", 2102 index, wgs->wgs_sender_index); 2103 if (wgs->wgs_sender_index == index) 2104 break; 2105 wgs = NULL; 2106 } 2107 if (wgs != NULL) 2108 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2109 pserialize_read_exit(s); 2110 2111 return wgs; 2112 } 2113 2114 static void 2115 wg_schedule_rekey_timer(struct wg_peer *wgp) 2116 { 2117 int timeout = MIN(wg_rekey_after_time, INT_MAX/hz); 2118 2119 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2120 } 2121 2122 static void 2123 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2124 { 2125 struct mbuf *m; 2126 2127 /* 2128 * [W] 6.5 Passive Keepalive 2129 * "A keepalive message is simply a transport data message with 2130 * a zero-length encapsulated encrypted inner-packet." 2131 */ 2132 m = m_gethdr(M_WAIT, MT_DATA); 2133 wg_send_data_msg(wgp, wgs, m); 2134 } 2135 2136 static bool 2137 wg_need_to_send_init_message(struct wg_session *wgs) 2138 { 2139 /* 2140 * [W] 6.2 Transport Message Limits 2141 * "if a peer is the initiator of a current secure session, 2142 * WireGuard will send a handshake initiation message to begin 2143 * a new secure session ... if after receiving a transport data 2144 * message, the current secure session is (REJECT-AFTER-TIME 2145 * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has 2146 * not yet acted upon this event." 2147 */ 2148 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2149 (time_uptime - wgs->wgs_time_established) >= 2150 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2151 } 2152 2153 static void 2154 wg_schedule_peer_task(struct wg_peer *wgp, int task) 2155 { 2156 2157 atomic_or_uint(&wgp->wgp_tasks, task); 2158 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2159 wg_wakeup_worker(wgp->wgp_sc->wg_worker, WG_WAKEUP_REASON_PEER); 2160 } 2161 2162 static void 2163 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2164 { 2165 2166 KASSERT(mutex_owned(wgp->wgp_lock)); 2167 2168 WG_TRACE("Changing endpoint"); 2169 2170 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2171 wgp->wgp_endpoint0 = atomic_swap_ptr(&wgp->wgp_endpoint, 2172 wgp->wgp_endpoint0); 2173 if (!wgp->wgp_endpoint_available) 2174 wgp->wgp_endpoint_available = true; 2175 wgp->wgp_endpoint_changing = true; 2176 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2177 } 2178 2179 static bool 2180 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2181 { 2182 uint16_t packet_len; 2183 const struct ip *ip; 2184 2185 if (__predict_false(decrypted_len < sizeof(struct ip))) 2186 return false; 2187 2188 ip = (const struct ip *)packet; 2189 if (ip->ip_v == 4) 2190 *af = AF_INET; 2191 else if (ip->ip_v == 6) 2192 *af = AF_INET6; 2193 else 2194 return false; 2195 2196 WG_DLOG("af=%d\n", *af); 2197 2198 if (*af == AF_INET) { 2199 packet_len = ntohs(ip->ip_len); 2200 } else { 2201 const struct ip6_hdr *ip6; 2202 2203 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr))) 2204 return false; 2205 2206 ip6 = (const struct ip6_hdr *)packet; 2207 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen); 2208 } 2209 2210 WG_DLOG("packet_len=%u\n", packet_len); 2211 if (packet_len > decrypted_len) 2212 return false; 2213 2214 return true; 2215 } 2216 2217 static bool 2218 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2219 int af, char *packet) 2220 { 2221 struct sockaddr_storage ss; 2222 struct sockaddr *sa; 2223 struct psref psref; 2224 struct wg_peer *wgp; 2225 bool ok; 2226 2227 /* 2228 * II CRYPTOKEY ROUTING 2229 * "it will only accept it if its source IP resolves in the 2230 * table to the public key used in the secure session for 2231 * decrypting it." 2232 */ 2233 2234 if (af == AF_INET) { 2235 const struct ip *ip = (const struct ip *)packet; 2236 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2237 sockaddr_in_init(sin, &ip->ip_src, 0); 2238 sa = sintosa(sin); 2239 #ifdef INET6 2240 } else { 2241 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2242 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2243 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2244 sa = sin6tosa(sin6); 2245 #endif 2246 } 2247 2248 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2249 ok = (wgp == wgp_expected); 2250 if (wgp != NULL) 2251 wg_put_peer(wgp, &psref); 2252 2253 return ok; 2254 } 2255 2256 static void 2257 wg_session_dtor_timer(void *arg) 2258 { 2259 struct wg_peer *wgp = arg; 2260 2261 WG_TRACE("enter"); 2262 2263 mutex_enter(wgp->wgp_lock); 2264 if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) { 2265 mutex_exit(wgp->wgp_lock); 2266 return; 2267 } 2268 mutex_exit(wgp->wgp_lock); 2269 2270 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2271 } 2272 2273 static void 2274 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2275 { 2276 2277 /* 1 second grace period */ 2278 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2279 } 2280 2281 static void 2282 wg_stop_session_dtor_timer(struct wg_peer *wgp) 2283 { 2284 2285 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 2286 } 2287 2288 static bool 2289 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2290 { 2291 if (sa1->sa_family != sa2->sa_family) 2292 return false; 2293 2294 switch (sa1->sa_family) { 2295 case AF_INET: 2296 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2297 case AF_INET6: 2298 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2299 default: 2300 return true; 2301 } 2302 } 2303 2304 static void 2305 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2306 const struct sockaddr *src) 2307 { 2308 2309 #ifdef WG_DEBUG_LOG 2310 char oldaddr[128], newaddr[128]; 2311 sockaddr_format(&wgp->wgp_sa, oldaddr, sizeof(oldaddr)); 2312 sockaddr_format(src, newaddr, sizeof(newaddr)); 2313 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2314 #endif 2315 2316 /* 2317 * III: "Since the packet has authenticated correctly, the source IP of 2318 * the outer UDP/IP packet is used to update the endpoint for peer..." 2319 */ 2320 if (__predict_false(sockaddr_cmp(src, &wgp->wgp_sa) != 0 || 2321 !sockaddr_port_match(src, &wgp->wgp_sa))) { 2322 mutex_enter(wgp->wgp_lock); 2323 /* XXX We can't change the endpoint twice in a short period */ 2324 if (!wgp->wgp_endpoint_changing) { 2325 wg_change_endpoint(wgp, src); 2326 } 2327 mutex_exit(wgp->wgp_lock); 2328 } 2329 } 2330 2331 static void 2332 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2333 const struct sockaddr *src) 2334 { 2335 struct wg_msg_data *wgmd; 2336 char *encrypted_buf = NULL, *decrypted_buf; 2337 size_t encrypted_len, decrypted_len; 2338 struct wg_session *wgs; 2339 struct wg_peer *wgp; 2340 size_t mlen; 2341 struct psref psref; 2342 int error, af; 2343 bool success, free_encrypted_buf = false, ok; 2344 struct mbuf *n; 2345 2346 if (m->m_len < sizeof(struct wg_msg_data)) { 2347 m = m_pullup(m, sizeof(struct wg_msg_data)); 2348 if (m == NULL) 2349 return; 2350 } 2351 wgmd = mtod(m, struct wg_msg_data *); 2352 2353 KASSERT(wgmd->wgmd_type == WG_MSG_TYPE_DATA); 2354 WG_TRACE("data"); 2355 2356 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2357 if (wgs == NULL) { 2358 WG_TRACE("No session found"); 2359 m_freem(m); 2360 return; 2361 } 2362 wgp = wgs->wgs_peer; 2363 2364 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2365 wgmd->wgmd_counter); 2366 if (error) { 2367 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2368 "out-of-window packet: %"PRIu64"\n", 2369 wgmd->wgmd_counter); 2370 goto out; 2371 } 2372 2373 mlen = m_length(m); 2374 encrypted_len = mlen - sizeof(*wgmd); 2375 2376 if (encrypted_len < WG_AUTHTAG_LEN) { 2377 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2378 goto out; 2379 } 2380 2381 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2382 if (success) { 2383 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2384 } else { 2385 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2386 if (encrypted_buf == NULL) { 2387 WG_DLOG("failed to allocate encrypted_buf\n"); 2388 goto out; 2389 } 2390 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2391 free_encrypted_buf = true; 2392 } 2393 /* m_ensure_contig may change m regardless of its result */ 2394 wgmd = mtod(m, struct wg_msg_data *); 2395 2396 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2397 if (decrypted_len > MCLBYTES) { 2398 /* FIXME handle larger data than MCLBYTES */ 2399 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2400 goto out; 2401 } 2402 2403 /* To avoid zero length */ 2404 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2405 if (n == NULL) { 2406 WG_DLOG("wg_get_mbuf failed\n"); 2407 goto out; 2408 } 2409 decrypted_buf = mtod(n, char *); 2410 2411 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2412 error = wg_algo_aead_dec(decrypted_buf, 2413 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2414 wgs->wgs_tkey_recv, wgmd->wgmd_counter, encrypted_buf, 2415 encrypted_len, NULL, 0); 2416 if (error != 0) { 2417 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2418 "failed to wg_algo_aead_dec\n"); 2419 m_freem(n); 2420 goto out; 2421 } 2422 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2423 2424 mutex_enter(&wgs->wgs_recvwin->lock); 2425 error = sliwin_update(&wgs->wgs_recvwin->window, 2426 wgmd->wgmd_counter); 2427 mutex_exit(&wgs->wgs_recvwin->lock); 2428 if (error) { 2429 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2430 "replay or out-of-window packet: %"PRIu64"\n", 2431 wgmd->wgmd_counter); 2432 m_freem(n); 2433 goto out; 2434 } 2435 2436 m_freem(m); 2437 m = NULL; 2438 wgmd = NULL; 2439 2440 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2441 if (!ok) { 2442 /* something wrong... */ 2443 m_freem(n); 2444 goto out; 2445 } 2446 2447 wg_update_endpoint_if_necessary(wgp, src); 2448 2449 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2450 if (ok) { 2451 wg->wg_ops->input(&wg->wg_if, n, af); 2452 } else { 2453 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2454 "invalid source address\n"); 2455 m_freem(n); 2456 /* 2457 * The inner address is invalid however the session is valid 2458 * so continue the session processing below. 2459 */ 2460 } 2461 n = NULL; 2462 2463 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) { 2464 struct wg_session *wgs_prev; 2465 2466 KASSERT(wgs == wgp->wgp_session_unstable); 2467 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2468 wgs->wgs_time_established = time_uptime; 2469 wgs->wgs_time_last_data_sent = 0; 2470 wgs->wgs_is_initiator = false; 2471 WG_TRACE("WGS_STATE_ESTABLISHED"); 2472 2473 mutex_enter(wgp->wgp_lock); 2474 wg_swap_sessions(wgp); 2475 wgs_prev = wgp->wgp_session_unstable; 2476 mutex_enter(wgs_prev->wgs_lock); 2477 getnanotime(&wgp->wgp_last_handshake_time); 2478 wgp->wgp_handshake_start_time = 0; 2479 wgp->wgp_last_sent_mac1_valid = false; 2480 wgp->wgp_last_sent_cookie_valid = false; 2481 mutex_exit(wgp->wgp_lock); 2482 2483 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2484 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2485 /* We can't destroy the old session immediately */ 2486 wg_schedule_session_dtor_timer(wgp); 2487 } else { 2488 wg_clear_states(wgs_prev); 2489 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 2490 } 2491 mutex_exit(wgs_prev->wgs_lock); 2492 2493 /* Anyway run a softint to flush pending packets */ 2494 kpreempt_disable(); 2495 softint_schedule(wgp->wgp_si); 2496 kpreempt_enable(); 2497 } else { 2498 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2499 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2500 } 2501 /* 2502 * [W] 6.5 Passive Keepalive 2503 * "If a peer has received a validly-authenticated transport 2504 * data message (section 5.4.6), but does not have any packets 2505 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2506 * a keepalive message." 2507 */ 2508 WG_DLOG("time_uptime=%lu wgs_time_last_data_sent=%lu\n", 2509 time_uptime, wgs->wgs_time_last_data_sent); 2510 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2511 wg_keepalive_timeout) { 2512 WG_TRACE("Schedule sending keepalive message"); 2513 /* 2514 * We can't send a keepalive message here to avoid 2515 * a deadlock; we already hold the solock of a socket 2516 * that is used to send the message. 2517 */ 2518 wg_schedule_peer_task(wgp, 2519 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2520 } 2521 } 2522 out: 2523 wg_put_session(wgs, &psref); 2524 if (m != NULL) 2525 m_freem(m); 2526 if (free_encrypted_buf) 2527 kmem_intr_free(encrypted_buf, encrypted_len); 2528 } 2529 2530 static void 2531 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2532 { 2533 struct wg_session *wgs; 2534 struct wg_peer *wgp; 2535 struct psref psref; 2536 int error; 2537 uint8_t key[WG_HASH_LEN]; 2538 uint8_t cookie[WG_COOKIE_LEN]; 2539 2540 WG_TRACE("cookie msg received"); 2541 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2542 if (wgs == NULL) { 2543 WG_TRACE("No session found"); 2544 return; 2545 } 2546 wgp = wgs->wgs_peer; 2547 2548 if (!wgp->wgp_last_sent_mac1_valid) { 2549 WG_TRACE("No valid mac1 sent (or expired)"); 2550 goto out; 2551 } 2552 2553 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2554 sizeof(wgp->wgp_pubkey)); 2555 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 0, 2556 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2557 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2558 wgmc->wgmc_salt); 2559 if (error != 0) { 2560 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2561 "wg_algo_aead_dec for cookie failed: error=%d\n", error); 2562 goto out; 2563 } 2564 /* 2565 * [W] 6.6: Interaction with Cookie Reply System 2566 * "it should simply store the decrypted cookie value from the cookie 2567 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2568 * timer for retrying a handshake initiation message." 2569 */ 2570 wgp->wgp_latest_cookie_time = time_uptime; 2571 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2572 out: 2573 wg_put_session(wgs, &psref); 2574 } 2575 2576 static bool 2577 wg_validate_msg_length(struct wg_softc *wg, const struct mbuf *m) 2578 { 2579 struct wg_msg *wgm; 2580 size_t mlen; 2581 2582 mlen = m_length(m); 2583 if (__predict_false(mlen < sizeof(struct wg_msg))) 2584 return false; 2585 2586 wgm = mtod(m, struct wg_msg *); 2587 switch (wgm->wgm_type) { 2588 case WG_MSG_TYPE_INIT: 2589 if (__predict_true(mlen >= sizeof(struct wg_msg_init))) 2590 return true; 2591 break; 2592 case WG_MSG_TYPE_RESP: 2593 if (__predict_true(mlen >= sizeof(struct wg_msg_resp))) 2594 return true; 2595 break; 2596 case WG_MSG_TYPE_COOKIE: 2597 if (__predict_true(mlen >= sizeof(struct wg_msg_cookie))) 2598 return true; 2599 break; 2600 case WG_MSG_TYPE_DATA: 2601 if (__predict_true(mlen >= sizeof(struct wg_msg_data))) 2602 return true; 2603 break; 2604 default: 2605 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2606 "Unexpected msg type: %u\n", wgm->wgm_type); 2607 return false; 2608 } 2609 WG_DLOG("Invalid msg size: mlen=%lu type=%u\n", mlen, wgm->wgm_type); 2610 2611 return false; 2612 } 2613 2614 static void 2615 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2616 const struct sockaddr *src) 2617 { 2618 struct wg_msg *wgm; 2619 bool valid; 2620 2621 valid = wg_validate_msg_length(wg, m); 2622 if (!valid) { 2623 m_freem(m); 2624 return; 2625 } 2626 2627 wgm = mtod(m, struct wg_msg *); 2628 switch (wgm->wgm_type) { 2629 case WG_MSG_TYPE_INIT: 2630 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2631 break; 2632 case WG_MSG_TYPE_RESP: 2633 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2634 break; 2635 case WG_MSG_TYPE_COOKIE: 2636 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2637 break; 2638 case WG_MSG_TYPE_DATA: 2639 wg_handle_msg_data(wg, m, src); 2640 break; 2641 default: 2642 /* wg_validate_msg_length should already reject this case */ 2643 break; 2644 } 2645 } 2646 2647 static void 2648 wg_receive_packets(struct wg_softc *wg, const int af) 2649 { 2650 2651 for (;;) { 2652 int error, flags; 2653 struct socket *so; 2654 struct mbuf *m = NULL; 2655 struct uio dummy_uio; 2656 struct mbuf *paddr = NULL; 2657 struct sockaddr *src; 2658 2659 so = wg_get_so_by_af(wg->wg_worker, af); 2660 flags = MSG_DONTWAIT; 2661 dummy_uio.uio_resid = 1000000000; 2662 2663 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2664 &flags); 2665 if (error || m == NULL) { 2666 //if (error == EWOULDBLOCK) 2667 return; 2668 } 2669 2670 KASSERT(paddr != NULL); 2671 src = mtod(paddr, struct sockaddr *); 2672 2673 wg_handle_packet(wg, m, src); 2674 } 2675 } 2676 2677 static void 2678 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2679 { 2680 2681 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2682 } 2683 2684 static void 2685 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2686 { 2687 2688 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 2689 } 2690 2691 static void 2692 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 2693 { 2694 struct psref psref; 2695 struct wg_session *wgs; 2696 2697 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 2698 2699 if (!wgp->wgp_endpoint_available) { 2700 WGLOG(LOG_DEBUG, "No endpoint available\n"); 2701 /* XXX should do something? */ 2702 return; 2703 } 2704 2705 wgs = wg_get_stable_session(wgp, &psref); 2706 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 2707 wg_put_session(wgs, &psref); 2708 wg_send_handshake_msg_init(wg, wgp); 2709 } else { 2710 wg_put_session(wgs, &psref); 2711 /* rekey */ 2712 wgs = wg_get_unstable_session(wgp, &psref); 2713 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2714 wg_send_handshake_msg_init(wg, wgp); 2715 wg_put_session(wgs, &psref); 2716 } 2717 } 2718 2719 static void 2720 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 2721 { 2722 2723 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 2724 2725 mutex_enter(wgp->wgp_lock); 2726 if (wgp->wgp_endpoint_changing) { 2727 pserialize_perform(wgp->wgp_psz); 2728 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 2729 wg_psref_class); 2730 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 2731 wg_psref_class); 2732 wgp->wgp_endpoint_changing = false; 2733 } 2734 mutex_exit(wgp->wgp_lock); 2735 } 2736 2737 static void 2738 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 2739 { 2740 struct psref psref; 2741 struct wg_session *wgs; 2742 2743 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 2744 2745 wgs = wg_get_stable_session(wgp, &psref); 2746 wg_send_keepalive_msg(wgp, wgs); 2747 wg_put_session(wgs, &psref); 2748 } 2749 2750 static void 2751 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 2752 { 2753 struct wg_session *wgs; 2754 2755 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 2756 2757 mutex_enter(wgp->wgp_lock); 2758 wgs = wgp->wgp_session_unstable; 2759 mutex_enter(wgs->wgs_lock); 2760 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 2761 pserialize_perform(wgp->wgp_psz); 2762 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 2763 psref_target_init(&wgs->wgs_psref, wg_psref_class); 2764 wg_clear_states(wgs); 2765 wgs->wgs_state = WGS_STATE_UNKNOWN; 2766 } 2767 mutex_exit(wgs->wgs_lock); 2768 mutex_exit(wgp->wgp_lock); 2769 } 2770 2771 static void 2772 wg_process_peer_tasks(struct wg_softc *wg) 2773 { 2774 struct wg_peer *wgp; 2775 int s; 2776 2777 /* XXX should avoid checking all peers */ 2778 s = pserialize_read_enter(); 2779 WG_PEER_READER_FOREACH(wgp, wg) { 2780 struct psref psref; 2781 unsigned int tasks; 2782 2783 if (wgp->wgp_tasks == 0) 2784 continue; 2785 2786 wg_get_peer(wgp, &psref); 2787 pserialize_read_exit(s); 2788 2789 restart: 2790 tasks = atomic_swap_uint(&wgp->wgp_tasks, 0); 2791 KASSERT(tasks != 0); 2792 2793 WG_DLOG("tasks=%x\n", tasks); 2794 2795 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 2796 wg_task_send_init_message(wg, wgp); 2797 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 2798 wg_task_endpoint_changed(wg, wgp); 2799 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 2800 wg_task_send_keepalive_message(wg, wgp); 2801 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 2802 wg_task_destroy_prev_session(wg, wgp); 2803 2804 /* New tasks may be scheduled during processing tasks */ 2805 WG_DLOG("wgp_tasks=%d\n", wgp->wgp_tasks); 2806 if (wgp->wgp_tasks != 0) 2807 goto restart; 2808 2809 s = pserialize_read_enter(); 2810 wg_put_peer(wgp, &psref); 2811 } 2812 pserialize_read_exit(s); 2813 } 2814 2815 static void 2816 wg_worker(void *arg) 2817 { 2818 struct wg_softc *wg = arg; 2819 struct wg_worker *wgw = wg->wg_worker; 2820 bool todie = false; 2821 2822 KASSERT(wg != NULL); 2823 KASSERT(wgw != NULL); 2824 2825 while (!todie) { 2826 int reasons; 2827 int bound; 2828 2829 mutex_enter(&wgw->wgw_lock); 2830 /* New tasks may come during task handling */ 2831 while ((reasons = wgw->wgw_wakeup_reasons) == 0 && 2832 !(todie = wgw->wgw_todie)) 2833 cv_wait(&wgw->wgw_cv, &wgw->wgw_lock); 2834 wgw->wgw_wakeup_reasons = 0; 2835 mutex_exit(&wgw->wgw_lock); 2836 2837 bound = curlwp_bind(); 2838 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4)) 2839 wg_receive_packets(wg, AF_INET); 2840 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6)) 2841 wg_receive_packets(wg, AF_INET6); 2842 if (ISSET(reasons, WG_WAKEUP_REASON_PEER)) 2843 wg_process_peer_tasks(wg); 2844 curlwp_bindx(bound); 2845 } 2846 kthread_exit(0); 2847 } 2848 2849 static void 2850 wg_wakeup_worker(struct wg_worker *wgw, const int reason) 2851 { 2852 2853 mutex_enter(&wgw->wgw_lock); 2854 wgw->wgw_wakeup_reasons |= reason; 2855 cv_broadcast(&wgw->wgw_cv); 2856 mutex_exit(&wgw->wgw_lock); 2857 } 2858 2859 static int 2860 wg_bind_port(struct wg_softc *wg, const uint16_t port) 2861 { 2862 int error; 2863 struct wg_worker *wgw = wg->wg_worker; 2864 uint16_t old_port = wg->wg_listen_port; 2865 2866 if (port != 0 && old_port == port) 2867 return 0; 2868 2869 struct sockaddr_in _sin, *sin = &_sin; 2870 sin->sin_len = sizeof(*sin); 2871 sin->sin_family = AF_INET; 2872 sin->sin_addr.s_addr = INADDR_ANY; 2873 sin->sin_port = htons(port); 2874 2875 error = sobind(wgw->wgw_so4, sintosa(sin), curlwp); 2876 if (error != 0) 2877 return error; 2878 2879 #ifdef INET6 2880 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 2881 sin6->sin6_len = sizeof(*sin6); 2882 sin6->sin6_family = AF_INET6; 2883 sin6->sin6_addr = in6addr_any; 2884 sin6->sin6_port = htons(port); 2885 2886 error = sobind(wgw->wgw_so6, sin6tosa(sin6), curlwp); 2887 if (error != 0) 2888 return error; 2889 #endif 2890 2891 wg->wg_listen_port = port; 2892 2893 return 0; 2894 } 2895 2896 static void 2897 wg_so_upcall(struct socket *so, void *arg, int events, int waitflag) 2898 { 2899 struct wg_worker *wgw = arg; 2900 int reason; 2901 2902 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 2903 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 : 2904 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6; 2905 wg_wakeup_worker(wgw, reason); 2906 } 2907 2908 static int 2909 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 2910 struct sockaddr *src, void *arg) 2911 { 2912 struct wg_softc *wg = arg; 2913 struct wg_msg wgm; 2914 struct mbuf *m = *mp; 2915 2916 WG_TRACE("enter"); 2917 2918 /* Verify the mbuf chain is long enough to have a wg msg header. */ 2919 KASSERT(offset <= m_length(m)); 2920 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 2921 m_freem(m); 2922 return -1; 2923 } 2924 2925 /* 2926 * Copy the message header (32-bit message type) out -- we'll 2927 * worry about contiguity and alignment later. 2928 */ 2929 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 2930 WG_DLOG("type=%d\n", wgm.wgm_type); 2931 2932 /* 2933 * Handle DATA packets promptly as they arrive. Other packets 2934 * may require expensive public-key crypto and are not as 2935 * sensitive to latency, so defer them to the worker thread. 2936 */ 2937 switch (wgm.wgm_type) { 2938 case WG_MSG_TYPE_DATA: 2939 m_adj(m, offset); 2940 wg_handle_msg_data(wg, m, src); 2941 *mp = NULL; 2942 return 1; 2943 default: 2944 break; 2945 } 2946 2947 return 0; 2948 } 2949 2950 static int 2951 wg_worker_socreate(struct wg_softc *wg, struct wg_worker *wgw, const int af, 2952 struct socket **sop) 2953 { 2954 int error; 2955 struct socket *so; 2956 2957 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 2958 if (error != 0) 2959 return error; 2960 2961 solock(so); 2962 so->so_upcallarg = wgw; 2963 so->so_upcall = wg_so_upcall; 2964 so->so_rcv.sb_flags |= SB_UPCALL; 2965 if (af == AF_INET) 2966 in_pcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 2967 #if INET6 2968 else 2969 in6_pcb_register_overudp_cb(sotoin6pcb(so), wg_overudp_cb, wg); 2970 #endif 2971 sounlock(so); 2972 2973 *sop = so; 2974 2975 return 0; 2976 } 2977 2978 static int 2979 wg_worker_init(struct wg_softc *wg) 2980 { 2981 int error; 2982 struct wg_worker *wgw; 2983 const char *ifname = wg->wg_if.if_xname; 2984 struct socket *so; 2985 2986 wgw = kmem_zalloc(sizeof(struct wg_worker), KM_SLEEP); 2987 2988 mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_NONE); 2989 cv_init(&wgw->wgw_cv, ifname); 2990 wgw->wgw_todie = false; 2991 wgw->wgw_wakeup_reasons = 0; 2992 2993 error = wg_worker_socreate(wg, wgw, AF_INET, &so); 2994 if (error != 0) 2995 goto error; 2996 wgw->wgw_so4 = so; 2997 #ifdef INET6 2998 error = wg_worker_socreate(wg, wgw, AF_INET6, &so); 2999 if (error != 0) 3000 goto error; 3001 wgw->wgw_so6 = so; 3002 #endif 3003 3004 wg->wg_worker = wgw; 3005 3006 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, 3007 NULL, wg_worker, wg, &wg->wg_worker_lwp, "%s", ifname); 3008 if (error != 0) 3009 goto error; 3010 3011 return 0; 3012 3013 error: 3014 #ifdef INET6 3015 if (wgw->wgw_so6 != NULL) 3016 soclose(wgw->wgw_so6); 3017 #endif 3018 if (wgw->wgw_so4 != NULL) 3019 soclose(wgw->wgw_so4); 3020 cv_destroy(&wgw->wgw_cv); 3021 mutex_destroy(&wgw->wgw_lock); 3022 3023 return error; 3024 } 3025 3026 static void 3027 wg_worker_destroy(struct wg_softc *wg) 3028 { 3029 struct wg_worker *wgw = wg->wg_worker; 3030 3031 mutex_enter(&wgw->wgw_lock); 3032 wgw->wgw_todie = true; 3033 wgw->wgw_wakeup_reasons = 0; 3034 cv_broadcast(&wgw->wgw_cv); 3035 mutex_exit(&wgw->wgw_lock); 3036 3037 kthread_join(wg->wg_worker_lwp); 3038 3039 #ifdef INET6 3040 soclose(wgw->wgw_so6); 3041 #endif 3042 soclose(wgw->wgw_so4); 3043 cv_destroy(&wgw->wgw_cv); 3044 mutex_destroy(&wgw->wgw_lock); 3045 kmem_free(wg->wg_worker, sizeof(struct wg_worker)); 3046 wg->wg_worker = NULL; 3047 } 3048 3049 static bool 3050 wg_session_hit_limits(struct wg_session *wgs) 3051 { 3052 3053 /* 3054 * [W] 6.2: Transport Message Limits 3055 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3056 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3057 * comes first, WireGuard will refuse to send any more transport data 3058 * messages using the current secure session, ..." 3059 */ 3060 KASSERT(wgs->wgs_time_established != 0); 3061 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3062 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3063 return true; 3064 } else if (wg_session_get_send_counter(wgs) > 3065 wg_reject_after_messages) { 3066 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3067 return true; 3068 } 3069 3070 return false; 3071 } 3072 3073 static void 3074 wg_peer_softint(void *arg) 3075 { 3076 struct wg_peer *wgp = arg; 3077 struct wg_session *wgs; 3078 struct mbuf *m; 3079 struct psref psref; 3080 3081 wgs = wg_get_stable_session(wgp, &psref); 3082 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) { 3083 /* XXX how to treat? */ 3084 WG_TRACE("skipped"); 3085 goto out; 3086 } 3087 if (wg_session_hit_limits(wgs)) { 3088 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3089 goto out; 3090 } 3091 WG_TRACE("running"); 3092 3093 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3094 wg_send_data_msg(wgp, wgs, m); 3095 } 3096 out: 3097 wg_put_session(wgs, &psref); 3098 } 3099 3100 static void 3101 wg_rekey_timer(void *arg) 3102 { 3103 struct wg_peer *wgp = arg; 3104 3105 mutex_enter(wgp->wgp_lock); 3106 if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) { 3107 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3108 } 3109 mutex_exit(wgp->wgp_lock); 3110 } 3111 3112 static void 3113 wg_purge_pending_packets(struct wg_peer *wgp) 3114 { 3115 struct mbuf *m; 3116 3117 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3118 m_freem(m); 3119 } 3120 } 3121 3122 static void 3123 wg_handshake_timeout_timer(void *arg) 3124 { 3125 struct wg_peer *wgp = arg; 3126 struct wg_session *wgs; 3127 struct psref psref; 3128 3129 WG_TRACE("enter"); 3130 3131 mutex_enter(wgp->wgp_lock); 3132 if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) { 3133 mutex_exit(wgp->wgp_lock); 3134 return; 3135 } 3136 mutex_exit(wgp->wgp_lock); 3137 3138 KASSERT(wgp->wgp_handshake_start_time != 0); 3139 wgs = wg_get_unstable_session(wgp, &psref); 3140 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 3141 3142 /* [W] 6.4 Handshake Initiation Retransmission */ 3143 if ((time_uptime - wgp->wgp_handshake_start_time) > 3144 wg_rekey_attempt_time) { 3145 /* Give up handshaking */ 3146 wgs->wgs_state = WGS_STATE_UNKNOWN; 3147 wg_clear_states(wgs); 3148 wgp->wgp_state = WGP_STATE_GIVEUP; 3149 wgp->wgp_handshake_start_time = 0; 3150 wg_put_session(wgs, &psref); 3151 WG_TRACE("give up"); 3152 /* 3153 * If a new data packet comes, handshaking will be retried 3154 * and a new session would be established at that time, 3155 * however we don't want to send pending packets then. 3156 */ 3157 wg_purge_pending_packets(wgp); 3158 return; 3159 } 3160 3161 /* No response for an initiation message sent, retry handshaking */ 3162 wgs->wgs_state = WGS_STATE_UNKNOWN; 3163 wg_clear_states(wgs); 3164 wg_put_session(wgs, &psref); 3165 3166 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3167 } 3168 3169 static struct wg_peer * 3170 wg_alloc_peer(struct wg_softc *wg) 3171 { 3172 struct wg_peer *wgp; 3173 3174 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3175 3176 wgp->wgp_sc = wg; 3177 wgp->wgp_state = WGP_STATE_INIT; 3178 wgp->wgp_q = pcq_create(1024, KM_SLEEP); 3179 wgp->wgp_si = softint_establish(SOFTINT_NET, wg_peer_softint, wgp); 3180 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3181 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3182 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3183 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3184 wg_handshake_timeout_timer, wgp); 3185 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3186 callout_setfunc(&wgp->wgp_session_dtor_timer, 3187 wg_session_dtor_timer, wgp); 3188 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3189 wgp->wgp_endpoint_changing = false; 3190 wgp->wgp_endpoint_available = false; 3191 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3192 wgp->wgp_psz = pserialize_create(); 3193 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3194 3195 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3196 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3197 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3198 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3199 3200 struct wg_session *wgs; 3201 wgp->wgp_session_stable = 3202 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3203 wgp->wgp_session_unstable = 3204 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3205 wgs = wgp->wgp_session_stable; 3206 wgs->wgs_peer = wgp; 3207 wgs->wgs_state = WGS_STATE_UNKNOWN; 3208 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3209 wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3210 #ifndef __HAVE_ATOMIC64_LOADSTORE 3211 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3212 #endif 3213 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3214 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE); 3215 3216 wgs = wgp->wgp_session_unstable; 3217 wgs->wgs_peer = wgp; 3218 wgs->wgs_state = WGS_STATE_UNKNOWN; 3219 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3220 wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3221 #ifndef __HAVE_ATOMIC64_LOADSTORE 3222 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3223 #endif 3224 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3225 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE); 3226 3227 return wgp; 3228 } 3229 3230 static void 3231 wg_destroy_peer(struct wg_peer *wgp) 3232 { 3233 struct wg_session *wgs; 3234 struct wg_softc *wg = wgp->wgp_sc; 3235 3236 rw_enter(wg->wg_rwlock, RW_WRITER); 3237 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3238 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3239 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3240 struct radix_node *rn; 3241 3242 KASSERT(rnh != NULL); 3243 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3244 &wga->wga_sa_mask, rnh); 3245 if (rn == NULL) { 3246 char addrstr[128]; 3247 sockaddr_format(&wga->wga_sa_addr, addrstr, 3248 sizeof(addrstr)); 3249 WGLOG(LOG_WARNING, "Couldn't delete %s", addrstr); 3250 } 3251 } 3252 rw_exit(wg->wg_rwlock); 3253 3254 softint_disestablish(wgp->wgp_si); 3255 callout_halt(&wgp->wgp_rekey_timer, NULL); 3256 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3257 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3258 3259 wgs = wgp->wgp_session_unstable; 3260 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 3261 mutex_obj_free(wgs->wgs_lock); 3262 mutex_destroy(&wgs->wgs_recvwin->lock); 3263 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3264 #ifndef __HAVE_ATOMIC64_LOADSTORE 3265 mutex_destroy(&wgs->wgs_send_counter_lock); 3266 #endif 3267 kmem_free(wgs, sizeof(*wgs)); 3268 wgs = wgp->wgp_session_stable; 3269 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 3270 mutex_obj_free(wgs->wgs_lock); 3271 mutex_destroy(&wgs->wgs_recvwin->lock); 3272 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3273 #ifndef __HAVE_ATOMIC64_LOADSTORE 3274 mutex_destroy(&wgs->wgs_send_counter_lock); 3275 #endif 3276 kmem_free(wgs, sizeof(*wgs)); 3277 3278 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3279 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3280 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3281 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3282 3283 pserialize_destroy(wgp->wgp_psz); 3284 pcq_destroy(wgp->wgp_q); 3285 mutex_obj_free(wgp->wgp_lock); 3286 3287 kmem_free(wgp, sizeof(*wgp)); 3288 } 3289 3290 static void 3291 wg_destroy_all_peers(struct wg_softc *wg) 3292 { 3293 struct wg_peer *wgp; 3294 3295 restart: 3296 mutex_enter(wg->wg_lock); 3297 WG_PEER_WRITER_FOREACH(wgp, wg) { 3298 WG_PEER_WRITER_REMOVE(wgp); 3299 mutex_enter(wgp->wgp_lock); 3300 wgp->wgp_state = WGP_STATE_DESTROYING; 3301 pserialize_perform(wgp->wgp_psz); 3302 mutex_exit(wgp->wgp_lock); 3303 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3304 break; 3305 } 3306 mutex_exit(wg->wg_lock); 3307 3308 if (wgp == NULL) 3309 return; 3310 3311 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3312 3313 wg_destroy_peer(wgp); 3314 3315 goto restart; 3316 } 3317 3318 static int 3319 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3320 { 3321 struct wg_peer *wgp; 3322 3323 mutex_enter(wg->wg_lock); 3324 WG_PEER_WRITER_FOREACH(wgp, wg) { 3325 if (strcmp(wgp->wgp_name, name) == 0) 3326 break; 3327 } 3328 if (wgp != NULL) { 3329 WG_PEER_WRITER_REMOVE(wgp); 3330 wg->wg_npeers--; 3331 mutex_enter(wgp->wgp_lock); 3332 wgp->wgp_state = WGP_STATE_DESTROYING; 3333 pserialize_perform(wgp->wgp_psz); 3334 mutex_exit(wgp->wgp_lock); 3335 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3336 } 3337 mutex_exit(wg->wg_lock); 3338 3339 if (wgp == NULL) 3340 return ENOENT; 3341 3342 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3343 3344 wg_destroy_peer(wgp); 3345 3346 return 0; 3347 } 3348 3349 static int 3350 wg_if_attach(struct wg_softc *wg) 3351 { 3352 int error; 3353 3354 wg->wg_if.if_addrlen = 0; 3355 wg->wg_if.if_mtu = WG_MTU; 3356 wg->wg_if.if_flags = IFF_POINTOPOINT; 3357 wg->wg_if.if_extflags = IFEF_NO_LINK_STATE_CHANGE; 3358 wg->wg_if.if_extflags |= IFEF_MPSAFE; 3359 wg->wg_if.if_ioctl = wg_ioctl; 3360 wg->wg_if.if_output = wg_output; 3361 wg->wg_if.if_init = wg_init; 3362 wg->wg_if.if_stop = wg_stop; 3363 wg->wg_if.if_type = IFT_OTHER; 3364 wg->wg_if.if_dlt = DLT_NULL; 3365 wg->wg_if.if_softc = wg; 3366 IFQ_SET_READY(&wg->wg_if.if_snd); 3367 3368 error = if_initialize(&wg->wg_if); 3369 if (error != 0) 3370 return error; 3371 3372 if_alloc_sadl(&wg->wg_if); 3373 if_register(&wg->wg_if); 3374 3375 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3376 3377 return 0; 3378 } 3379 3380 static int 3381 wg_clone_create(struct if_clone *ifc, int unit) 3382 { 3383 struct wg_softc *wg; 3384 int error; 3385 3386 wg = kmem_zalloc(sizeof(struct wg_softc), KM_SLEEP); 3387 3388 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3389 3390 error = wg_worker_init(wg); 3391 if (error != 0) { 3392 kmem_free(wg, sizeof(struct wg_softc)); 3393 return error; 3394 } 3395 3396 rn_inithead((void **)&wg->wg_rtable_ipv4, 3397 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3398 #ifdef INET6 3399 rn_inithead((void **)&wg->wg_rtable_ipv6, 3400 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3401 #endif 3402 3403 PSLIST_INIT(&wg->wg_peers); 3404 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3405 wg->wg_rwlock = rw_obj_alloc(); 3406 wg->wg_ops = &wg_ops_rumpkernel; 3407 3408 error = wg_if_attach(wg); 3409 if (error != 0) { 3410 wg_worker_destroy(wg); 3411 if (wg->wg_rtable_ipv4 != NULL) 3412 free(wg->wg_rtable_ipv4, M_RTABLE); 3413 if (wg->wg_rtable_ipv6 != NULL) 3414 free(wg->wg_rtable_ipv6, M_RTABLE); 3415 PSLIST_DESTROY(&wg->wg_peers); 3416 mutex_obj_free(wg->wg_lock); 3417 kmem_free(wg, sizeof(struct wg_softc)); 3418 return error; 3419 } 3420 3421 mutex_enter(&wg_softcs.lock); 3422 LIST_INSERT_HEAD(&wg_softcs.list, wg, wg_list); 3423 mutex_exit(&wg_softcs.lock); 3424 3425 return 0; 3426 } 3427 3428 static int 3429 wg_clone_destroy(struct ifnet *ifp) 3430 { 3431 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3432 3433 mutex_enter(&wg_softcs.lock); 3434 LIST_REMOVE(wg, wg_list); 3435 mutex_exit(&wg_softcs.lock); 3436 3437 #ifdef WG_RUMPKERNEL 3438 if (wg_user_mode(wg)) { 3439 rumpuser_wg_destroy(wg->wg_user); 3440 wg->wg_user = NULL; 3441 } 3442 #endif 3443 3444 bpf_detach(ifp); 3445 if_detach(ifp); 3446 wg_worker_destroy(wg); 3447 wg_destroy_all_peers(wg); 3448 if (wg->wg_rtable_ipv4 != NULL) 3449 free(wg->wg_rtable_ipv4, M_RTABLE); 3450 if (wg->wg_rtable_ipv6 != NULL) 3451 free(wg->wg_rtable_ipv6, M_RTABLE); 3452 3453 PSLIST_DESTROY(&wg->wg_peers); 3454 mutex_obj_free(wg->wg_lock); 3455 rw_obj_free(wg->wg_rwlock); 3456 3457 kmem_free(wg, sizeof(struct wg_softc)); 3458 3459 return 0; 3460 } 3461 3462 static struct wg_peer * 3463 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3464 struct psref *psref) 3465 { 3466 struct radix_node_head *rnh; 3467 struct radix_node *rn; 3468 struct wg_peer *wgp = NULL; 3469 struct wg_allowedip *wga; 3470 3471 #ifdef WG_DEBUG_LOG 3472 char addrstr[128]; 3473 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3474 WG_DLOG("sa=%s\n", addrstr); 3475 #endif 3476 3477 rw_enter(wg->wg_rwlock, RW_READER); 3478 3479 rnh = wg_rnh(wg, sa->sa_family); 3480 if (rnh == NULL) 3481 goto out; 3482 3483 rn = rnh->rnh_matchaddr(sa, rnh); 3484 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3485 goto out; 3486 3487 WG_TRACE("success"); 3488 3489 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3490 wgp = wga->wga_peer; 3491 wg_get_peer(wgp, psref); 3492 3493 out: 3494 rw_exit(wg->wg_rwlock); 3495 return wgp; 3496 } 3497 3498 static void 3499 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3500 struct wg_session *wgs, struct wg_msg_data *wgmd) 3501 { 3502 3503 memset(wgmd, 0, sizeof(*wgmd)); 3504 wgmd->wgmd_type = WG_MSG_TYPE_DATA; 3505 wgmd->wgmd_receiver = wgs->wgs_receiver_index; 3506 /* [W] 5.4.6: msg.counter := Nm^send */ 3507 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3508 wgmd->wgmd_counter = wg_session_inc_send_counter(wgs); 3509 WG_DLOG("counter=%"PRIu64"\n", wgmd->wgmd_counter); 3510 } 3511 3512 static int 3513 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3514 const struct rtentry *rt) 3515 { 3516 struct wg_softc *wg = ifp->if_softc; 3517 int error = 0; 3518 int bound; 3519 struct psref psref; 3520 3521 /* TODO make the nest limit configurable via sysctl */ 3522 error = if_tunnel_check_nesting(ifp, m, 1); 3523 if (error != 0) { 3524 m_freem(m); 3525 WGLOG(LOG_ERR, "tunneling loop detected and packet dropped\n"); 3526 return error; 3527 } 3528 3529 bound = curlwp_bind(); 3530 3531 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3532 3533 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3534 3535 m->m_flags &= ~(M_BCAST|M_MCAST); 3536 3537 struct wg_peer *wgp = wg_pick_peer_by_sa(wg, dst, &psref); 3538 if (wgp == NULL) { 3539 WG_TRACE("peer not found"); 3540 error = EHOSTUNREACH; 3541 goto error; 3542 } 3543 3544 /* Clear checksum-offload flags. */ 3545 m->m_pkthdr.csum_flags = 0; 3546 m->m_pkthdr.csum_data = 0; 3547 3548 if (!pcq_put(wgp->wgp_q, m)) { 3549 error = ENOBUFS; 3550 goto error; 3551 } 3552 3553 struct psref psref_wgs; 3554 struct wg_session *wgs; 3555 wgs = wg_get_stable_session(wgp, &psref_wgs); 3556 if (wgs->wgs_state == WGS_STATE_ESTABLISHED && 3557 !wg_session_hit_limits(wgs)) { 3558 kpreempt_disable(); 3559 softint_schedule(wgp->wgp_si); 3560 kpreempt_enable(); 3561 WG_TRACE("softint scheduled"); 3562 } else { 3563 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3564 WG_TRACE("softint NOT scheduled"); 3565 } 3566 wg_put_session(wgs, &psref_wgs); 3567 wg_put_peer(wgp, &psref); 3568 3569 return 0; 3570 3571 error: 3572 if (wgp != NULL) 3573 wg_put_peer(wgp, &psref); 3574 if (m != NULL) 3575 m_freem(m); 3576 curlwp_bindx(bound); 3577 return error; 3578 } 3579 3580 static int 3581 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3582 { 3583 struct psref psref; 3584 struct wg_sockaddr *wgsa; 3585 int error; 3586 struct socket *so = wg_get_so_by_peer(wgp); 3587 3588 solock(so); 3589 wgsa = wg_get_endpoint_sa(wgp, &psref); 3590 if (wgsatosa(wgsa)->sa_family == AF_INET) { 3591 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 3592 } else { 3593 #ifdef INET6 3594 error = udp6_output(sotoin6pcb(so), m, wgsatosin6(wgsa), 3595 NULL, curlwp); 3596 #else 3597 error = EPROTONOSUPPORT; 3598 #endif 3599 } 3600 wg_put_sa(wgp, wgsa, &psref); 3601 sounlock(so); 3602 3603 return error; 3604 } 3605 3606 /* Inspired by pppoe_get_mbuf */ 3607 static struct mbuf * 3608 wg_get_mbuf(size_t leading_len, size_t len) 3609 { 3610 struct mbuf *m; 3611 3612 m = m_gethdr(M_DONTWAIT, MT_DATA); 3613 if (m == NULL) 3614 return NULL; 3615 if (len + leading_len > MHLEN) { 3616 m_clget(m, M_DONTWAIT); 3617 if ((m->m_flags & M_EXT) == 0) { 3618 m_free(m); 3619 return NULL; 3620 } 3621 } 3622 m->m_data += leading_len; 3623 m->m_pkthdr.len = m->m_len = len; 3624 3625 return m; 3626 } 3627 3628 static int 3629 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 3630 struct mbuf *m) 3631 { 3632 struct wg_softc *wg = wgp->wgp_sc; 3633 int error; 3634 size_t inner_len, padded_len, encrypted_len; 3635 char *padded_buf = NULL; 3636 size_t mlen; 3637 struct wg_msg_data *wgmd; 3638 bool free_padded_buf = false; 3639 struct mbuf *n; 3640 size_t leading_len = max_linkhdr + sizeof(struct ip6_hdr) + 3641 sizeof(struct udphdr); 3642 3643 mlen = m_length(m); 3644 inner_len = mlen; 3645 padded_len = roundup(mlen, 16); 3646 encrypted_len = padded_len + WG_AUTHTAG_LEN; 3647 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 3648 inner_len, padded_len, encrypted_len); 3649 if (mlen != 0) { 3650 bool success; 3651 success = m_ensure_contig(&m, padded_len); 3652 if (success) { 3653 padded_buf = mtod(m, char *); 3654 } else { 3655 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 3656 if (padded_buf == NULL) { 3657 error = ENOBUFS; 3658 goto end; 3659 } 3660 free_padded_buf = true; 3661 m_copydata(m, 0, mlen, padded_buf); 3662 } 3663 memset(padded_buf + mlen, 0, padded_len - inner_len); 3664 } 3665 3666 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 3667 if (n == NULL) { 3668 error = ENOBUFS; 3669 goto end; 3670 } 3671 wgmd = mtod(n, struct wg_msg_data *); 3672 wg_fill_msg_data(wg, wgp, wgs, wgmd); 3673 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 3674 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 3675 wgs->wgs_tkey_send, wgmd->wgmd_counter, padded_buf, padded_len, 3676 NULL, 0); 3677 3678 error = wg->wg_ops->send_data_msg(wgp, n); 3679 if (error == 0) { 3680 struct ifnet *ifp = &wg->wg_if; 3681 if_statadd(ifp, if_obytes, mlen); 3682 if_statinc(ifp, if_opackets); 3683 if (wgs->wgs_is_initiator && 3684 wgs->wgs_time_last_data_sent == 0) { 3685 /* 3686 * [W] 6.2 Transport Message Limits 3687 * "if a peer is the initiator of a current secure 3688 * session, WireGuard will send a handshake initiation 3689 * message to begin a new secure session if, after 3690 * transmitting a transport data message, the current 3691 * secure session is REKEY-AFTER-TIME seconds old," 3692 */ 3693 wg_schedule_rekey_timer(wgp); 3694 } 3695 wgs->wgs_time_last_data_sent = time_uptime; 3696 if (wg_session_get_send_counter(wgs) >= 3697 wg_rekey_after_messages) { 3698 /* 3699 * [W] 6.2 Transport Message Limits 3700 * "WireGuard will try to create a new session, by 3701 * sending a handshake initiation message (section 3702 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 3703 * transport data messages..." 3704 */ 3705 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3706 } 3707 } 3708 end: 3709 m_freem(m); 3710 if (free_padded_buf) 3711 kmem_intr_free(padded_buf, padded_len); 3712 return error; 3713 } 3714 3715 static void 3716 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 3717 { 3718 pktqueue_t *pktq; 3719 size_t pktlen; 3720 3721 KASSERT(af == AF_INET || af == AF_INET6); 3722 3723 WG_TRACE(""); 3724 3725 m_set_rcvif(m, ifp); 3726 pktlen = m->m_pkthdr.len; 3727 3728 bpf_mtap_af(ifp, af, m, BPF_D_IN); 3729 3730 switch (af) { 3731 case AF_INET: 3732 pktq = ip_pktq; 3733 break; 3734 #ifdef INET6 3735 case AF_INET6: 3736 pktq = ip6_pktq; 3737 break; 3738 #endif 3739 default: 3740 panic("invalid af=%d", af); 3741 } 3742 3743 const u_int h = curcpu()->ci_index; 3744 if (__predict_true(pktq_enqueue(pktq, m, h))) { 3745 if_statadd(ifp, if_ibytes, pktlen); 3746 if_statinc(ifp, if_ipackets); 3747 } else { 3748 m_freem(m); 3749 } 3750 } 3751 3752 static void 3753 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 3754 const uint8_t privkey[WG_STATIC_KEY_LEN]) 3755 { 3756 3757 crypto_scalarmult_base(pubkey, privkey); 3758 } 3759 3760 static int 3761 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 3762 { 3763 struct radix_node_head *rnh; 3764 struct radix_node *rn; 3765 int error = 0; 3766 3767 rw_enter(wg->wg_rwlock, RW_WRITER); 3768 rnh = wg_rnh(wg, wga->wga_family); 3769 KASSERT(rnh != NULL); 3770 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 3771 wga->wga_nodes); 3772 rw_exit(wg->wg_rwlock); 3773 3774 if (rn == NULL) 3775 error = EEXIST; 3776 3777 return error; 3778 } 3779 3780 static int 3781 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 3782 struct wg_peer **wgpp) 3783 { 3784 int error = 0; 3785 const void *pubkey; 3786 size_t pubkey_len; 3787 const void *psk; 3788 size_t psk_len; 3789 const char *name = NULL; 3790 3791 if (prop_dictionary_get_string(peer, "name", &name)) { 3792 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 3793 error = EINVAL; 3794 goto out; 3795 } 3796 } 3797 3798 if (!prop_dictionary_get_data(peer, "public_key", 3799 &pubkey, &pubkey_len)) { 3800 error = EINVAL; 3801 goto out; 3802 } 3803 #ifdef WG_DEBUG_DUMP 3804 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n", pubkey, pubkey_len); 3805 for (int _i = 0; _i < pubkey_len; _i++) 3806 log(LOG_DEBUG, "%c", ((const char *)pubkey)[_i]); 3807 log(LOG_DEBUG, "\n"); 3808 #endif 3809 3810 struct wg_peer *wgp = wg_alloc_peer(wg); 3811 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 3812 if (name != NULL) 3813 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 3814 3815 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 3816 if (psk_len != sizeof(wgp->wgp_psk)) { 3817 error = EINVAL; 3818 goto out; 3819 } 3820 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 3821 } 3822 3823 struct sockaddr_storage sockaddr; 3824 const void *addr; 3825 size_t addr_len; 3826 3827 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 3828 goto skip_endpoint; 3829 memcpy(&sockaddr, addr, addr_len); 3830 switch (sockaddr.ss_family) { 3831 case AF_INET: { 3832 struct sockaddr_in sin; 3833 sockaddr_copy(sintosa(&sin), sizeof(sin), 3834 (const struct sockaddr *)&sockaddr); 3835 sockaddr_copy(sintosa(&wgp->wgp_sin), 3836 sizeof(wgp->wgp_sin), (const struct sockaddr *)&sockaddr); 3837 char addrstr[128]; 3838 sockaddr_format(sintosa(&sin), addrstr, sizeof(addrstr)); 3839 WG_DLOG("addr=%s\n", addrstr); 3840 break; 3841 } 3842 #ifdef INET6 3843 case AF_INET6: { 3844 struct sockaddr_in6 sin6; 3845 char addrstr[128]; 3846 sockaddr_copy(sintosa(&sin6), sizeof(sin6), 3847 (const struct sockaddr *)&sockaddr); 3848 sockaddr_format(sintosa(&sin6), addrstr, sizeof(addrstr)); 3849 WG_DLOG("addr=%s\n", addrstr); 3850 sockaddr_copy(sin6tosa(&wgp->wgp_sin6), 3851 sizeof(wgp->wgp_sin6), (const struct sockaddr *)&sockaddr); 3852 break; 3853 } 3854 #endif 3855 default: 3856 break; 3857 } 3858 wgp->wgp_endpoint_available = true; 3859 3860 prop_array_t allowedips; 3861 skip_endpoint: 3862 allowedips = prop_dictionary_get(peer, "allowedips"); 3863 if (allowedips == NULL) 3864 goto skip; 3865 3866 prop_object_iterator_t _it = prop_array_iterator(allowedips); 3867 prop_dictionary_t prop_allowedip; 3868 int j = 0; 3869 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 3870 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 3871 3872 if (!prop_dictionary_get_int(prop_allowedip, "family", 3873 &wga->wga_family)) 3874 continue; 3875 if (!prop_dictionary_get_data(prop_allowedip, "ip", 3876 &addr, &addr_len)) 3877 continue; 3878 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 3879 &wga->wga_cidr)) 3880 continue; 3881 3882 switch (wga->wga_family) { 3883 case AF_INET: { 3884 struct sockaddr_in sin; 3885 char addrstr[128]; 3886 struct in_addr mask; 3887 struct sockaddr_in sin_mask; 3888 3889 if (addr_len != sizeof(struct in_addr)) 3890 return EINVAL; 3891 memcpy(&wga->wga_addr4, addr, addr_len); 3892 3893 sockaddr_in_init(&sin, (const struct in_addr *)addr, 3894 0); 3895 sockaddr_copy(&wga->wga_sa_addr, 3896 sizeof(sin), sintosa(&sin)); 3897 3898 sockaddr_format(sintosa(&sin), 3899 addrstr, sizeof(addrstr)); 3900 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 3901 3902 in_len2mask(&mask, wga->wga_cidr); 3903 sockaddr_in_init(&sin_mask, &mask, 0); 3904 sockaddr_copy(&wga->wga_sa_mask, 3905 sizeof(sin_mask), sintosa(&sin_mask)); 3906 3907 break; 3908 } 3909 #ifdef INET6 3910 case AF_INET6: { 3911 struct sockaddr_in6 sin6; 3912 char addrstr[128]; 3913 struct in6_addr mask; 3914 struct sockaddr_in6 sin6_mask; 3915 3916 if (addr_len != sizeof(struct in6_addr)) 3917 return EINVAL; 3918 memcpy(&wga->wga_addr6, addr, addr_len); 3919 3920 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 3921 0, 0, 0); 3922 sockaddr_copy(&wga->wga_sa_addr, 3923 sizeof(sin6), sin6tosa(&sin6)); 3924 3925 sockaddr_format(sin6tosa(&sin6), 3926 addrstr, sizeof(addrstr)); 3927 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 3928 3929 in6_prefixlen2mask(&mask, wga->wga_cidr); 3930 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 3931 sockaddr_copy(&wga->wga_sa_mask, 3932 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 3933 3934 break; 3935 } 3936 #endif 3937 default: 3938 error = EINVAL; 3939 goto out; 3940 } 3941 wga->wga_peer = wgp; 3942 3943 error = wg_rtable_add_route(wg, wga); 3944 if (error != 0) 3945 goto out; 3946 3947 j++; 3948 } 3949 wgp->wgp_n_allowedips = j; 3950 skip: 3951 *wgpp = wgp; 3952 out: 3953 return error; 3954 } 3955 3956 static int 3957 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 3958 { 3959 int error; 3960 char *buf; 3961 3962 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 3963 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 3964 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 3965 if (error != 0) 3966 return error; 3967 buf[ifd->ifd_len] = '\0'; 3968 #ifdef WG_DEBUG_DUMP 3969 for (int i = 0; i < ifd->ifd_len; i++) 3970 log(LOG_DEBUG, "%c", buf[i]); 3971 log(LOG_DEBUG, "\n"); 3972 #endif 3973 *_buf = buf; 3974 return 0; 3975 } 3976 3977 static int 3978 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 3979 { 3980 int error; 3981 prop_dictionary_t prop_dict; 3982 char *buf = NULL; 3983 const void *privkey; 3984 size_t privkey_len; 3985 3986 error = wg_alloc_prop_buf(&buf, ifd); 3987 if (error != 0) 3988 return error; 3989 error = EINVAL; 3990 prop_dict = prop_dictionary_internalize(buf); 3991 if (prop_dict == NULL) 3992 goto out; 3993 if (!prop_dictionary_get_data(prop_dict, "private_key", 3994 &privkey, &privkey_len)) 3995 goto out; 3996 #ifdef WG_DEBUG_DUMP 3997 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n", privkey, privkey_len); 3998 for (int i = 0; i < privkey_len; i++) 3999 log(LOG_DEBUG, "%c", ((const char *)privkey)[i]); 4000 log(LOG_DEBUG, "\n"); 4001 #endif 4002 if (privkey_len != WG_STATIC_KEY_LEN) 4003 goto out; 4004 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4005 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4006 error = 0; 4007 4008 out: 4009 kmem_free(buf, ifd->ifd_len + 1); 4010 return error; 4011 } 4012 4013 static int 4014 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4015 { 4016 int error; 4017 prop_dictionary_t prop_dict; 4018 char *buf = NULL; 4019 uint16_t port; 4020 4021 error = wg_alloc_prop_buf(&buf, ifd); 4022 if (error != 0) 4023 return error; 4024 error = EINVAL; 4025 prop_dict = prop_dictionary_internalize(buf); 4026 if (prop_dict == NULL) 4027 goto out; 4028 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4029 goto out; 4030 4031 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4032 4033 out: 4034 kmem_free(buf, ifd->ifd_len + 1); 4035 return error; 4036 } 4037 4038 static int 4039 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4040 { 4041 int error; 4042 prop_dictionary_t prop_dict; 4043 char *buf = NULL; 4044 struct wg_peer *wgp = NULL; 4045 4046 error = wg_alloc_prop_buf(&buf, ifd); 4047 if (error != 0) 4048 return error; 4049 error = EINVAL; 4050 prop_dict = prop_dictionary_internalize(buf); 4051 if (prop_dict == NULL) 4052 goto out; 4053 4054 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4055 if (error != 0) 4056 goto out; 4057 4058 mutex_enter(wg->wg_lock); 4059 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4060 wg->wg_npeers++; 4061 mutex_exit(wg->wg_lock); 4062 4063 out: 4064 kmem_free(buf, ifd->ifd_len + 1); 4065 return error; 4066 } 4067 4068 static int 4069 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4070 { 4071 int error; 4072 prop_dictionary_t prop_dict; 4073 char *buf = NULL; 4074 const char *name; 4075 4076 error = wg_alloc_prop_buf(&buf, ifd); 4077 if (error != 0) 4078 return error; 4079 error = EINVAL; 4080 prop_dict = prop_dictionary_internalize(buf); 4081 if (prop_dict == NULL) 4082 goto out; 4083 4084 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4085 goto out; 4086 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4087 goto out; 4088 4089 error = wg_destroy_peer_name(wg, name); 4090 out: 4091 kmem_free(buf, ifd->ifd_len + 1); 4092 return error; 4093 } 4094 4095 static int 4096 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4097 { 4098 int error = ENOMEM; 4099 prop_dictionary_t prop_dict; 4100 prop_array_t peers = NULL; 4101 char *buf; 4102 struct wg_peer *wgp; 4103 int s, i; 4104 4105 prop_dict = prop_dictionary_create(); 4106 if (prop_dict == NULL) 4107 goto error; 4108 4109 if (!prop_dictionary_set_data(prop_dict, "private_key", wg->wg_privkey, 4110 WG_STATIC_KEY_LEN)) 4111 goto error; 4112 4113 if (wg->wg_listen_port != 0) { 4114 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4115 wg->wg_listen_port)) 4116 goto error; 4117 } 4118 4119 if (wg->wg_npeers == 0) 4120 goto skip_peers; 4121 4122 peers = prop_array_create(); 4123 if (peers == NULL) 4124 goto error; 4125 4126 s = pserialize_read_enter(); 4127 i = 0; 4128 WG_PEER_READER_FOREACH(wgp, wg) { 4129 struct psref psref; 4130 prop_dictionary_t prop_peer; 4131 4132 wg_get_peer(wgp, &psref); 4133 pserialize_read_exit(s); 4134 4135 prop_peer = prop_dictionary_create(); 4136 if (prop_peer == NULL) 4137 goto next; 4138 4139 if (strlen(wgp->wgp_name) > 0) { 4140 if (!prop_dictionary_set_string(prop_peer, "name", 4141 wgp->wgp_name)) 4142 goto next; 4143 } 4144 4145 if (!prop_dictionary_set_data(prop_peer, "public_key", 4146 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4147 goto next; 4148 4149 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4150 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4151 sizeof(wgp->wgp_psk))) { 4152 if (!prop_dictionary_set_data(prop_peer, 4153 "preshared_key", 4154 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4155 goto next; 4156 } 4157 4158 switch (wgp->wgp_sa.sa_family) { 4159 case AF_INET: 4160 if (!prop_dictionary_set_data(prop_peer, "endpoint", 4161 &wgp->wgp_sin, sizeof(wgp->wgp_sin))) 4162 goto next; 4163 break; 4164 #ifdef INET6 4165 case AF_INET6: 4166 if (!prop_dictionary_set_data(prop_peer, "endpoint", 4167 &wgp->wgp_sin6, sizeof(wgp->wgp_sin6))) 4168 goto next; 4169 break; 4170 #endif 4171 } 4172 4173 const struct timespec *t = &wgp->wgp_last_handshake_time; 4174 4175 if (!prop_dictionary_set_uint64(prop_peer, 4176 "last_handshake_time_sec", t->tv_sec)) 4177 goto next; 4178 if (!prop_dictionary_set_uint32(prop_peer, 4179 "last_handshake_time_nsec", t->tv_nsec)) 4180 goto next; 4181 4182 if (wgp->wgp_n_allowedips == 0) 4183 goto skip_allowedips; 4184 4185 prop_array_t allowedips = prop_array_create(); 4186 if (allowedips == NULL) 4187 goto next; 4188 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4189 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4190 prop_dictionary_t prop_allowedip; 4191 4192 prop_allowedip = prop_dictionary_create(); 4193 if (prop_allowedip == NULL) 4194 break; 4195 4196 if (!prop_dictionary_set_int(prop_allowedip, "family", 4197 wga->wga_family)) 4198 goto _next; 4199 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4200 wga->wga_cidr)) 4201 goto _next; 4202 4203 switch (wga->wga_family) { 4204 case AF_INET: 4205 if (!prop_dictionary_set_data(prop_allowedip, 4206 "ip", &wga->wga_addr4, 4207 sizeof(wga->wga_addr4))) 4208 goto _next; 4209 break; 4210 #ifdef INET6 4211 case AF_INET6: 4212 if (!prop_dictionary_set_data(prop_allowedip, 4213 "ip", &wga->wga_addr6, 4214 sizeof(wga->wga_addr6))) 4215 goto _next; 4216 break; 4217 #endif 4218 default: 4219 break; 4220 } 4221 prop_array_set(allowedips, j, prop_allowedip); 4222 _next: 4223 prop_object_release(prop_allowedip); 4224 } 4225 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4226 prop_object_release(allowedips); 4227 4228 skip_allowedips: 4229 4230 prop_array_set(peers, i, prop_peer); 4231 next: 4232 if (prop_peer) 4233 prop_object_release(prop_peer); 4234 i++; 4235 4236 s = pserialize_read_enter(); 4237 wg_put_peer(wgp, &psref); 4238 } 4239 pserialize_read_exit(s); 4240 4241 prop_dictionary_set(prop_dict, "peers", peers); 4242 prop_object_release(peers); 4243 peers = NULL; 4244 4245 skip_peers: 4246 buf = prop_dictionary_externalize(prop_dict); 4247 if (buf == NULL) 4248 goto error; 4249 if (ifd->ifd_len < (strlen(buf) + 1)) { 4250 error = EINVAL; 4251 goto error; 4252 } 4253 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4254 4255 free(buf, 0); 4256 error: 4257 if (peers != NULL) 4258 prop_object_release(peers); 4259 if (prop_dict != NULL) 4260 prop_object_release(prop_dict); 4261 4262 return error; 4263 } 4264 4265 static int 4266 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4267 { 4268 struct wg_softc *wg = ifp->if_softc; 4269 struct ifreq *ifr = data; 4270 struct ifaddr *ifa = data; 4271 struct ifdrv *ifd = data; 4272 int error = 0; 4273 4274 switch (cmd) { 4275 case SIOCINITIFADDR: 4276 if (ifa->ifa_addr->sa_family != AF_LINK && 4277 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4278 (IFF_UP | IFF_RUNNING)) { 4279 ifp->if_flags |= IFF_UP; 4280 error = ifp->if_init(ifp); 4281 } 4282 return error; 4283 case SIOCADDMULTI: 4284 case SIOCDELMULTI: 4285 switch (ifr->ifr_addr.sa_family) { 4286 case AF_INET: /* IP supports Multicast */ 4287 break; 4288 #ifdef INET6 4289 case AF_INET6: /* IP6 supports Multicast */ 4290 break; 4291 #endif 4292 default: /* Other protocols doesn't support Multicast */ 4293 error = EAFNOSUPPORT; 4294 break; 4295 } 4296 return error; 4297 case SIOCSDRVSPEC: 4298 switch (ifd->ifd_cmd) { 4299 case WG_IOCTL_SET_PRIVATE_KEY: 4300 error = wg_ioctl_set_private_key(wg, ifd); 4301 break; 4302 case WG_IOCTL_SET_LISTEN_PORT: 4303 error = wg_ioctl_set_listen_port(wg, ifd); 4304 break; 4305 case WG_IOCTL_ADD_PEER: 4306 error = wg_ioctl_add_peer(wg, ifd); 4307 break; 4308 case WG_IOCTL_DELETE_PEER: 4309 error = wg_ioctl_delete_peer(wg, ifd); 4310 break; 4311 default: 4312 error = EINVAL; 4313 break; 4314 } 4315 return error; 4316 case SIOCGDRVSPEC: 4317 return wg_ioctl_get(wg, ifd); 4318 case SIOCSIFFLAGS: 4319 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4320 break; 4321 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4322 case IFF_RUNNING: 4323 /* 4324 * If interface is marked down and it is running, 4325 * then stop and disable it. 4326 */ 4327 (*ifp->if_stop)(ifp, 1); 4328 break; 4329 case IFF_UP: 4330 /* 4331 * If interface is marked up and it is stopped, then 4332 * start it. 4333 */ 4334 error = (*ifp->if_init)(ifp); 4335 break; 4336 default: 4337 break; 4338 } 4339 return error; 4340 #ifdef WG_RUMPKERNEL 4341 case SIOCSLINKSTR: 4342 error = wg_ioctl_linkstr(wg, ifd); 4343 if (error == 0) 4344 wg->wg_ops = &wg_ops_rumpuser; 4345 return error; 4346 #endif 4347 default: 4348 break; 4349 } 4350 4351 error = ifioctl_common(ifp, cmd, data); 4352 4353 #ifdef WG_RUMPKERNEL 4354 if (!wg_user_mode(wg)) 4355 return error; 4356 4357 /* Do the same to the corresponding tun device on the host */ 4358 /* 4359 * XXX Actually the command has not been handled yet. It 4360 * will be handled via pr_ioctl form doifioctl later. 4361 */ 4362 switch (cmd) { 4363 case SIOCAIFADDR: 4364 case SIOCDIFADDR: { 4365 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4366 struct in_aliasreq *ifra = &_ifra; 4367 KASSERT(error == ENOTTY); 4368 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4369 IFNAMSIZ); 4370 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4371 if (error == 0) 4372 error = ENOTTY; 4373 break; 4374 } 4375 #ifdef INET6 4376 case SIOCAIFADDR_IN6: 4377 case SIOCDIFADDR_IN6: { 4378 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4379 struct in6_aliasreq *ifra = &_ifra; 4380 KASSERT(error == ENOTTY); 4381 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4382 IFNAMSIZ); 4383 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4384 if (error == 0) 4385 error = ENOTTY; 4386 break; 4387 } 4388 #endif 4389 } 4390 #endif /* WG_RUMPKERNEL */ 4391 4392 return error; 4393 } 4394 4395 static int 4396 wg_init(struct ifnet *ifp) 4397 { 4398 4399 ifp->if_flags |= IFF_RUNNING; 4400 4401 /* TODO flush pending packets. */ 4402 return 0; 4403 } 4404 4405 static void 4406 wg_stop(struct ifnet *ifp, int disable) 4407 { 4408 4409 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4410 ifp->if_flags &= ~IFF_RUNNING; 4411 4412 /* Need to do something? */ 4413 } 4414 4415 #ifdef WG_DEBUG_PARAMS 4416 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4417 { 4418 const struct sysctlnode *node = NULL; 4419 4420 sysctl_createv(clog, 0, NULL, &node, 4421 CTLFLAG_PERMANENT, 4422 CTLTYPE_NODE, "wg", 4423 SYSCTL_DESCR("wg(4)"), 4424 NULL, 0, NULL, 0, 4425 CTL_NET, CTL_CREATE, CTL_EOL); 4426 sysctl_createv(clog, 0, &node, NULL, 4427 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4428 CTLTYPE_QUAD, "rekey_after_messages", 4429 SYSCTL_DESCR("session liftime by messages"), 4430 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4431 sysctl_createv(clog, 0, &node, NULL, 4432 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4433 CTLTYPE_INT, "rekey_after_time", 4434 SYSCTL_DESCR("session liftime"), 4435 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4436 sysctl_createv(clog, 0, &node, NULL, 4437 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4438 CTLTYPE_INT, "rekey_timeout", 4439 SYSCTL_DESCR("session handshake retry time"), 4440 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4441 sysctl_createv(clog, 0, &node, NULL, 4442 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4443 CTLTYPE_INT, "rekey_attempt_time", 4444 SYSCTL_DESCR("session handshake timeout"), 4445 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4446 sysctl_createv(clog, 0, &node, NULL, 4447 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4448 CTLTYPE_INT, "keepalive_timeout", 4449 SYSCTL_DESCR("keepalive timeout"), 4450 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4451 sysctl_createv(clog, 0, &node, NULL, 4452 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4453 CTLTYPE_BOOL, "force_underload", 4454 SYSCTL_DESCR("force to detemine under load"), 4455 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4456 } 4457 #endif 4458 4459 #ifdef WG_RUMPKERNEL 4460 static bool 4461 wg_user_mode(struct wg_softc *wg) 4462 { 4463 4464 return wg->wg_user != NULL; 4465 } 4466 4467 static int 4468 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4469 { 4470 struct ifnet *ifp = &wg->wg_if; 4471 int error; 4472 4473 if (ifp->if_flags & IFF_UP) 4474 return EBUSY; 4475 4476 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4477 /* XXX do nothing */ 4478 return 0; 4479 } else if (ifd->ifd_cmd != 0) { 4480 return EINVAL; 4481 } else if (wg->wg_user != NULL) { 4482 return EBUSY; 4483 } 4484 4485 /* Assume \0 included */ 4486 if (ifd->ifd_len > IFNAMSIZ) { 4487 return E2BIG; 4488 } else if (ifd->ifd_len < 1) { 4489 return EINVAL; 4490 } 4491 4492 char tun_name[IFNAMSIZ]; 4493 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4494 if (error != 0) 4495 return error; 4496 4497 if (strncmp(tun_name, "tun", 3) != 0) 4498 return EINVAL; 4499 4500 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 4501 4502 return error; 4503 } 4504 4505 static int 4506 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 4507 { 4508 int error; 4509 struct psref psref; 4510 struct wg_sockaddr *wgsa; 4511 struct wg_softc *wg = wgp->wgp_sc; 4512 struct iovec iov[1]; 4513 4514 wgsa = wg_get_endpoint_sa(wgp, &psref); 4515 4516 iov[0].iov_base = mtod(m, void *); 4517 iov[0].iov_len = m->m_len; 4518 4519 /* Send messages to a peer via an ordinary socket. */ 4520 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 4521 4522 wg_put_sa(wgp, wgsa, &psref); 4523 4524 return error; 4525 } 4526 4527 static void 4528 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 4529 { 4530 struct wg_softc *wg = ifp->if_softc; 4531 struct iovec iov[2]; 4532 struct sockaddr_storage ss; 4533 4534 KASSERT(af == AF_INET || af == AF_INET6); 4535 4536 WG_TRACE(""); 4537 4538 if (af == AF_INET) { 4539 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 4540 struct ip *ip; 4541 ip = mtod(m, struct ip *); 4542 sockaddr_in_init(sin, &ip->ip_dst, 0); 4543 } else { 4544 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 4545 struct ip6_hdr *ip6; 4546 ip6 = mtod(m, struct ip6_hdr *); 4547 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 4548 } 4549 4550 iov[0].iov_base = &ss; 4551 iov[0].iov_len = ss.ss_len; 4552 iov[1].iov_base = mtod(m, void *); 4553 iov[1].iov_len = m->m_len; 4554 4555 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4556 4557 /* Send decrypted packets to users via a tun. */ 4558 rumpuser_wg_send_user(wg->wg_user, iov, 2); 4559 } 4560 4561 static int 4562 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 4563 { 4564 int error; 4565 uint16_t old_port = wg->wg_listen_port; 4566 4567 if (port != 0 && old_port == port) 4568 return 0; 4569 4570 error = rumpuser_wg_sock_bind(wg->wg_user, port); 4571 if (error == 0) 4572 wg->wg_listen_port = port; 4573 return error; 4574 } 4575 4576 /* 4577 * Receive user packets. 4578 */ 4579 void 4580 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4581 { 4582 struct ifnet *ifp = &wg->wg_if; 4583 struct mbuf *m; 4584 const struct sockaddr *dst; 4585 4586 WG_TRACE(""); 4587 4588 dst = iov[0].iov_base; 4589 4590 m = m_gethdr(M_NOWAIT, MT_DATA); 4591 if (m == NULL) 4592 return; 4593 m->m_len = m->m_pkthdr.len = 0; 4594 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4595 4596 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4597 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4598 4599 (void)wg_output(ifp, m, dst, NULL); 4600 } 4601 4602 /* 4603 * Receive packets from a peer. 4604 */ 4605 void 4606 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4607 { 4608 struct mbuf *m; 4609 const struct sockaddr *src; 4610 4611 WG_TRACE(""); 4612 4613 src = iov[0].iov_base; 4614 4615 m = m_gethdr(M_NOWAIT, MT_DATA); 4616 if (m == NULL) 4617 return; 4618 m->m_len = m->m_pkthdr.len = 0; 4619 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4620 4621 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4622 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4623 4624 wg_handle_packet(wg, m, src); 4625 } 4626 #endif /* WG_RUMPKERNEL */ 4627 4628 /* 4629 * Module infrastructure 4630 */ 4631 #include "if_module.h" 4632 4633 IF_MODULE(MODULE_CLASS_DRIVER, wg, "") 4634
Indexes created Wed Sep 24 22:09:59 GMT 2025