if_wg.c revision 1.115 1 1.115 riastrad /* $NetBSD: if_wg.c,v 1.115 2024/07/29 02:33:27 riastradh Exp $ */
2 1.1 riastrad
3 1.1 riastrad /*
4 1.1 riastrad * Copyright (C) Ryota Ozaki <ozaki.ryota (at) gmail.com>
5 1.1 riastrad * All rights reserved.
6 1.1 riastrad *
7 1.1 riastrad * Redistribution and use in source and binary forms, with or without
8 1.1 riastrad * modification, are permitted provided that the following conditions
9 1.1 riastrad * are met:
10 1.1 riastrad * 1. Redistributions of source code must retain the above copyright
11 1.1 riastrad * notice, this list of conditions and the following disclaimer.
12 1.1 riastrad * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 riastrad * notice, this list of conditions and the following disclaimer in the
14 1.1 riastrad * documentation and/or other materials provided with the distribution.
15 1.1 riastrad * 3. Neither the name of the project nor the names of its contributors
16 1.1 riastrad * may be used to endorse or promote products derived from this software
17 1.1 riastrad * without specific prior written permission.
18 1.1 riastrad *
19 1.1 riastrad * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
20 1.1 riastrad * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 1.1 riastrad * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 1.1 riastrad * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
23 1.1 riastrad * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 1.1 riastrad * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 1.1 riastrad * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 1.1 riastrad * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 1.1 riastrad * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 1.1 riastrad * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 1.1 riastrad * SUCH DAMAGE.
30 1.1 riastrad */
31 1.1 riastrad
32 1.1 riastrad /*
33 1.24 riastrad * This network interface aims to implement the WireGuard protocol.
34 1.24 riastrad * The implementation is based on the paper of WireGuard as of
35 1.24 riastrad * 2018-06-30 [1]. The paper is referred in the source code with label
36 1.24 riastrad * [W]. Also the specification of the Noise protocol framework as of
37 1.24 riastrad * 2018-07-11 [2] is referred with label [N].
38 1.1 riastrad *
39 1.1 riastrad * [1] https://www.wireguard.com/papers/wireguard.pdf
40 1.1 riastrad * [2] http://noiseprotocol.org/noise.pdf
41 1.1 riastrad */
42 1.1 riastrad
43 1.1 riastrad #include <sys/cdefs.h>
44 1.115 riastrad __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.115 2024/07/29 02:33:27 riastradh Exp $");
45 1.1 riastrad
46 1.1 riastrad #ifdef _KERNEL_OPT
47 1.60 riastrad #include "opt_altq_enabled.h"
48 1.1 riastrad #include "opt_inet.h"
49 1.1 riastrad #endif
50 1.1 riastrad
51 1.1 riastrad #include <sys/param.h>
52 1.32 riastrad #include <sys/types.h>
53 1.32 riastrad
54 1.32 riastrad #include <sys/atomic.h>
55 1.32 riastrad #include <sys/callout.h>
56 1.32 riastrad #include <sys/cprng.h>
57 1.32 riastrad #include <sys/cpu.h>
58 1.32 riastrad #include <sys/device.h>
59 1.32 riastrad #include <sys/domain.h>
60 1.1 riastrad #include <sys/errno.h>
61 1.32 riastrad #include <sys/intr.h>
62 1.1 riastrad #include <sys/ioctl.h>
63 1.32 riastrad #include <sys/kernel.h>
64 1.1 riastrad #include <sys/kmem.h>
65 1.32 riastrad #include <sys/mbuf.h>
66 1.1 riastrad #include <sys/module.h>
67 1.1 riastrad #include <sys/mutex.h>
68 1.58 riastrad #include <sys/once.h>
69 1.32 riastrad #include <sys/percpu.h>
70 1.1 riastrad #include <sys/pserialize.h>
71 1.1 riastrad #include <sys/psref.h>
72 1.32 riastrad #include <sys/queue.h>
73 1.32 riastrad #include <sys/rwlock.h>
74 1.32 riastrad #include <sys/socket.h>
75 1.32 riastrad #include <sys/socketvar.h>
76 1.32 riastrad #include <sys/sockio.h>
77 1.1 riastrad #include <sys/sysctl.h>
78 1.32 riastrad #include <sys/syslog.h>
79 1.32 riastrad #include <sys/systm.h>
80 1.37 riastrad #include <sys/thmap.h>
81 1.55 riastrad #include <sys/threadpool.h>
82 1.32 riastrad #include <sys/time.h>
83 1.32 riastrad #include <sys/timespec.h>
84 1.55 riastrad #include <sys/workqueue.h>
85 1.1 riastrad
86 1.86 christos #include <lib/libkern/libkern.h>
87 1.86 christos
88 1.1 riastrad #include <net/bpf.h>
89 1.1 riastrad #include <net/if.h>
90 1.1 riastrad #include <net/if_types.h>
91 1.32 riastrad #include <net/if_wg.h>
92 1.54 riastrad #include <net/pktqueue.h>
93 1.1 riastrad #include <net/route.h>
94 1.1 riastrad
95 1.109 riastrad #ifdef INET
96 1.1 riastrad #include <netinet/in.h>
97 1.32 riastrad #include <netinet/in_pcb.h>
98 1.32 riastrad #include <netinet/in_var.h>
99 1.1 riastrad #include <netinet/ip.h>
100 1.1 riastrad #include <netinet/ip_var.h>
101 1.1 riastrad #include <netinet/udp.h>
102 1.1 riastrad #include <netinet/udp_var.h>
103 1.109 riastrad #endif /* INET */
104 1.1 riastrad
105 1.1 riastrad #ifdef INET6
106 1.32 riastrad #include <netinet/ip6.h>
107 1.32 riastrad #include <netinet6/in6_pcb.h>
108 1.1 riastrad #include <netinet6/in6_var.h>
109 1.1 riastrad #include <netinet6/ip6_var.h>
110 1.1 riastrad #include <netinet6/udp6_var.h>
111 1.109 riastrad #endif /* INET6 */
112 1.1 riastrad
113 1.1 riastrad #include <prop/proplib.h>
114 1.1 riastrad
115 1.1 riastrad #include <crypto/blake2/blake2s.h>
116 1.1 riastrad #include <crypto/sodium/crypto_aead_chacha20poly1305.h>
117 1.1 riastrad #include <crypto/sodium/crypto_aead_xchacha20poly1305.h>
118 1.32 riastrad #include <crypto/sodium/crypto_scalarmult.h>
119 1.1 riastrad
120 1.1 riastrad #include "ioconf.h"
121 1.1 riastrad
122 1.1 riastrad #ifdef WG_RUMPKERNEL
123 1.1 riastrad #include "wg_user.h"
124 1.1 riastrad #endif
125 1.1 riastrad
126 1.105 riastrad #ifndef time_uptime32
127 1.105 riastrad #define time_uptime32 ((uint32_t)time_uptime)
128 1.105 riastrad #endif
129 1.105 riastrad
130 1.1 riastrad /*
131 1.1 riastrad * Data structures
132 1.1 riastrad * - struct wg_softc is an instance of wg interfaces
133 1.1 riastrad * - It has a list of peers (struct wg_peer)
134 1.55 riastrad * - It has a threadpool job that sends/receives handshake messages and
135 1.1 riastrad * runs event handlers
136 1.1 riastrad * - It has its own two routing tables: one is for IPv4 and the other IPv6
137 1.1 riastrad * - struct wg_peer is a representative of a peer
138 1.55 riastrad * - It has a struct work to handle handshakes and timer tasks
139 1.1 riastrad * - It has a pair of session instances (struct wg_session)
140 1.1 riastrad * - It has a pair of endpoint instances (struct wg_sockaddr)
141 1.1 riastrad * - Normally one endpoint is used and the second one is used only on
142 1.1 riastrad * a peer migration (a change of peer's IP address)
143 1.1 riastrad * - It has a list of IP addresses and sub networks called allowedips
144 1.1 riastrad * (struct wg_allowedip)
145 1.1 riastrad * - A packets sent over a session is allowed if its destination matches
146 1.1 riastrad * any IP addresses or sub networks of the list
147 1.1 riastrad * - struct wg_session represents a session of a secure tunnel with a peer
148 1.1 riastrad * - Two instances of sessions belong to a peer; a stable session and a
149 1.1 riastrad * unstable session
150 1.49 riastrad * - A handshake process of a session always starts with a unstable instance
151 1.1 riastrad * - Once a session is established, its instance becomes stable and the
152 1.1 riastrad * other becomes unstable instead
153 1.1 riastrad * - Data messages are always sent via a stable session
154 1.1 riastrad *
155 1.1 riastrad * Locking notes:
156 1.49 riastrad * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock
157 1.49 riastrad * - Changes to the peer list are serialized by wg_lock
158 1.49 riastrad * - The peer list may be read with pserialize(9) and psref(9)
159 1.1 riastrad * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46])
160 1.49 riastrad * => XXX replace by pserialize when routing table is psz-safe
161 1.49 riastrad * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken
162 1.49 riastrad * only in thread context and serializes:
163 1.49 riastrad * - the stable and unstable session pointers
164 1.49 riastrad * - all unstable session state
165 1.49 riastrad * - Packet processing may be done in softint context:
166 1.49 riastrad * - The stable session can be read under pserialize(9) or psref(9)
167 1.49 riastrad * - The stable session is always ESTABLISHED
168 1.14 riastrad * - On a session swap, we must wait for all readers to release a
169 1.14 riastrad * reference to a stable session before changing wgs_state and
170 1.14 riastrad * session states
171 1.49 riastrad * - Lock order: wg_lock -> wgp_lock
172 1.1 riastrad */
173 1.1 riastrad
174 1.1 riastrad
175 1.14 riastrad #define WGLOG(level, fmt, args...) \
176 1.14 riastrad log(level, "%s: " fmt, __func__, ##args)
177 1.1 riastrad
178 1.85 christos #define WG_DEBUG
179 1.80 christos
180 1.1 riastrad /* Debug options */
181 1.1 riastrad #ifdef WG_DEBUG
182 1.1 riastrad /* Output debug logs */
183 1.1 riastrad #ifndef WG_DEBUG_LOG
184 1.1 riastrad #define WG_DEBUG_LOG
185 1.1 riastrad #endif
186 1.1 riastrad /* Output trace logs */
187 1.1 riastrad #ifndef WG_DEBUG_TRACE
188 1.1 riastrad #define WG_DEBUG_TRACE
189 1.1 riastrad #endif
190 1.1 riastrad /* Output hash values, etc. */
191 1.1 riastrad #ifndef WG_DEBUG_DUMP
192 1.1 riastrad #define WG_DEBUG_DUMP
193 1.1 riastrad #endif
194 1.1 riastrad /* Make some internal parameters configurable for testing and debugging */
195 1.1 riastrad #ifndef WG_DEBUG_PARAMS
196 1.1 riastrad #define WG_DEBUG_PARAMS
197 1.1 riastrad #endif
198 1.83 kre #endif /* WG_DEBUG */
199 1.83 kre
200 1.83 kre #ifndef WG_DEBUG
201 1.83 kre # if defined(WG_DEBUG_LOG) || defined(WG_DEBUG_TRACE) || \
202 1.111 riastrad defined(WG_DEBUG_DUMP) || defined(WG_DEBUG_PARAMS)
203 1.83 kre # define WG_DEBUG
204 1.83 kre # endif
205 1.83 kre #endif
206 1.83 kre
207 1.83 kre #ifdef WG_DEBUG
208 1.80 christos int wg_debug;
209 1.80 christos #define WG_DEBUG_FLAGS_LOG 1
210 1.80 christos #define WG_DEBUG_FLAGS_TRACE 2
211 1.80 christos #define WG_DEBUG_FLAGS_DUMP 4
212 1.1 riastrad #endif
213 1.1 riastrad
214 1.1 riastrad #ifdef WG_DEBUG_TRACE
215 1.80 christos #define WG_TRACE(msg) do { \
216 1.80 christos if (wg_debug & WG_DEBUG_FLAGS_TRACE) \
217 1.80 christos log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)); \
218 1.80 christos } while (0)
219 1.1 riastrad #else
220 1.1 riastrad #define WG_TRACE(msg) __nothing
221 1.1 riastrad #endif
222 1.1 riastrad
223 1.1 riastrad #ifdef WG_DEBUG_LOG
224 1.80 christos #define WG_DLOG(fmt, args...) do { \
225 1.80 christos if (wg_debug & WG_DEBUG_FLAGS_LOG) \
226 1.80 christos log(LOG_DEBUG, "%s: " fmt, __func__, ##args); \
227 1.80 christos } while (0)
228 1.1 riastrad #else
229 1.1 riastrad #define WG_DLOG(fmt, args...) __nothing
230 1.1 riastrad #endif
231 1.1 riastrad
232 1.1 riastrad #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \
233 1.81 christos if (ppsratecheck(&(wgprc)->wgprc_lasttime, \
234 1.1 riastrad &(wgprc)->wgprc_curpps, 1)) { \
235 1.1 riastrad log(level, fmt, ##args); \
236 1.1 riastrad } \
237 1.1 riastrad } while (0)
238 1.1 riastrad
239 1.1 riastrad #ifdef WG_DEBUG_PARAMS
240 1.1 riastrad static bool wg_force_underload = false;
241 1.1 riastrad #endif
242 1.1 riastrad
243 1.1 riastrad #ifdef WG_DEBUG_DUMP
244 1.7 riastrad
245 1.89 kre static char enomem[10] = "[enomem]";
246 1.89 kre
247 1.93 christos #define MAX_HDUMP_LEN 10000 /* large enough */
248 1.93 christos
249 1.112 riastrad /*
250 1.112 riastrad * gethexdump(p, n)
251 1.112 riastrad *
252 1.112 riastrad * Allocate a string returning a hexdump of bytes p[0..n),
253 1.112 riastrad * truncated to MAX_HDUMP_LEN. Must be freed with puthexdump.
254 1.112 riastrad *
255 1.112 riastrad * We use this instead of libkern hexdump() because the result is
256 1.112 riastrad * logged with log(LOG_DEBUG, ...), which puts a priority tag on
257 1.112 riastrad * every message, so it can't be done incrementally.
258 1.112 riastrad */
259 1.53 riastrad static char *
260 1.84 christos gethexdump(const void *vp, size_t n)
261 1.53 riastrad {
262 1.53 riastrad char *buf;
263 1.84 christos const uint8_t *p = vp;
264 1.93 christos size_t i, alloc;
265 1.53 riastrad
266 1.93 christos alloc = n;
267 1.93 christos if (n > MAX_HDUMP_LEN)
268 1.93 christos alloc = MAX_HDUMP_LEN;
269 1.112 riastrad buf = kmem_alloc(3*alloc + 5, KM_NOSLEEP);
270 1.53 riastrad if (buf == NULL)
271 1.89 kre return enomem;
272 1.93 christos for (i = 0; i < alloc; i++)
273 1.112 riastrad snprintf(buf + 3*i, 3 + 1, " %02hhx", p[i]);
274 1.93 christos if (alloc != n)
275 1.112 riastrad snprintf(buf + 3*i, 4 + 1, " ...");
276 1.53 riastrad return buf;
277 1.53 riastrad }
278 1.53 riastrad
279 1.53 riastrad static void
280 1.53 riastrad puthexdump(char *buf, const void *p, size_t n)
281 1.53 riastrad {
282 1.53 riastrad
283 1.89 kre if (buf == NULL || buf == enomem)
284 1.53 riastrad return;
285 1.93 christos if (n > MAX_HDUMP_LEN)
286 1.93 christos n = MAX_HDUMP_LEN;
287 1.112 riastrad kmem_free(buf, 3*n + 5);
288 1.53 riastrad }
289 1.53 riastrad
290 1.7 riastrad #ifdef WG_RUMPKERNEL
291 1.1 riastrad static void
292 1.1 riastrad wg_dump_buf(const char *func, const char *buf, const size_t size)
293 1.1 riastrad {
294 1.80 christos if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0)
295 1.80 christos return;
296 1.80 christos
297 1.53 riastrad char *hex = gethexdump(buf, size);
298 1.1 riastrad
299 1.89 kre log(LOG_DEBUG, "%s: %s\n", func, hex);
300 1.53 riastrad puthexdump(hex, buf, size);
301 1.1 riastrad }
302 1.7 riastrad #endif
303 1.1 riastrad
304 1.1 riastrad static void
305 1.1 riastrad wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash,
306 1.1 riastrad const size_t size)
307 1.1 riastrad {
308 1.80 christos if ((wg_debug & WG_DEBUG_FLAGS_DUMP) == 0)
309 1.80 christos return;
310 1.80 christos
311 1.53 riastrad char *hex = gethexdump(hash, size);
312 1.1 riastrad
313 1.89 kre log(LOG_DEBUG, "%s: %s: %s\n", func, name, hex);
314 1.53 riastrad puthexdump(hex, hash, size);
315 1.1 riastrad }
316 1.1 riastrad
317 1.1 riastrad #define WG_DUMP_HASH(name, hash) \
318 1.1 riastrad wg_dump_hash(__func__, name, hash, WG_HASH_LEN)
319 1.1 riastrad #define WG_DUMP_HASH48(name, hash) \
320 1.1 riastrad wg_dump_hash(__func__, name, hash, 48)
321 1.1 riastrad #define WG_DUMP_BUF(buf, size) \
322 1.1 riastrad wg_dump_buf(__func__, buf, size)
323 1.1 riastrad #else
324 1.1 riastrad #define WG_DUMP_HASH(name, hash) __nothing
325 1.1 riastrad #define WG_DUMP_HASH48(name, hash) __nothing
326 1.1 riastrad #define WG_DUMP_BUF(buf, size) __nothing
327 1.1 riastrad #endif /* WG_DEBUG_DUMP */
328 1.1 riastrad
329 1.75 andvar /* chosen somewhat arbitrarily -- fits in signed 16 bits NUL-terminated */
330 1.68 riastrad #define WG_MAX_PROPLEN 32766
331 1.68 riastrad
332 1.1 riastrad #define WG_MTU 1420
333 1.1 riastrad #define WG_ALLOWEDIPS 16
334 1.1 riastrad
335 1.1 riastrad #define CURVE25519_KEY_LEN 32
336 1.110 riastrad #define TAI64N_LEN (sizeof(uint32_t) * 3)
337 1.1 riastrad #define POLY1305_AUTHTAG_LEN 16
338 1.1 riastrad #define HMAC_BLOCK_LEN 64
339 1.1 riastrad
340 1.1 riastrad /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */
341 1.1 riastrad /* [N] 4.3: Hash functions */
342 1.1 riastrad #define NOISE_DHLEN 32
343 1.1 riastrad /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */
344 1.1 riastrad #define NOISE_HASHLEN 32
345 1.1 riastrad #define NOISE_BLOCKLEN 64
346 1.1 riastrad #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN
347 1.1 riastrad /* [N] 5.1: "k" */
348 1.1 riastrad #define NOISE_CIPHER_KEY_LEN 32
349 1.1 riastrad /*
350 1.1 riastrad * [N] 9.2: "psk"
351 1.1 riastrad * "... psk is a 32-byte secret value provided by the application."
352 1.1 riastrad */
353 1.1 riastrad #define NOISE_PRESHARED_KEY_LEN 32
354 1.1 riastrad
355 1.1 riastrad #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN
356 1.1 riastrad #define WG_TIMESTAMP_LEN TAI64N_LEN
357 1.1 riastrad
358 1.1 riastrad #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN
359 1.1 riastrad
360 1.1 riastrad #define WG_COOKIE_LEN 16
361 1.1 riastrad #define WG_MAC_LEN 16
362 1.98 riastrad #define WG_COOKIESECRET_LEN 32
363 1.1 riastrad
364 1.1 riastrad #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN
365 1.1 riastrad /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */
366 1.1 riastrad #define WG_CHAINING_KEY_LEN NOISE_HASHLEN
367 1.1 riastrad /* [N] 5.2: "h: A hash output of HASHLEN bytes" */
368 1.1 riastrad #define WG_HASH_LEN NOISE_HASHLEN
369 1.1 riastrad #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN
370 1.1 riastrad #define WG_DH_OUTPUT_LEN NOISE_DHLEN
371 1.1 riastrad #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN
372 1.1 riastrad #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN
373 1.1 riastrad #define WG_DATA_KEY_LEN 32
374 1.1 riastrad #define WG_SALT_LEN 24
375 1.1 riastrad
376 1.1 riastrad /*
377 1.1 riastrad * The protocol messages
378 1.1 riastrad */
379 1.14 riastrad struct wg_msg {
380 1.1 riastrad uint32_t wgm_type;
381 1.1 riastrad } __packed;
382 1.1 riastrad
383 1.1 riastrad /* [W] 5.4.2 First Message: Initiator to Responder */
384 1.1 riastrad struct wg_msg_init {
385 1.1 riastrad uint32_t wgmi_type;
386 1.1 riastrad uint32_t wgmi_sender;
387 1.1 riastrad uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN];
388 1.1 riastrad uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN];
389 1.1 riastrad uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN];
390 1.1 riastrad uint8_t wgmi_mac1[WG_MAC_LEN];
391 1.1 riastrad uint8_t wgmi_mac2[WG_MAC_LEN];
392 1.1 riastrad } __packed;
393 1.1 riastrad
394 1.1 riastrad /* [W] 5.4.3 Second Message: Responder to Initiator */
395 1.1 riastrad struct wg_msg_resp {
396 1.1 riastrad uint32_t wgmr_type;
397 1.1 riastrad uint32_t wgmr_sender;
398 1.1 riastrad uint32_t wgmr_receiver;
399 1.1 riastrad uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN];
400 1.1 riastrad uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN];
401 1.1 riastrad uint8_t wgmr_mac1[WG_MAC_LEN];
402 1.1 riastrad uint8_t wgmr_mac2[WG_MAC_LEN];
403 1.1 riastrad } __packed;
404 1.1 riastrad
405 1.1 riastrad /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */
406 1.1 riastrad struct wg_msg_data {
407 1.1 riastrad uint32_t wgmd_type;
408 1.1 riastrad uint32_t wgmd_receiver;
409 1.1 riastrad uint64_t wgmd_counter;
410 1.114 riastrad uint32_t wgmd_packet[];
411 1.1 riastrad } __packed;
412 1.1 riastrad
413 1.1 riastrad /* [W] 5.4.7 Under Load: Cookie Reply Message */
414 1.1 riastrad struct wg_msg_cookie {
415 1.1 riastrad uint32_t wgmc_type;
416 1.1 riastrad uint32_t wgmc_receiver;
417 1.1 riastrad uint8_t wgmc_salt[WG_SALT_LEN];
418 1.1 riastrad uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN];
419 1.1 riastrad } __packed;
420 1.1 riastrad
421 1.1 riastrad #define WG_MSG_TYPE_INIT 1
422 1.1 riastrad #define WG_MSG_TYPE_RESP 2
423 1.1 riastrad #define WG_MSG_TYPE_COOKIE 3
424 1.1 riastrad #define WG_MSG_TYPE_DATA 4
425 1.1 riastrad #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA
426 1.1 riastrad
427 1.6 riastrad /* Sliding windows */
428 1.6 riastrad
429 1.6 riastrad #define SLIWIN_BITS 2048u
430 1.6 riastrad #define SLIWIN_TYPE uint32_t
431 1.110 riastrad #define SLIWIN_BPW (NBBY*sizeof(SLIWIN_TYPE))
432 1.6 riastrad #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW)
433 1.6 riastrad #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE))
434 1.6 riastrad
435 1.6 riastrad struct sliwin {
436 1.6 riastrad SLIWIN_TYPE B[SLIWIN_WORDS];
437 1.6 riastrad uint64_t T;
438 1.6 riastrad };
439 1.6 riastrad
440 1.6 riastrad static void
441 1.6 riastrad sliwin_reset(struct sliwin *W)
442 1.6 riastrad {
443 1.6 riastrad
444 1.6 riastrad memset(W, 0, sizeof(*W));
445 1.6 riastrad }
446 1.6 riastrad
447 1.6 riastrad static int
448 1.6 riastrad sliwin_check_fast(const volatile struct sliwin *W, uint64_t S)
449 1.6 riastrad {
450 1.6 riastrad
451 1.6 riastrad /*
452 1.6 riastrad * If it's more than one window older than the highest sequence
453 1.6 riastrad * number we've seen, reject.
454 1.6 riastrad */
455 1.20 riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
456 1.6 riastrad if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T))
457 1.6 riastrad return EAUTH;
458 1.20 riastrad #endif
459 1.6 riastrad
460 1.6 riastrad /*
461 1.6 riastrad * Otherwise, we need to take the lock to decide, so don't
462 1.6 riastrad * reject just yet. Caller must serialize a call to
463 1.6 riastrad * sliwin_update in this case.
464 1.6 riastrad */
465 1.6 riastrad return 0;
466 1.6 riastrad }
467 1.6 riastrad
468 1.6 riastrad static int
469 1.6 riastrad sliwin_update(struct sliwin *W, uint64_t S)
470 1.6 riastrad {
471 1.6 riastrad unsigned word, bit;
472 1.6 riastrad
473 1.6 riastrad /*
474 1.6 riastrad * If it's more than one window older than the highest sequence
475 1.6 riastrad * number we've seen, reject.
476 1.6 riastrad */
477 1.6 riastrad if (S + SLIWIN_NPKT < W->T)
478 1.6 riastrad return EAUTH;
479 1.6 riastrad
480 1.6 riastrad /*
481 1.6 riastrad * If it's higher than the highest sequence number we've seen,
482 1.6 riastrad * advance the window.
483 1.6 riastrad */
484 1.6 riastrad if (S > W->T) {
485 1.6 riastrad uint64_t i = W->T / SLIWIN_BPW;
486 1.6 riastrad uint64_t j = S / SLIWIN_BPW;
487 1.6 riastrad unsigned k;
488 1.6 riastrad
489 1.6 riastrad for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++)
490 1.6 riastrad W->B[(i + k + 1) % SLIWIN_WORDS] = 0;
491 1.20 riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
492 1.6 riastrad atomic_store_relaxed(&W->T, S);
493 1.20 riastrad #else
494 1.20 riastrad W->T = S;
495 1.20 riastrad #endif
496 1.6 riastrad }
497 1.6 riastrad
498 1.6 riastrad /* Test and set the bit -- if already set, reject. */
499 1.6 riastrad word = (S / SLIWIN_BPW) % SLIWIN_WORDS;
500 1.6 riastrad bit = S % SLIWIN_BPW;
501 1.6 riastrad if (W->B[word] & (1UL << bit))
502 1.6 riastrad return EAUTH;
503 1.65 christos W->B[word] |= 1U << bit;
504 1.6 riastrad
505 1.6 riastrad /* Accept! */
506 1.6 riastrad return 0;
507 1.6 riastrad }
508 1.6 riastrad
509 1.1 riastrad struct wg_session {
510 1.1 riastrad struct wg_peer *wgs_peer;
511 1.1 riastrad struct psref_target
512 1.1 riastrad wgs_psref;
513 1.1 riastrad
514 1.1 riastrad int wgs_state;
515 1.1 riastrad #define WGS_STATE_UNKNOWN 0
516 1.1 riastrad #define WGS_STATE_INIT_ACTIVE 1
517 1.1 riastrad #define WGS_STATE_INIT_PASSIVE 2
518 1.1 riastrad #define WGS_STATE_ESTABLISHED 3
519 1.1 riastrad #define WGS_STATE_DESTROYING 4
520 1.1 riastrad
521 1.104 riastrad volatile uint32_t
522 1.104 riastrad wgs_time_established;
523 1.104 riastrad volatile uint32_t
524 1.104 riastrad wgs_time_last_data_sent;
525 1.113 riastrad volatile bool wgs_force_rekey;
526 1.1 riastrad bool wgs_is_initiator;
527 1.1 riastrad
528 1.49 riastrad uint32_t wgs_local_index;
529 1.49 riastrad uint32_t wgs_remote_index;
530 1.22 riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
531 1.1 riastrad volatile uint64_t
532 1.1 riastrad wgs_send_counter;
533 1.22 riastrad #else
534 1.22 riastrad kmutex_t wgs_send_counter_lock;
535 1.22 riastrad uint64_t wgs_send_counter;
536 1.22 riastrad #endif
537 1.6 riastrad
538 1.6 riastrad struct {
539 1.6 riastrad kmutex_t lock;
540 1.6 riastrad struct sliwin window;
541 1.6 riastrad } *wgs_recvwin;
542 1.1 riastrad
543 1.1 riastrad uint8_t wgs_handshake_hash[WG_HASH_LEN];
544 1.1 riastrad uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN];
545 1.1 riastrad uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN];
546 1.1 riastrad uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN];
547 1.1 riastrad uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN];
548 1.1 riastrad uint8_t wgs_tkey_send[WG_DATA_KEY_LEN];
549 1.1 riastrad uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN];
550 1.1 riastrad };
551 1.1 riastrad
552 1.1 riastrad struct wg_sockaddr {
553 1.1 riastrad union {
554 1.1 riastrad struct sockaddr_storage _ss;
555 1.1 riastrad struct sockaddr _sa;
556 1.1 riastrad struct sockaddr_in _sin;
557 1.1 riastrad struct sockaddr_in6 _sin6;
558 1.1 riastrad };
559 1.1 riastrad struct psref_target wgsa_psref;
560 1.1 riastrad };
561 1.1 riastrad
562 1.47 riastrad #define wgsatoss(wgsa) (&(wgsa)->_ss)
563 1.1 riastrad #define wgsatosa(wgsa) (&(wgsa)->_sa)
564 1.1 riastrad #define wgsatosin(wgsa) (&(wgsa)->_sin)
565 1.1 riastrad #define wgsatosin6(wgsa) (&(wgsa)->_sin6)
566 1.1 riastrad
567 1.47 riastrad #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family)
568 1.47 riastrad
569 1.1 riastrad struct wg_peer;
570 1.1 riastrad struct wg_allowedip {
571 1.1 riastrad struct radix_node wga_nodes[2];
572 1.1 riastrad struct wg_sockaddr _wga_sa_addr;
573 1.1 riastrad struct wg_sockaddr _wga_sa_mask;
574 1.1 riastrad #define wga_sa_addr _wga_sa_addr._sa
575 1.1 riastrad #define wga_sa_mask _wga_sa_mask._sa
576 1.1 riastrad
577 1.1 riastrad int wga_family;
578 1.1 riastrad uint8_t wga_cidr;
579 1.1 riastrad union {
580 1.1 riastrad struct in_addr _ip4;
581 1.1 riastrad struct in6_addr _ip6;
582 1.1 riastrad } wga_addr;
583 1.1 riastrad #define wga_addr4 wga_addr._ip4
584 1.1 riastrad #define wga_addr6 wga_addr._ip6
585 1.1 riastrad
586 1.1 riastrad struct wg_peer *wga_peer;
587 1.1 riastrad };
588 1.1 riastrad
589 1.1 riastrad typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN];
590 1.1 riastrad
591 1.1 riastrad struct wg_ppsratecheck {
592 1.1 riastrad struct timeval wgprc_lasttime;
593 1.1 riastrad int wgprc_curpps;
594 1.1 riastrad };
595 1.1 riastrad
596 1.1 riastrad struct wg_softc;
597 1.1 riastrad struct wg_peer {
598 1.1 riastrad struct wg_softc *wgp_sc;
599 1.1 riastrad char wgp_name[WG_PEER_NAME_MAXLEN + 1];
600 1.1 riastrad struct pslist_entry wgp_peerlist_entry;
601 1.1 riastrad pserialize_t wgp_psz;
602 1.1 riastrad struct psref_target wgp_psref;
603 1.1 riastrad kmutex_t *wgp_lock;
604 1.55 riastrad kmutex_t *wgp_intr_lock;
605 1.1 riastrad
606 1.1 riastrad uint8_t wgp_pubkey[WG_STATIC_KEY_LEN];
607 1.1 riastrad struct wg_sockaddr *wgp_endpoint;
608 1.1 riastrad struct wg_sockaddr *wgp_endpoint0;
609 1.49 riastrad volatile unsigned wgp_endpoint_changing;
610 1.1 riastrad bool wgp_endpoint_available;
611 1.1 riastrad
612 1.1 riastrad /* The preshared key (optional) */
613 1.1 riastrad uint8_t wgp_psk[WG_PRESHARED_KEY_LEN];
614 1.1 riastrad
615 1.1 riastrad struct wg_session *wgp_session_stable;
616 1.1 riastrad struct wg_session *wgp_session_unstable;
617 1.1 riastrad
618 1.54 riastrad /* first outgoing packet awaiting session initiation */
619 1.99 riastrad struct mbuf *volatile wgp_pending;
620 1.54 riastrad
621 1.1 riastrad /* timestamp in big-endian */
622 1.1 riastrad wg_timestamp_t wgp_timestamp_latest_init;
623 1.1 riastrad
624 1.1 riastrad struct timespec wgp_last_handshake_time;
625 1.1 riastrad
626 1.1 riastrad callout_t wgp_handshake_timeout_timer;
627 1.1 riastrad callout_t wgp_session_dtor_timer;
628 1.1 riastrad
629 1.1 riastrad time_t wgp_handshake_start_time;
630 1.1 riastrad
631 1.14 riastrad int wgp_n_allowedips;
632 1.1 riastrad struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS];
633 1.1 riastrad
634 1.1 riastrad time_t wgp_latest_cookie_time;
635 1.1 riastrad uint8_t wgp_latest_cookie[WG_COOKIE_LEN];
636 1.1 riastrad uint8_t wgp_last_sent_mac1[WG_MAC_LEN];
637 1.1 riastrad bool wgp_last_sent_mac1_valid;
638 1.1 riastrad uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN];
639 1.1 riastrad bool wgp_last_sent_cookie_valid;
640 1.1 riastrad
641 1.1 riastrad time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX];
642 1.1 riastrad
643 1.98 riastrad time_t wgp_last_cookiesecret_time;
644 1.98 riastrad uint8_t wgp_cookiesecret[WG_COOKIESECRET_LEN];
645 1.1 riastrad
646 1.1 riastrad struct wg_ppsratecheck wgp_ppsratecheck;
647 1.1 riastrad
648 1.55 riastrad struct work wgp_work;
649 1.55 riastrad unsigned int wgp_tasks;
650 1.1 riastrad #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0)
651 1.49 riastrad #define WGP_TASK_RETRY_HANDSHAKE __BIT(1)
652 1.49 riastrad #define WGP_TASK_ESTABLISH_SESSION __BIT(2)
653 1.49 riastrad #define WGP_TASK_ENDPOINT_CHANGED __BIT(3)
654 1.49 riastrad #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4)
655 1.49 riastrad #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5)
656 1.1 riastrad };
657 1.1 riastrad
658 1.1 riastrad struct wg_ops;
659 1.1 riastrad
660 1.1 riastrad struct wg_softc {
661 1.1 riastrad struct ifnet wg_if;
662 1.1 riastrad LIST_ENTRY(wg_softc) wg_list;
663 1.1 riastrad kmutex_t *wg_lock;
664 1.55 riastrad kmutex_t *wg_intr_lock;
665 1.1 riastrad krwlock_t *wg_rwlock;
666 1.1 riastrad
667 1.1 riastrad uint8_t wg_privkey[WG_STATIC_KEY_LEN];
668 1.1 riastrad uint8_t wg_pubkey[WG_STATIC_KEY_LEN];
669 1.1 riastrad
670 1.1 riastrad int wg_npeers;
671 1.1 riastrad struct pslist_head wg_peers;
672 1.37 riastrad struct thmap *wg_peers_bypubkey;
673 1.37 riastrad struct thmap *wg_peers_byname;
674 1.37 riastrad struct thmap *wg_sessions_byindex;
675 1.1 riastrad uint16_t wg_listen_port;
676 1.1 riastrad
677 1.55 riastrad struct threadpool *wg_threadpool;
678 1.1 riastrad
679 1.55 riastrad struct threadpool_job wg_job;
680 1.55 riastrad int wg_upcalls;
681 1.55 riastrad #define WG_UPCALL_INET __BIT(0)
682 1.55 riastrad #define WG_UPCALL_INET6 __BIT(1)
683 1.55 riastrad
684 1.55 riastrad #ifdef INET
685 1.55 riastrad struct socket *wg_so4;
686 1.1 riastrad struct radix_node_head *wg_rtable_ipv4;
687 1.55 riastrad #endif
688 1.55 riastrad #ifdef INET6
689 1.55 riastrad struct socket *wg_so6;
690 1.1 riastrad struct radix_node_head *wg_rtable_ipv6;
691 1.55 riastrad #endif
692 1.1 riastrad
693 1.1 riastrad struct wg_ppsratecheck wg_ppsratecheck;
694 1.1 riastrad
695 1.1 riastrad struct wg_ops *wg_ops;
696 1.1 riastrad
697 1.1 riastrad #ifdef WG_RUMPKERNEL
698 1.1 riastrad struct wg_user *wg_user;
699 1.1 riastrad #endif
700 1.1 riastrad };
701 1.1 riastrad
702 1.21 riastrad /* [W] 6.1 Preliminaries */
703 1.21 riastrad #define WG_REKEY_AFTER_MESSAGES (1ULL << 60)
704 1.21 riastrad #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13))
705 1.1 riastrad #define WG_REKEY_AFTER_TIME 120
706 1.1 riastrad #define WG_REJECT_AFTER_TIME 180
707 1.1 riastrad #define WG_REKEY_ATTEMPT_TIME 90
708 1.1 riastrad #define WG_REKEY_TIMEOUT 5
709 1.1 riastrad #define WG_KEEPALIVE_TIMEOUT 10
710 1.1 riastrad
711 1.1 riastrad #define WG_COOKIE_TIME 120
712 1.98 riastrad #define WG_COOKIESECRET_TIME (2 * 60)
713 1.1 riastrad
714 1.1 riastrad static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES;
715 1.1 riastrad static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES;
716 1.21 riastrad static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME;
717 1.21 riastrad static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME;
718 1.21 riastrad static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME;
719 1.21 riastrad static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT;
720 1.21 riastrad static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT;
721 1.1 riastrad
722 1.1 riastrad static struct mbuf *
723 1.1 riastrad wg_get_mbuf(size_t, size_t);
724 1.1 riastrad
725 1.108 riastrad static void wg_send_data_msg(struct wg_peer *, struct wg_session *,
726 1.1 riastrad struct mbuf *);
727 1.108 riastrad static void wg_send_cookie_msg(struct wg_softc *, struct wg_peer *,
728 1.114 riastrad const uint32_t, const uint8_t[static WG_MAC_LEN],
729 1.77 mrg const struct sockaddr *);
730 1.108 riastrad static void wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *,
731 1.49 riastrad struct wg_session *, const struct wg_msg_init *);
732 1.1 riastrad static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *);
733 1.1 riastrad
734 1.1 riastrad static struct wg_peer *
735 1.1 riastrad wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *,
736 1.1 riastrad struct psref *);
737 1.1 riastrad static struct wg_peer *
738 1.1 riastrad wg_lookup_peer_by_pubkey(struct wg_softc *,
739 1.114 riastrad const uint8_t[static WG_STATIC_KEY_LEN], struct psref *);
740 1.1 riastrad
741 1.1 riastrad static struct wg_session *
742 1.1 riastrad wg_lookup_session_by_index(struct wg_softc *,
743 1.1 riastrad const uint32_t, struct psref *);
744 1.1 riastrad
745 1.1 riastrad static void wg_update_endpoint_if_necessary(struct wg_peer *,
746 1.1 riastrad const struct sockaddr *);
747 1.1 riastrad
748 1.1 riastrad static void wg_schedule_session_dtor_timer(struct wg_peer *);
749 1.1 riastrad
750 1.1 riastrad static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int);
751 1.1 riastrad static void wg_calculate_keys(struct wg_session *, const bool);
752 1.1 riastrad
753 1.1 riastrad static void wg_clear_states(struct wg_session *);
754 1.1 riastrad
755 1.1 riastrad static void wg_get_peer(struct wg_peer *, struct psref *);
756 1.1 riastrad static void wg_put_peer(struct wg_peer *, struct psref *);
757 1.1 riastrad
758 1.1 riastrad static int wg_send_so(struct wg_peer *, struct mbuf *);
759 1.1 riastrad static int wg_send_udp(struct wg_peer *, struct mbuf *);
760 1.1 riastrad static int wg_output(struct ifnet *, struct mbuf *,
761 1.1 riastrad const struct sockaddr *, const struct rtentry *);
762 1.1 riastrad static void wg_input(struct ifnet *, struct mbuf *, const int);
763 1.1 riastrad static int wg_ioctl(struct ifnet *, u_long, void *);
764 1.1 riastrad static int wg_bind_port(struct wg_softc *, const uint16_t);
765 1.1 riastrad static int wg_init(struct ifnet *);
766 1.60 riastrad #ifdef ALTQ
767 1.60 riastrad static void wg_start(struct ifnet *);
768 1.60 riastrad #endif
769 1.1 riastrad static void wg_stop(struct ifnet *, int);
770 1.1 riastrad
771 1.55 riastrad static void wg_peer_work(struct work *, void *);
772 1.55 riastrad static void wg_job(struct threadpool_job *);
773 1.54 riastrad static void wgintr(void *);
774 1.49 riastrad static void wg_purge_pending_packets(struct wg_peer *);
775 1.49 riastrad
776 1.1 riastrad static int wg_clone_create(struct if_clone *, int);
777 1.1 riastrad static int wg_clone_destroy(struct ifnet *);
778 1.1 riastrad
779 1.1 riastrad struct wg_ops {
780 1.1 riastrad int (*send_hs_msg)(struct wg_peer *, struct mbuf *);
781 1.1 riastrad int (*send_data_msg)(struct wg_peer *, struct mbuf *);
782 1.1 riastrad void (*input)(struct ifnet *, struct mbuf *, const int);
783 1.1 riastrad int (*bind_port)(struct wg_softc *, const uint16_t);
784 1.1 riastrad };
785 1.1 riastrad
786 1.1 riastrad struct wg_ops wg_ops_rumpkernel = {
787 1.1 riastrad .send_hs_msg = wg_send_so,
788 1.1 riastrad .send_data_msg = wg_send_udp,
789 1.1 riastrad .input = wg_input,
790 1.1 riastrad .bind_port = wg_bind_port,
791 1.1 riastrad };
792 1.1 riastrad
793 1.1 riastrad #ifdef WG_RUMPKERNEL
794 1.1 riastrad static bool wg_user_mode(struct wg_softc *);
795 1.1 riastrad static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *);
796 1.1 riastrad
797 1.1 riastrad static int wg_send_user(struct wg_peer *, struct mbuf *);
798 1.1 riastrad static void wg_input_user(struct ifnet *, struct mbuf *, const int);
799 1.1 riastrad static int wg_bind_port_user(struct wg_softc *, const uint16_t);
800 1.1 riastrad
801 1.1 riastrad struct wg_ops wg_ops_rumpuser = {
802 1.1 riastrad .send_hs_msg = wg_send_user,
803 1.1 riastrad .send_data_msg = wg_send_user,
804 1.1 riastrad .input = wg_input_user,
805 1.1 riastrad .bind_port = wg_bind_port_user,
806 1.1 riastrad };
807 1.1 riastrad #endif
808 1.1 riastrad
809 1.1 riastrad #define WG_PEER_READER_FOREACH(wgp, wg) \
810 1.1 riastrad PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \
811 1.1 riastrad wgp_peerlist_entry)
812 1.1 riastrad #define WG_PEER_WRITER_FOREACH(wgp, wg) \
813 1.1 riastrad PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \
814 1.1 riastrad wgp_peerlist_entry)
815 1.1 riastrad #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \
816 1.1 riastrad PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry)
817 1.1 riastrad #define WG_PEER_WRITER_REMOVE(wgp) \
818 1.1 riastrad PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry)
819 1.1 riastrad
820 1.1 riastrad struct wg_route {
821 1.1 riastrad struct radix_node wgr_nodes[2];
822 1.1 riastrad struct wg_peer *wgr_peer;
823 1.1 riastrad };
824 1.1 riastrad
825 1.1 riastrad static struct radix_node_head *
826 1.1 riastrad wg_rnh(struct wg_softc *wg, const int family)
827 1.1 riastrad {
828 1.1 riastrad
829 1.1 riastrad switch (family) {
830 1.109 riastrad #ifdef INET
831 1.1 riastrad case AF_INET:
832 1.1 riastrad return wg->wg_rtable_ipv4;
833 1.109 riastrad #endif
834 1.1 riastrad #ifdef INET6
835 1.1 riastrad case AF_INET6:
836 1.1 riastrad return wg->wg_rtable_ipv6;
837 1.1 riastrad #endif
838 1.1 riastrad default:
839 1.1 riastrad return NULL;
840 1.1 riastrad }
841 1.1 riastrad }
842 1.1 riastrad
843 1.1 riastrad
844 1.1 riastrad /*
845 1.1 riastrad * Global variables
846 1.1 riastrad */
847 1.59 riastrad static volatile unsigned wg_count __cacheline_aligned;
848 1.1 riastrad
849 1.1 riastrad struct psref_class *wg_psref_class __read_mostly;
850 1.1 riastrad
851 1.1 riastrad static struct if_clone wg_cloner =
852 1.1 riastrad IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy);
853 1.1 riastrad
854 1.54 riastrad static struct pktqueue *wg_pktq __read_mostly;
855 1.55 riastrad static struct workqueue *wg_wq __read_mostly;
856 1.1 riastrad
857 1.1 riastrad void wgattach(int);
858 1.1 riastrad /* ARGSUSED */
859 1.1 riastrad void
860 1.1 riastrad wgattach(int count)
861 1.1 riastrad {
862 1.1 riastrad /*
863 1.1 riastrad * Nothing to do here, initialization is handled by the
864 1.1 riastrad * module initialization code in wginit() below).
865 1.1 riastrad */
866 1.1 riastrad }
867 1.1 riastrad
868 1.1 riastrad static void
869 1.1 riastrad wginit(void)
870 1.1 riastrad {
871 1.1 riastrad
872 1.1 riastrad wg_psref_class = psref_class_create("wg", IPL_SOFTNET);
873 1.1 riastrad
874 1.58 riastrad if_clone_attach(&wg_cloner);
875 1.58 riastrad }
876 1.58 riastrad
877 1.58 riastrad /*
878 1.58 riastrad * XXX Kludge: This should just happen in wginit, but workqueue_create
879 1.58 riastrad * cannot be run until after CPUs have been detected, and wginit runs
880 1.58 riastrad * before configure.
881 1.58 riastrad */
882 1.58 riastrad static int
883 1.58 riastrad wginitqueues(void)
884 1.58 riastrad {
885 1.58 riastrad int error __diagused;
886 1.58 riastrad
887 1.54 riastrad wg_pktq = pktq_create(IFQ_MAXLEN, wgintr, NULL);
888 1.54 riastrad KASSERT(wg_pktq != NULL);
889 1.54 riastrad
890 1.55 riastrad error = workqueue_create(&wg_wq, "wgpeer", wg_peer_work, NULL,
891 1.55 riastrad PRI_NONE, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU);
892 1.108 riastrad KASSERTMSG(error == 0, "error=%d", error);
893 1.55 riastrad
894 1.58 riastrad return 0;
895 1.58 riastrad }
896 1.58 riastrad
897 1.58 riastrad static void
898 1.58 riastrad wg_guarantee_initialized(void)
899 1.58 riastrad {
900 1.58 riastrad static ONCE_DECL(init);
901 1.58 riastrad int error __diagused;
902 1.58 riastrad
903 1.58 riastrad error = RUN_ONCE(&init, wginitqueues);
904 1.108 riastrad KASSERTMSG(error == 0, "error=%d", error);
905 1.1 riastrad }
906 1.1 riastrad
907 1.1 riastrad static int
908 1.59 riastrad wg_count_inc(void)
909 1.59 riastrad {
910 1.59 riastrad unsigned o, n;
911 1.59 riastrad
912 1.59 riastrad do {
913 1.59 riastrad o = atomic_load_relaxed(&wg_count);
914 1.59 riastrad if (o == UINT_MAX)
915 1.59 riastrad return ENFILE;
916 1.59 riastrad n = o + 1;
917 1.59 riastrad } while (atomic_cas_uint(&wg_count, o, n) != o);
918 1.59 riastrad
919 1.59 riastrad return 0;
920 1.59 riastrad }
921 1.59 riastrad
922 1.59 riastrad static void
923 1.59 riastrad wg_count_dec(void)
924 1.59 riastrad {
925 1.59 riastrad unsigned c __diagused;
926 1.59 riastrad
927 1.59 riastrad c = atomic_dec_uint_nv(&wg_count);
928 1.59 riastrad KASSERT(c != UINT_MAX);
929 1.59 riastrad }
930 1.59 riastrad
931 1.59 riastrad static int
932 1.1 riastrad wgdetach(void)
933 1.1 riastrad {
934 1.1 riastrad
935 1.59 riastrad /* Prevent new interface creation. */
936 1.59 riastrad if_clone_detach(&wg_cloner);
937 1.59 riastrad
938 1.59 riastrad /* Check whether there are any existing interfaces. */
939 1.59 riastrad if (atomic_load_relaxed(&wg_count)) {
940 1.59 riastrad /* Back out -- reattach the cloner. */
941 1.59 riastrad if_clone_attach(&wg_cloner);
942 1.59 riastrad return EBUSY;
943 1.1 riastrad }
944 1.1 riastrad
945 1.59 riastrad /* No interfaces left. Nuke it. */
946 1.92 riastrad if (wg_wq)
947 1.92 riastrad workqueue_destroy(wg_wq);
948 1.92 riastrad if (wg_pktq)
949 1.92 riastrad pktq_destroy(wg_pktq);
950 1.59 riastrad psref_class_destroy(wg_psref_class);
951 1.1 riastrad
952 1.59 riastrad return 0;
953 1.1 riastrad }
954 1.1 riastrad
955 1.1 riastrad static void
956 1.114 riastrad wg_init_key_and_hash(uint8_t ckey[static WG_CHAINING_KEY_LEN],
957 1.114 riastrad uint8_t hash[static WG_HASH_LEN])
958 1.1 riastrad {
959 1.1 riastrad /* [W] 5.4: CONSTRUCTION */
960 1.1 riastrad const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s";
961 1.1 riastrad /* [W] 5.4: IDENTIFIER */
962 1.1 riastrad const char *id = "WireGuard v1 zx2c4 Jason (at) zx2c4.com";
963 1.1 riastrad struct blake2s state;
964 1.1 riastrad
965 1.1 riastrad blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0,
966 1.1 riastrad signature, strlen(signature));
967 1.1 riastrad
968 1.1 riastrad CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN);
969 1.1 riastrad memcpy(hash, ckey, WG_CHAINING_KEY_LEN);
970 1.1 riastrad
971 1.1 riastrad blake2s_init(&state, WG_HASH_LEN, NULL, 0);
972 1.1 riastrad blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN);
973 1.1 riastrad blake2s_update(&state, id, strlen(id));
974 1.1 riastrad blake2s_final(&state, hash);
975 1.1 riastrad
976 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
977 1.1 riastrad WG_DUMP_HASH("hash", hash);
978 1.1 riastrad }
979 1.1 riastrad
980 1.1 riastrad static void
981 1.114 riastrad wg_algo_hash(uint8_t hash[static WG_HASH_LEN], const uint8_t input[],
982 1.1 riastrad const size_t inputsize)
983 1.1 riastrad {
984 1.1 riastrad struct blake2s state;
985 1.1 riastrad
986 1.1 riastrad blake2s_init(&state, WG_HASH_LEN, NULL, 0);
987 1.1 riastrad blake2s_update(&state, hash, WG_HASH_LEN);
988 1.1 riastrad blake2s_update(&state, input, inputsize);
989 1.1 riastrad blake2s_final(&state, hash);
990 1.1 riastrad }
991 1.1 riastrad
992 1.1 riastrad static void
993 1.1 riastrad wg_algo_mac(uint8_t out[], const size_t outsize,
994 1.1 riastrad const uint8_t key[], const size_t keylen,
995 1.1 riastrad const uint8_t input1[], const size_t input1len,
996 1.1 riastrad const uint8_t input2[], const size_t input2len)
997 1.1 riastrad {
998 1.1 riastrad struct blake2s state;
999 1.1 riastrad
1000 1.1 riastrad blake2s_init(&state, outsize, key, keylen);
1001 1.1 riastrad
1002 1.1 riastrad blake2s_update(&state, input1, input1len);
1003 1.1 riastrad if (input2 != NULL)
1004 1.1 riastrad blake2s_update(&state, input2, input2len);
1005 1.1 riastrad blake2s_final(&state, out);
1006 1.1 riastrad }
1007 1.1 riastrad
1008 1.1 riastrad static void
1009 1.1 riastrad wg_algo_mac_mac1(uint8_t out[], const size_t outsize,
1010 1.1 riastrad const uint8_t input1[], const size_t input1len,
1011 1.1 riastrad const uint8_t input2[], const size_t input2len)
1012 1.1 riastrad {
1013 1.1 riastrad struct blake2s state;
1014 1.1 riastrad /* [W] 5.4: LABEL-MAC1 */
1015 1.1 riastrad const char *label = "mac1----";
1016 1.1 riastrad uint8_t key[WG_HASH_LEN];
1017 1.1 riastrad
1018 1.1 riastrad blake2s_init(&state, sizeof(key), NULL, 0);
1019 1.1 riastrad blake2s_update(&state, label, strlen(label));
1020 1.1 riastrad blake2s_update(&state, input1, input1len);
1021 1.1 riastrad blake2s_final(&state, key);
1022 1.1 riastrad
1023 1.1 riastrad blake2s_init(&state, outsize, key, sizeof(key));
1024 1.1 riastrad if (input2 != NULL)
1025 1.1 riastrad blake2s_update(&state, input2, input2len);
1026 1.1 riastrad blake2s_final(&state, out);
1027 1.1 riastrad }
1028 1.1 riastrad
1029 1.1 riastrad static void
1030 1.1 riastrad wg_algo_mac_cookie(uint8_t out[], const size_t outsize,
1031 1.1 riastrad const uint8_t input1[], const size_t input1len)
1032 1.1 riastrad {
1033 1.1 riastrad struct blake2s state;
1034 1.1 riastrad /* [W] 5.4: LABEL-COOKIE */
1035 1.1 riastrad const char *label = "cookie--";
1036 1.1 riastrad
1037 1.1 riastrad blake2s_init(&state, outsize, NULL, 0);
1038 1.1 riastrad blake2s_update(&state, label, strlen(label));
1039 1.1 riastrad blake2s_update(&state, input1, input1len);
1040 1.1 riastrad blake2s_final(&state, out);
1041 1.1 riastrad }
1042 1.1 riastrad
1043 1.1 riastrad static void
1044 1.114 riastrad wg_algo_generate_keypair(uint8_t pubkey[static WG_EPHEMERAL_KEY_LEN],
1045 1.114 riastrad uint8_t privkey[static WG_EPHEMERAL_KEY_LEN])
1046 1.1 riastrad {
1047 1.1 riastrad
1048 1.1 riastrad CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES);
1049 1.1 riastrad
1050 1.3 riastrad cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0);
1051 1.1 riastrad crypto_scalarmult_base(pubkey, privkey);
1052 1.1 riastrad }
1053 1.1 riastrad
1054 1.1 riastrad static void
1055 1.114 riastrad wg_algo_dh(uint8_t out[static WG_DH_OUTPUT_LEN],
1056 1.114 riastrad const uint8_t privkey[static WG_STATIC_KEY_LEN],
1057 1.114 riastrad const uint8_t pubkey[static WG_STATIC_KEY_LEN])
1058 1.1 riastrad {
1059 1.1 riastrad
1060 1.1 riastrad CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES);
1061 1.1 riastrad
1062 1.19 riastrad int ret __diagused = crypto_scalarmult(out, privkey, pubkey);
1063 1.1 riastrad KASSERT(ret == 0);
1064 1.1 riastrad }
1065 1.1 riastrad
1066 1.1 riastrad static void
1067 1.1 riastrad wg_algo_hmac(uint8_t out[], const size_t outlen,
1068 1.1 riastrad const uint8_t key[], const size_t keylen,
1069 1.1 riastrad const uint8_t in[], const size_t inlen)
1070 1.1 riastrad {
1071 1.1 riastrad #define IPAD 0x36
1072 1.1 riastrad #define OPAD 0x5c
1073 1.1 riastrad uint8_t hmackey[HMAC_BLOCK_LEN] = {0};
1074 1.1 riastrad uint8_t ipad[HMAC_BLOCK_LEN];
1075 1.1 riastrad uint8_t opad[HMAC_BLOCK_LEN];
1076 1.65 christos size_t i;
1077 1.1 riastrad struct blake2s state;
1078 1.1 riastrad
1079 1.1 riastrad KASSERT(outlen == WG_HASH_LEN);
1080 1.1 riastrad KASSERT(keylen <= HMAC_BLOCK_LEN);
1081 1.1 riastrad
1082 1.1 riastrad memcpy(hmackey, key, keylen);
1083 1.1 riastrad
1084 1.1 riastrad for (i = 0; i < sizeof(hmackey); i++) {
1085 1.1 riastrad ipad[i] = hmackey[i] ^ IPAD;
1086 1.1 riastrad opad[i] = hmackey[i] ^ OPAD;
1087 1.1 riastrad }
1088 1.1 riastrad
1089 1.1 riastrad blake2s_init(&state, WG_HASH_LEN, NULL, 0);
1090 1.1 riastrad blake2s_update(&state, ipad, sizeof(ipad));
1091 1.1 riastrad blake2s_update(&state, in, inlen);
1092 1.1 riastrad blake2s_final(&state, out);
1093 1.1 riastrad
1094 1.1 riastrad blake2s_init(&state, WG_HASH_LEN, NULL, 0);
1095 1.1 riastrad blake2s_update(&state, opad, sizeof(opad));
1096 1.1 riastrad blake2s_update(&state, out, WG_HASH_LEN);
1097 1.1 riastrad blake2s_final(&state, out);
1098 1.1 riastrad #undef IPAD
1099 1.1 riastrad #undef OPAD
1100 1.1 riastrad }
1101 1.1 riastrad
1102 1.1 riastrad static void
1103 1.114 riastrad wg_algo_kdf(uint8_t out1[static WG_KDF_OUTPUT_LEN],
1104 1.114 riastrad uint8_t out2[WG_KDF_OUTPUT_LEN],
1105 1.114 riastrad uint8_t out3[WG_KDF_OUTPUT_LEN],
1106 1.114 riastrad const uint8_t ckey[static WG_CHAINING_KEY_LEN],
1107 1.1 riastrad const uint8_t input[], const size_t inputlen)
1108 1.1 riastrad {
1109 1.1 riastrad uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1];
1110 1.1 riastrad uint8_t one[1];
1111 1.1 riastrad
1112 1.1 riastrad /*
1113 1.14 riastrad * [N] 4.3: "an input_key_material byte sequence with length
1114 1.14 riastrad * either zero bytes, 32 bytes, or DHLEN bytes."
1115 1.1 riastrad */
1116 1.1 riastrad KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN);
1117 1.1 riastrad
1118 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
1119 1.1 riastrad if (input != NULL)
1120 1.1 riastrad WG_DUMP_HASH("input", input);
1121 1.1 riastrad wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN,
1122 1.1 riastrad input, inputlen);
1123 1.1 riastrad WG_DUMP_HASH("tmp1", tmp1);
1124 1.1 riastrad one[0] = 1;
1125 1.1 riastrad wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1),
1126 1.1 riastrad one, sizeof(one));
1127 1.1 riastrad WG_DUMP_HASH("out1", out1);
1128 1.1 riastrad if (out2 == NULL)
1129 1.1 riastrad return;
1130 1.1 riastrad memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN);
1131 1.1 riastrad tmp2[WG_KDF_OUTPUT_LEN] = 2;
1132 1.1 riastrad wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1),
1133 1.1 riastrad tmp2, sizeof(tmp2));
1134 1.1 riastrad WG_DUMP_HASH("out2", out2);
1135 1.1 riastrad if (out3 == NULL)
1136 1.1 riastrad return;
1137 1.1 riastrad memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN);
1138 1.1 riastrad tmp2[WG_KDF_OUTPUT_LEN] = 3;
1139 1.1 riastrad wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1),
1140 1.1 riastrad tmp2, sizeof(tmp2));
1141 1.1 riastrad WG_DUMP_HASH("out3", out3);
1142 1.1 riastrad }
1143 1.1 riastrad
1144 1.63 riastrad static void __noinline
1145 1.114 riastrad wg_algo_dh_kdf(uint8_t ckey[static WG_CHAINING_KEY_LEN],
1146 1.1 riastrad uint8_t cipher_key[WG_CIPHER_KEY_LEN],
1147 1.114 riastrad const uint8_t local_key[static WG_STATIC_KEY_LEN],
1148 1.114 riastrad const uint8_t remote_key[static WG_STATIC_KEY_LEN])
1149 1.1 riastrad {
1150 1.1 riastrad uint8_t dhout[WG_DH_OUTPUT_LEN];
1151 1.1 riastrad
1152 1.1 riastrad wg_algo_dh(dhout, local_key, remote_key);
1153 1.1 riastrad wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout));
1154 1.1 riastrad
1155 1.1 riastrad WG_DUMP_HASH("dhout", dhout);
1156 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
1157 1.1 riastrad if (cipher_key != NULL)
1158 1.1 riastrad WG_DUMP_HASH("cipher_key", cipher_key);
1159 1.1 riastrad }
1160 1.1 riastrad
1161 1.1 riastrad static void
1162 1.114 riastrad wg_algo_aead_enc(uint8_t out[], size_t expected_outsize,
1163 1.114 riastrad const uint8_t key[static crypto_aead_chacha20poly1305_ietf_KEYBYTES],
1164 1.114 riastrad const uint64_t counter,
1165 1.114 riastrad const uint8_t plain[], const size_t plainsize,
1166 1.1 riastrad const uint8_t auth[], size_t authlen)
1167 1.1 riastrad {
1168 1.1 riastrad uint8_t nonce[(32 + 64) / 8] = {0};
1169 1.1 riastrad long long unsigned int outsize;
1170 1.1 riastrad int error __diagused;
1171 1.1 riastrad
1172 1.39 riastrad le64enc(&nonce[4], counter);
1173 1.1 riastrad
1174 1.1 riastrad error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain,
1175 1.1 riastrad plainsize, auth, authlen, NULL, nonce, key);
1176 1.1 riastrad KASSERT(error == 0);
1177 1.1 riastrad KASSERT(outsize == expected_outsize);
1178 1.1 riastrad }
1179 1.1 riastrad
1180 1.1 riastrad static int
1181 1.114 riastrad wg_algo_aead_dec(uint8_t out[], size_t expected_outsize,
1182 1.114 riastrad const uint8_t key[static crypto_aead_chacha20poly1305_ietf_KEYBYTES],
1183 1.114 riastrad const uint64_t counter,
1184 1.114 riastrad const uint8_t encrypted[], const size_t encryptedsize,
1185 1.114 riastrad const uint8_t auth[], size_t authlen)
1186 1.1 riastrad {
1187 1.1 riastrad uint8_t nonce[(32 + 64) / 8] = {0};
1188 1.1 riastrad long long unsigned int outsize;
1189 1.1 riastrad int error;
1190 1.1 riastrad
1191 1.39 riastrad le64enc(&nonce[4], counter);
1192 1.1 riastrad
1193 1.1 riastrad error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL,
1194 1.1 riastrad encrypted, encryptedsize, auth, authlen, nonce, key);
1195 1.1 riastrad if (error == 0)
1196 1.1 riastrad KASSERT(outsize == expected_outsize);
1197 1.1 riastrad return error;
1198 1.1 riastrad }
1199 1.1 riastrad
1200 1.1 riastrad static void
1201 1.1 riastrad wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize,
1202 1.114 riastrad const uint8_t key[static crypto_aead_xchacha20poly1305_ietf_KEYBYTES],
1203 1.114 riastrad const uint8_t plain[], const size_t plainsize,
1204 1.1 riastrad const uint8_t auth[], size_t authlen,
1205 1.114 riastrad const uint8_t nonce[static WG_SALT_LEN])
1206 1.1 riastrad {
1207 1.1 riastrad long long unsigned int outsize;
1208 1.1 riastrad int error __diagused;
1209 1.1 riastrad
1210 1.1 riastrad CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES);
1211 1.14 riastrad error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize,
1212 1.14 riastrad plain, plainsize, auth, authlen, NULL, nonce, key);
1213 1.1 riastrad KASSERT(error == 0);
1214 1.1 riastrad KASSERT(outsize == expected_outsize);
1215 1.1 riastrad }
1216 1.1 riastrad
1217 1.1 riastrad static int
1218 1.1 riastrad wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize,
1219 1.114 riastrad const uint8_t key[static crypto_aead_xchacha20poly1305_ietf_KEYBYTES],
1220 1.114 riastrad const uint8_t encrypted[], const size_t encryptedsize,
1221 1.1 riastrad const uint8_t auth[], size_t authlen,
1222 1.114 riastrad const uint8_t nonce[static WG_SALT_LEN])
1223 1.1 riastrad {
1224 1.1 riastrad long long unsigned int outsize;
1225 1.1 riastrad int error;
1226 1.1 riastrad
1227 1.1 riastrad error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL,
1228 1.1 riastrad encrypted, encryptedsize, auth, authlen, nonce, key);
1229 1.1 riastrad if (error == 0)
1230 1.1 riastrad KASSERT(outsize == expected_outsize);
1231 1.1 riastrad return error;
1232 1.1 riastrad }
1233 1.1 riastrad
1234 1.1 riastrad static void
1235 1.15 riastrad wg_algo_tai64n(wg_timestamp_t timestamp)
1236 1.1 riastrad {
1237 1.1 riastrad struct timespec ts;
1238 1.1 riastrad
1239 1.1 riastrad /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */
1240 1.1 riastrad getnanotime(&ts);
1241 1.1 riastrad /* TAI64 label in external TAI64 format */
1242 1.65 christos be32enc(timestamp, 0x40000000U + (uint32_t)(ts.tv_sec >> 32));
1243 1.1 riastrad /* second beginning from 1970 TAI */
1244 1.65 christos be32enc(timestamp + 4, (uint32_t)(ts.tv_sec & 0xffffffffU));
1245 1.1 riastrad /* nanosecond in big-endian format */
1246 1.65 christos be32enc(timestamp + 8, (uint32_t)ts.tv_nsec);
1247 1.1 riastrad }
1248 1.1 riastrad
1249 1.49 riastrad /*
1250 1.49 riastrad * wg_get_stable_session(wgp, psref)
1251 1.49 riastrad *
1252 1.49 riastrad * Get a passive reference to the current stable session, or
1253 1.49 riastrad * return NULL if there is no current stable session.
1254 1.49 riastrad *
1255 1.49 riastrad * The pointer is always there but the session is not necessarily
1256 1.49 riastrad * ESTABLISHED; if it is not ESTABLISHED, return NULL. However,
1257 1.49 riastrad * the session may transition from ESTABLISHED to DESTROYING while
1258 1.49 riastrad * holding the passive reference.
1259 1.49 riastrad */
1260 1.1 riastrad static struct wg_session *
1261 1.49 riastrad wg_get_stable_session(struct wg_peer *wgp, struct psref *psref)
1262 1.1 riastrad {
1263 1.1 riastrad int s;
1264 1.1 riastrad struct wg_session *wgs;
1265 1.1 riastrad
1266 1.1 riastrad s = pserialize_read_enter();
1267 1.49 riastrad wgs = atomic_load_consume(&wgp->wgp_session_stable);
1268 1.49 riastrad if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED))
1269 1.49 riastrad wgs = NULL;
1270 1.49 riastrad else
1271 1.49 riastrad psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
1272 1.1 riastrad pserialize_read_exit(s);
1273 1.1 riastrad
1274 1.1 riastrad return wgs;
1275 1.1 riastrad }
1276 1.1 riastrad
1277 1.1 riastrad static void
1278 1.49 riastrad wg_put_session(struct wg_session *wgs, struct psref *psref)
1279 1.1 riastrad {
1280 1.1 riastrad
1281 1.49 riastrad psref_release(psref, &wgs->wgs_psref, wg_psref_class);
1282 1.1 riastrad }
1283 1.1 riastrad
1284 1.1 riastrad static void
1285 1.49 riastrad wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs)
1286 1.1 riastrad {
1287 1.49 riastrad struct wg_peer *wgp = wgs->wgs_peer;
1288 1.49 riastrad struct wg_session *wgs0 __diagused;
1289 1.49 riastrad void *garbage;
1290 1.49 riastrad
1291 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1292 1.49 riastrad KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN);
1293 1.1 riastrad
1294 1.49 riastrad /* Remove the session from the table. */
1295 1.49 riastrad wgs0 = thmap_del(wg->wg_sessions_byindex,
1296 1.49 riastrad &wgs->wgs_local_index, sizeof(wgs->wgs_local_index));
1297 1.49 riastrad KASSERT(wgs0 == wgs);
1298 1.49 riastrad garbage = thmap_stage_gc(wg->wg_sessions_byindex);
1299 1.1 riastrad
1300 1.49 riastrad /* Wait for passive references to drain. */
1301 1.49 riastrad pserialize_perform(wgp->wgp_psz);
1302 1.49 riastrad psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
1303 1.1 riastrad
1304 1.94 riastrad /*
1305 1.94 riastrad * Free memory, zero state, and transition to UNKNOWN. We have
1306 1.94 riastrad * exclusive access to the session now, so there is no need for
1307 1.94 riastrad * an atomic store.
1308 1.94 riastrad */
1309 1.49 riastrad thmap_gc(wg->wg_sessions_byindex, garbage);
1310 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_UNKNOWN\n",
1311 1.94 riastrad wgs->wgs_local_index, wgs->wgs_remote_index);
1312 1.94 riastrad wgs->wgs_local_index = 0;
1313 1.94 riastrad wgs->wgs_remote_index = 0;
1314 1.49 riastrad wg_clear_states(wgs);
1315 1.49 riastrad wgs->wgs_state = WGS_STATE_UNKNOWN;
1316 1.113 riastrad wgs->wgs_force_rekey = false;
1317 1.1 riastrad }
1318 1.1 riastrad
1319 1.49 riastrad /*
1320 1.49 riastrad * wg_get_session_index(wg, wgs)
1321 1.49 riastrad *
1322 1.49 riastrad * Choose a session index for wgs->wgs_local_index, and store it
1323 1.49 riastrad * in wg's table of sessions by index.
1324 1.49 riastrad *
1325 1.49 riastrad * wgs must be the unstable session of its peer, and must be
1326 1.49 riastrad * transitioning out of the UNKNOWN state.
1327 1.49 riastrad */
1328 1.1 riastrad static void
1329 1.49 riastrad wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs)
1330 1.1 riastrad {
1331 1.49 riastrad struct wg_peer *wgp __diagused = wgs->wgs_peer;
1332 1.37 riastrad struct wg_session *wgs0;
1333 1.37 riastrad uint32_t index;
1334 1.37 riastrad
1335 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1336 1.49 riastrad KASSERT(wgs == wgp->wgp_session_unstable);
1337 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1338 1.94 riastrad wgs->wgs_state);
1339 1.37 riastrad
1340 1.49 riastrad do {
1341 1.49 riastrad /* Pick a uniform random index. */
1342 1.49 riastrad index = cprng_strong32();
1343 1.49 riastrad
1344 1.49 riastrad /* Try to take it. */
1345 1.49 riastrad wgs->wgs_local_index = index;
1346 1.49 riastrad wgs0 = thmap_put(wg->wg_sessions_byindex,
1347 1.49 riastrad &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs);
1348 1.37 riastrad
1349 1.49 riastrad /* If someone else beat us, start over. */
1350 1.49 riastrad } while (__predict_false(wgs0 != wgs));
1351 1.49 riastrad }
1352 1.37 riastrad
1353 1.49 riastrad /*
1354 1.49 riastrad * wg_put_session_index(wg, wgs)
1355 1.49 riastrad *
1356 1.49 riastrad * Remove wgs from the table of sessions by index, wait for any
1357 1.49 riastrad * passive references to drain, and transition the session to the
1358 1.49 riastrad * UNKNOWN state.
1359 1.49 riastrad *
1360 1.49 riastrad * wgs must be the unstable session of its peer, and must not be
1361 1.49 riastrad * UNKNOWN or ESTABLISHED.
1362 1.49 riastrad */
1363 1.49 riastrad static void
1364 1.49 riastrad wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs)
1365 1.49 riastrad {
1366 1.52 riastrad struct wg_peer *wgp __diagused = wgs->wgs_peer;
1367 1.37 riastrad
1368 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1369 1.49 riastrad KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN);
1370 1.49 riastrad KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED);
1371 1.37 riastrad
1372 1.49 riastrad wg_destroy_session(wg, wgs);
1373 1.49 riastrad psref_target_init(&wgs->wgs_psref, wg_psref_class);
1374 1.37 riastrad }
1375 1.37 riastrad
1376 1.1 riastrad /*
1377 1.1 riastrad * Handshake patterns
1378 1.1 riastrad *
1379 1.1 riastrad * [W] 5: "These messages use the "IK" pattern from Noise"
1380 1.1 riastrad * [N] 7.5. Interactive handshake patterns (fundamental)
1381 1.1 riastrad * "The first character refers to the initiators static key:"
1382 1.1 riastrad * "I = Static key for initiator Immediately transmitted to responder,
1383 1.1 riastrad * despite reduced or absent identity hiding"
1384 1.1 riastrad * "The second character refers to the responders static key:"
1385 1.1 riastrad * "K = Static key for responder Known to initiator"
1386 1.1 riastrad * "IK:
1387 1.1 riastrad * <- s
1388 1.1 riastrad * ...
1389 1.1 riastrad * -> e, es, s, ss
1390 1.1 riastrad * <- e, ee, se"
1391 1.1 riastrad * [N] 9.4. Pattern modifiers
1392 1.1 riastrad * "IKpsk2:
1393 1.1 riastrad * <- s
1394 1.1 riastrad * ...
1395 1.1 riastrad * -> e, es, s, ss
1396 1.1 riastrad * <- e, ee, se, psk"
1397 1.1 riastrad */
1398 1.1 riastrad static void
1399 1.1 riastrad wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp,
1400 1.1 riastrad struct wg_session *wgs, struct wg_msg_init *wgmi)
1401 1.1 riastrad {
1402 1.1 riastrad uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
1403 1.1 riastrad uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
1404 1.1 riastrad uint8_t cipher_key[WG_CIPHER_KEY_LEN];
1405 1.1 riastrad uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
1406 1.1 riastrad uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
1407 1.1 riastrad
1408 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1409 1.49 riastrad KASSERT(wgs == wgp->wgp_session_unstable);
1410 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_INIT_ACTIVE, "state=%d",
1411 1.94 riastrad wgs->wgs_state);
1412 1.49 riastrad
1413 1.39 riastrad wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT);
1414 1.49 riastrad wgmi->wgmi_sender = wgs->wgs_local_index;
1415 1.1 riastrad
1416 1.1 riastrad /* [W] 5.4.2: First Message: Initiator to Responder */
1417 1.1 riastrad
1418 1.1 riastrad /* Ci := HASH(CONSTRUCTION) */
1419 1.1 riastrad /* Hi := HASH(Ci || IDENTIFIER) */
1420 1.1 riastrad wg_init_key_and_hash(ckey, hash);
1421 1.1 riastrad /* Hi := HASH(Hi || Sr^pub) */
1422 1.1 riastrad wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey));
1423 1.1 riastrad
1424 1.1 riastrad WG_DUMP_HASH("hash", hash);
1425 1.1 riastrad
1426 1.1 riastrad /* [N] 2.2: "e" */
1427 1.1 riastrad /* Ei^priv, Ei^pub := DH-GENERATE() */
1428 1.1 riastrad wg_algo_generate_keypair(pubkey, privkey);
1429 1.1 riastrad /* Ci := KDF1(Ci, Ei^pub) */
1430 1.1 riastrad wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey));
1431 1.1 riastrad /* msg.ephemeral := Ei^pub */
1432 1.1 riastrad memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral));
1433 1.1 riastrad /* Hi := HASH(Hi || msg.ephemeral) */
1434 1.1 riastrad wg_algo_hash(hash, pubkey, sizeof(pubkey));
1435 1.1 riastrad
1436 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
1437 1.1 riastrad WG_DUMP_HASH("hash", hash);
1438 1.1 riastrad
1439 1.1 riastrad /* [N] 2.2: "es" */
1440 1.1 riastrad /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */
1441 1.1 riastrad wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey);
1442 1.1 riastrad
1443 1.1 riastrad /* [N] 2.2: "s" */
1444 1.1 riastrad /* msg.static := AEAD(k, 0, Si^pub, Hi) */
1445 1.1 riastrad wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static),
1446 1.1 riastrad cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey),
1447 1.1 riastrad hash, sizeof(hash));
1448 1.1 riastrad /* Hi := HASH(Hi || msg.static) */
1449 1.1 riastrad wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static));
1450 1.1 riastrad
1451 1.1 riastrad WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static);
1452 1.1 riastrad
1453 1.1 riastrad /* [N] 2.2: "ss" */
1454 1.1 riastrad /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */
1455 1.1 riastrad wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey);
1456 1.1 riastrad
1457 1.1 riastrad /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */
1458 1.1 riastrad wg_timestamp_t timestamp;
1459 1.1 riastrad wg_algo_tai64n(timestamp);
1460 1.1 riastrad wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp),
1461 1.1 riastrad cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash));
1462 1.1 riastrad /* Hi := HASH(Hi || msg.timestamp) */
1463 1.1 riastrad wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp));
1464 1.1 riastrad
1465 1.1 riastrad /* [W] 5.4.4 Cookie MACs */
1466 1.1 riastrad wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1),
1467 1.1 riastrad wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey),
1468 1.17 riastrad (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1));
1469 1.1 riastrad /* Need mac1 to decrypt a cookie from a cookie message */
1470 1.1 riastrad memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1,
1471 1.1 riastrad sizeof(wgp->wgp_last_sent_mac1));
1472 1.1 riastrad wgp->wgp_last_sent_mac1_valid = true;
1473 1.1 riastrad
1474 1.1 riastrad if (wgp->wgp_latest_cookie_time == 0 ||
1475 1.1 riastrad (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME)
1476 1.1 riastrad memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2));
1477 1.1 riastrad else {
1478 1.1 riastrad wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2),
1479 1.1 riastrad wgp->wgp_latest_cookie, WG_COOKIE_LEN,
1480 1.17 riastrad (const uint8_t *)wgmi,
1481 1.17 riastrad offsetof(struct wg_msg_init, wgmi_mac2),
1482 1.1 riastrad NULL, 0);
1483 1.1 riastrad }
1484 1.1 riastrad
1485 1.1 riastrad memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
1486 1.1 riastrad memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
1487 1.1 riastrad memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
1488 1.1 riastrad memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
1489 1.49 riastrad WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index);
1490 1.1 riastrad }
1491 1.1 riastrad
1492 1.63 riastrad static void __noinline
1493 1.1 riastrad wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi,
1494 1.1 riastrad const struct sockaddr *src)
1495 1.1 riastrad {
1496 1.1 riastrad uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
1497 1.1 riastrad uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
1498 1.1 riastrad uint8_t cipher_key[WG_CIPHER_KEY_LEN];
1499 1.1 riastrad uint8_t peer_pubkey[WG_STATIC_KEY_LEN];
1500 1.1 riastrad struct wg_peer *wgp;
1501 1.1 riastrad struct wg_session *wgs;
1502 1.1 riastrad int error, ret;
1503 1.1 riastrad struct psref psref_peer;
1504 1.1 riastrad uint8_t mac1[WG_MAC_LEN];
1505 1.1 riastrad
1506 1.1 riastrad WG_TRACE("init msg received");
1507 1.1 riastrad
1508 1.44 riastrad wg_algo_mac_mac1(mac1, sizeof(mac1),
1509 1.44 riastrad wg->wg_pubkey, sizeof(wg->wg_pubkey),
1510 1.44 riastrad (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1));
1511 1.44 riastrad
1512 1.44 riastrad /*
1513 1.44 riastrad * [W] 5.3: Denial of Service Mitigation & Cookies
1514 1.44 riastrad * "the responder, ..., must always reject messages with an invalid
1515 1.44 riastrad * msg.mac1"
1516 1.44 riastrad */
1517 1.44 riastrad if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) {
1518 1.44 riastrad WG_DLOG("mac1 is invalid\n");
1519 1.44 riastrad return;
1520 1.44 riastrad }
1521 1.44 riastrad
1522 1.1 riastrad /*
1523 1.1 riastrad * [W] 5.4.2: First Message: Initiator to Responder
1524 1.1 riastrad * "When the responder receives this message, it does the same
1525 1.1 riastrad * operations so that its final state variables are identical,
1526 1.1 riastrad * replacing the operands of the DH function to produce equivalent
1527 1.1 riastrad * values."
1528 1.1 riastrad * Note that the following comments of operations are just copies of
1529 1.1 riastrad * the initiator's ones.
1530 1.1 riastrad */
1531 1.1 riastrad
1532 1.1 riastrad /* Ci := HASH(CONSTRUCTION) */
1533 1.1 riastrad /* Hi := HASH(Ci || IDENTIFIER) */
1534 1.1 riastrad wg_init_key_and_hash(ckey, hash);
1535 1.1 riastrad /* Hi := HASH(Hi || Sr^pub) */
1536 1.1 riastrad wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey));
1537 1.1 riastrad
1538 1.1 riastrad /* [N] 2.2: "e" */
1539 1.1 riastrad /* Ci := KDF1(Ci, Ei^pub) */
1540 1.1 riastrad wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral,
1541 1.1 riastrad sizeof(wgmi->wgmi_ephemeral));
1542 1.1 riastrad /* Hi := HASH(Hi || msg.ephemeral) */
1543 1.1 riastrad wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral));
1544 1.1 riastrad
1545 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
1546 1.1 riastrad
1547 1.1 riastrad /* [N] 2.2: "es" */
1548 1.1 riastrad /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */
1549 1.1 riastrad wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral);
1550 1.1 riastrad
1551 1.1 riastrad WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static);
1552 1.1 riastrad
1553 1.1 riastrad /* [N] 2.2: "s" */
1554 1.1 riastrad /* msg.static := AEAD(k, 0, Si^pub, Hi) */
1555 1.1 riastrad error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0,
1556 1.1 riastrad wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash));
1557 1.1 riastrad if (error != 0) {
1558 1.1 riastrad WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG,
1559 1.76 jakllsch "%s: wg_algo_aead_dec for secret key failed\n",
1560 1.76 jakllsch if_name(&wg->wg_if));
1561 1.1 riastrad return;
1562 1.1 riastrad }
1563 1.1 riastrad /* Hi := HASH(Hi || msg.static) */
1564 1.1 riastrad wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static));
1565 1.1 riastrad
1566 1.1 riastrad wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer);
1567 1.1 riastrad if (wgp == NULL) {
1568 1.1 riastrad WG_DLOG("peer not found\n");
1569 1.1 riastrad return;
1570 1.1 riastrad }
1571 1.1 riastrad
1572 1.49 riastrad /*
1573 1.49 riastrad * Lock the peer to serialize access to cookie state.
1574 1.49 riastrad *
1575 1.49 riastrad * XXX Can we safely avoid holding the lock across DH? Take it
1576 1.49 riastrad * just to verify mac2 and then unlock/DH/lock?
1577 1.49 riastrad */
1578 1.49 riastrad mutex_enter(wgp->wgp_lock);
1579 1.49 riastrad
1580 1.1 riastrad if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) {
1581 1.1 riastrad WG_TRACE("under load");
1582 1.1 riastrad /*
1583 1.1 riastrad * [W] 5.3: Denial of Service Mitigation & Cookies
1584 1.1 riastrad * "the responder, ..., and when under load may reject messages
1585 1.1 riastrad * with an invalid msg.mac2. If the responder receives a
1586 1.1 riastrad * message with a valid msg.mac1 yet with an invalid msg.mac2,
1587 1.1 riastrad * and is under load, it may respond with a cookie reply
1588 1.1 riastrad * message"
1589 1.1 riastrad */
1590 1.1 riastrad uint8_t zero[WG_MAC_LEN] = {0};
1591 1.13 riastrad if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) {
1592 1.1 riastrad WG_TRACE("sending a cookie message: no cookie included");
1593 1.108 riastrad wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender,
1594 1.1 riastrad wgmi->wgmi_mac1, src);
1595 1.49 riastrad goto out;
1596 1.1 riastrad }
1597 1.1 riastrad if (!wgp->wgp_last_sent_cookie_valid) {
1598 1.1 riastrad WG_TRACE("sending a cookie message: no cookie sent ever");
1599 1.108 riastrad wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender,
1600 1.1 riastrad wgmi->wgmi_mac1, src);
1601 1.49 riastrad goto out;
1602 1.1 riastrad }
1603 1.1 riastrad uint8_t mac2[WG_MAC_LEN];
1604 1.1 riastrad wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie,
1605 1.1 riastrad WG_COOKIE_LEN, (const uint8_t *)wgmi,
1606 1.1 riastrad offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0);
1607 1.13 riastrad if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) {
1608 1.1 riastrad WG_DLOG("mac2 is invalid\n");
1609 1.49 riastrad goto out;
1610 1.1 riastrad }
1611 1.1 riastrad WG_TRACE("under load, but continue to sending");
1612 1.1 riastrad }
1613 1.1 riastrad
1614 1.46 riastrad /* [N] 2.2: "ss" */
1615 1.46 riastrad /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */
1616 1.46 riastrad wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey);
1617 1.46 riastrad
1618 1.46 riastrad /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */
1619 1.46 riastrad wg_timestamp_t timestamp;
1620 1.46 riastrad error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0,
1621 1.46 riastrad wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp),
1622 1.46 riastrad hash, sizeof(hash));
1623 1.46 riastrad if (error != 0) {
1624 1.46 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
1625 1.76 jakllsch "%s: peer %s: wg_algo_aead_dec for timestamp failed\n",
1626 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name);
1627 1.49 riastrad goto out;
1628 1.46 riastrad }
1629 1.46 riastrad /* Hi := HASH(Hi || msg.timestamp) */
1630 1.46 riastrad wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp));
1631 1.46 riastrad
1632 1.1 riastrad /*
1633 1.14 riastrad * [W] 5.1 "The responder keeps track of the greatest timestamp
1634 1.14 riastrad * received per peer and discards packets containing
1635 1.14 riastrad * timestamps less than or equal to it."
1636 1.1 riastrad */
1637 1.1 riastrad ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init,
1638 1.1 riastrad sizeof(timestamp));
1639 1.1 riastrad if (ret <= 0) {
1640 1.1 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
1641 1.76 jakllsch "%s: peer %s: invalid init msg: timestamp is old\n",
1642 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name);
1643 1.1 riastrad goto out;
1644 1.1 riastrad }
1645 1.1 riastrad memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp));
1646 1.1 riastrad
1647 1.49 riastrad /*
1648 1.49 riastrad * Message is good -- we're committing to handle it now, unless
1649 1.49 riastrad * we were already initiating a session.
1650 1.49 riastrad */
1651 1.49 riastrad wgs = wgp->wgp_session_unstable;
1652 1.49 riastrad switch (wgs->wgs_state) {
1653 1.49 riastrad case WGS_STATE_UNKNOWN: /* new session initiated by peer */
1654 1.49 riastrad break;
1655 1.49 riastrad case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */
1656 1.94 riastrad /* XXX Who wins if both sides send INIT? */
1657 1.49 riastrad WG_TRACE("Session already initializing, ignoring the message");
1658 1.49 riastrad goto out;
1659 1.49 riastrad case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */
1660 1.49 riastrad WG_TRACE("Session already initializing, destroying old states");
1661 1.94 riastrad /*
1662 1.94 riastrad * XXX Avoid this -- just resend our response -- if the
1663 1.94 riastrad * INIT message is identical to the previous one.
1664 1.94 riastrad */
1665 1.94 riastrad wg_put_session_index(wg, wgs);
1666 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1667 1.94 riastrad wgs->wgs_state);
1668 1.49 riastrad break;
1669 1.49 riastrad case WGS_STATE_ESTABLISHED: /* can't happen */
1670 1.49 riastrad panic("unstable session can't be established");
1671 1.49 riastrad case WGS_STATE_DESTROYING: /* rekey initiated by peer */
1672 1.49 riastrad WG_TRACE("Session destroying, but force to clear");
1673 1.94 riastrad wg_put_session_index(wg, wgs);
1674 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1675 1.94 riastrad wgs->wgs_state);
1676 1.49 riastrad break;
1677 1.49 riastrad default:
1678 1.49 riastrad panic("invalid session state: %d", wgs->wgs_state);
1679 1.49 riastrad }
1680 1.94 riastrad
1681 1.94 riastrad /*
1682 1.94 riastrad * Assign a fresh session index.
1683 1.94 riastrad */
1684 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1685 1.94 riastrad wgs->wgs_state);
1686 1.94 riastrad wg_get_session_index(wg, wgs);
1687 1.49 riastrad
1688 1.1 riastrad memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
1689 1.1 riastrad memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
1690 1.1 riastrad memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral,
1691 1.1 riastrad sizeof(wgmi->wgmi_ephemeral));
1692 1.1 riastrad
1693 1.1 riastrad wg_update_endpoint_if_necessary(wgp, src);
1694 1.1 riastrad
1695 1.94 riastrad /*
1696 1.100 riastrad * Count the time of the INIT message as the time of
1697 1.100 riastrad * establishment -- this is used to decide when to erase keys,
1698 1.100 riastrad * and we want to start counting as soon as we have generated
1699 1.100 riastrad * keys.
1700 1.104 riastrad *
1701 1.104 riastrad * No need for atomic store because the session can't be used
1702 1.104 riastrad * in the rx or tx paths yet -- not until we transition to
1703 1.104 riastrad * INTI_PASSIVE.
1704 1.100 riastrad */
1705 1.104 riastrad wgs->wgs_time_established = time_uptime32;
1706 1.100 riastrad wg_schedule_session_dtor_timer(wgp);
1707 1.100 riastrad
1708 1.100 riastrad /*
1709 1.94 riastrad * Respond to the initiator with our ephemeral public key.
1710 1.94 riastrad */
1711 1.108 riastrad wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi);
1712 1.1 riastrad
1713 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]:"
1714 1.94 riastrad " calculate keys as responder\n",
1715 1.94 riastrad wgs->wgs_local_index, wgs->wgs_remote_index);
1716 1.1 riastrad wg_calculate_keys(wgs, false);
1717 1.1 riastrad wg_clear_states(wgs);
1718 1.1 riastrad
1719 1.94 riastrad /*
1720 1.94 riastrad * Session is ready to receive data now that we have received
1721 1.94 riastrad * the peer initiator's ephemeral key pair, generated our
1722 1.94 riastrad * responder's ephemeral key pair, and derived a session key.
1723 1.94 riastrad *
1724 1.94 riastrad * Transition from UNKNOWN to INIT_PASSIVE to publish it to the
1725 1.94 riastrad * data rx path, wg_handle_msg_data, where the
1726 1.94 riastrad * atomic_load_acquire matching this atomic_store_release
1727 1.94 riastrad * happens.
1728 1.94 riastrad *
1729 1.94 riastrad * (Session is not, however, ready to send data until the peer
1730 1.94 riastrad * has acknowledged our response by sending its first data
1731 1.94 riastrad * packet. So don't swap the sessions yet.)
1732 1.94 riastrad */
1733 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_INIT_PASSIVE\n",
1734 1.94 riastrad wgs->wgs_local_index, wgs->wgs_remote_index);
1735 1.94 riastrad atomic_store_release(&wgs->wgs_state, WGS_STATE_INIT_PASSIVE);
1736 1.94 riastrad WG_TRACE("WGS_STATE_INIT_PASSIVE");
1737 1.94 riastrad
1738 1.1 riastrad out:
1739 1.49 riastrad mutex_exit(wgp->wgp_lock);
1740 1.1 riastrad wg_put_peer(wgp, &psref_peer);
1741 1.1 riastrad }
1742 1.1 riastrad
1743 1.1 riastrad static struct socket *
1744 1.55 riastrad wg_get_so_by_af(struct wg_softc *wg, const int af)
1745 1.1 riastrad {
1746 1.1 riastrad
1747 1.62 riastrad switch (af) {
1748 1.62 riastrad #ifdef INET
1749 1.62 riastrad case AF_INET:
1750 1.62 riastrad return wg->wg_so4;
1751 1.62 riastrad #endif
1752 1.62 riastrad #ifdef INET6
1753 1.62 riastrad case AF_INET6:
1754 1.62 riastrad return wg->wg_so6;
1755 1.62 riastrad #endif
1756 1.62 riastrad default:
1757 1.62 riastrad panic("wg: no such af: %d", af);
1758 1.62 riastrad }
1759 1.1 riastrad }
1760 1.1 riastrad
1761 1.1 riastrad static struct socket *
1762 1.47 riastrad wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa)
1763 1.1 riastrad {
1764 1.1 riastrad
1765 1.55 riastrad return wg_get_so_by_af(wgp->wgp_sc, wgsa_family(wgsa));
1766 1.1 riastrad }
1767 1.1 riastrad
1768 1.1 riastrad static struct wg_sockaddr *
1769 1.1 riastrad wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref)
1770 1.1 riastrad {
1771 1.1 riastrad struct wg_sockaddr *wgsa;
1772 1.1 riastrad int s;
1773 1.1 riastrad
1774 1.1 riastrad s = pserialize_read_enter();
1775 1.47 riastrad wgsa = atomic_load_consume(&wgp->wgp_endpoint);
1776 1.1 riastrad psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class);
1777 1.1 riastrad pserialize_read_exit(s);
1778 1.1 riastrad
1779 1.1 riastrad return wgsa;
1780 1.1 riastrad }
1781 1.1 riastrad
1782 1.1 riastrad static void
1783 1.1 riastrad wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref)
1784 1.1 riastrad {
1785 1.1 riastrad
1786 1.1 riastrad psref_release(psref, &wgsa->wgsa_psref, wg_psref_class);
1787 1.1 riastrad }
1788 1.1 riastrad
1789 1.1 riastrad static int
1790 1.1 riastrad wg_send_so(struct wg_peer *wgp, struct mbuf *m)
1791 1.1 riastrad {
1792 1.1 riastrad int error;
1793 1.1 riastrad struct socket *so;
1794 1.1 riastrad struct psref psref;
1795 1.1 riastrad struct wg_sockaddr *wgsa;
1796 1.1 riastrad
1797 1.1 riastrad wgsa = wg_get_endpoint_sa(wgp, &psref);
1798 1.47 riastrad so = wg_get_so_by_peer(wgp, wgsa);
1799 1.1 riastrad error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp);
1800 1.1 riastrad wg_put_sa(wgp, wgsa, &psref);
1801 1.1 riastrad
1802 1.1 riastrad return error;
1803 1.1 riastrad }
1804 1.1 riastrad
1805 1.108 riastrad static void
1806 1.1 riastrad wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp)
1807 1.1 riastrad {
1808 1.1 riastrad int error;
1809 1.1 riastrad struct mbuf *m;
1810 1.1 riastrad struct wg_msg_init *wgmi;
1811 1.1 riastrad struct wg_session *wgs;
1812 1.1 riastrad
1813 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1814 1.49 riastrad
1815 1.49 riastrad wgs = wgp->wgp_session_unstable;
1816 1.49 riastrad /* XXX pull dispatch out into wg_task_send_init_message */
1817 1.49 riastrad switch (wgs->wgs_state) {
1818 1.49 riastrad case WGS_STATE_UNKNOWN: /* new session initiated by us */
1819 1.49 riastrad break;
1820 1.49 riastrad case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */
1821 1.49 riastrad WG_TRACE("Session already initializing, skip starting new one");
1822 1.108 riastrad return;
1823 1.49 riastrad case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */
1824 1.94 riastrad WG_TRACE("Session already initializing, waiting for peer");
1825 1.108 riastrad return;
1826 1.49 riastrad case WGS_STATE_ESTABLISHED: /* can't happen */
1827 1.49 riastrad panic("unstable session can't be established");
1828 1.49 riastrad case WGS_STATE_DESTROYING: /* rekey initiated by us too early */
1829 1.1 riastrad WG_TRACE("Session destroying");
1830 1.94 riastrad wg_put_session_index(wg, wgs);
1831 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1832 1.94 riastrad wgs->wgs_state);
1833 1.94 riastrad break;
1834 1.1 riastrad }
1835 1.94 riastrad
1836 1.94 riastrad /*
1837 1.94 riastrad * Assign a fresh session index.
1838 1.94 riastrad */
1839 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1840 1.94 riastrad wgs->wgs_state);
1841 1.94 riastrad wg_get_session_index(wg, wgs);
1842 1.94 riastrad
1843 1.94 riastrad /*
1844 1.94 riastrad * We have initiated a session. Transition to INIT_ACTIVE.
1845 1.94 riastrad * This doesn't publish it for use in the data rx path,
1846 1.94 riastrad * wg_handle_msg_data, or in the data tx path, wg_output -- we
1847 1.94 riastrad * have to wait for the peer to respond with their ephemeral
1848 1.94 riastrad * public key before we can derive a session key for tx/rx.
1849 1.94 riastrad * Hence only atomic_store_relaxed.
1850 1.94 riastrad */
1851 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=(unknown)] -> WGS_STATE_INIT_ACTIVE\n",
1852 1.94 riastrad wgs->wgs_local_index);
1853 1.94 riastrad atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_INIT_ACTIVE);
1854 1.1 riastrad
1855 1.1 riastrad m = m_gethdr(M_WAIT, MT_DATA);
1856 1.69 hannken if (sizeof(*wgmi) > MHLEN) {
1857 1.69 hannken m_clget(m, M_WAIT);
1858 1.69 hannken CTASSERT(sizeof(*wgmi) <= MCLBYTES);
1859 1.69 hannken }
1860 1.1 riastrad m->m_pkthdr.len = m->m_len = sizeof(*wgmi);
1861 1.1 riastrad wgmi = mtod(m, struct wg_msg_init *);
1862 1.1 riastrad wg_fill_msg_init(wg, wgp, wgs, wgmi);
1863 1.1 riastrad
1864 1.108 riastrad error = wg->wg_ops->send_hs_msg(wgp, m); /* consumes m */
1865 1.108 riastrad if (error) {
1866 1.108 riastrad /*
1867 1.108 riastrad * Sending out an initiation packet failed; give up on
1868 1.108 riastrad * this session and toss packet waiting for it if any.
1869 1.108 riastrad *
1870 1.108 riastrad * XXX Why don't we just let the periodic handshake
1871 1.108 riastrad * retry logic work in this case?
1872 1.108 riastrad */
1873 1.108 riastrad WG_DLOG("send_hs_msg failed, error=%d\n", error);
1874 1.49 riastrad wg_put_session_index(wg, wgs);
1875 1.79 rin m = atomic_swap_ptr(&wgp->wgp_pending, NULL);
1876 1.79 rin m_freem(m);
1877 1.108 riastrad return;
1878 1.1 riastrad }
1879 1.1 riastrad
1880 1.108 riastrad WG_TRACE("init msg sent");
1881 1.108 riastrad if (wgp->wgp_handshake_start_time == 0)
1882 1.108 riastrad wgp->wgp_handshake_start_time = time_uptime;
1883 1.108 riastrad callout_schedule(&wgp->wgp_handshake_timeout_timer,
1884 1.108 riastrad MIN(wg_rekey_timeout, (unsigned)(INT_MAX / hz)) * hz);
1885 1.1 riastrad }
1886 1.1 riastrad
1887 1.1 riastrad static void
1888 1.1 riastrad wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp,
1889 1.49 riastrad struct wg_session *wgs, struct wg_msg_resp *wgmr,
1890 1.49 riastrad const struct wg_msg_init *wgmi)
1891 1.1 riastrad {
1892 1.1 riastrad uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
1893 1.1 riastrad uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */
1894 1.1 riastrad uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
1895 1.1 riastrad uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
1896 1.1 riastrad uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
1897 1.1 riastrad
1898 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1899 1.49 riastrad KASSERT(wgs == wgp->wgp_session_unstable);
1900 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
1901 1.94 riastrad wgs->wgs_state);
1902 1.49 riastrad
1903 1.1 riastrad memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash));
1904 1.1 riastrad memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey));
1905 1.1 riastrad
1906 1.39 riastrad wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP);
1907 1.49 riastrad wgmr->wgmr_sender = wgs->wgs_local_index;
1908 1.1 riastrad wgmr->wgmr_receiver = wgmi->wgmi_sender;
1909 1.1 riastrad
1910 1.1 riastrad /* [W] 5.4.3 Second Message: Responder to Initiator */
1911 1.1 riastrad
1912 1.1 riastrad /* [N] 2.2: "e" */
1913 1.1 riastrad /* Er^priv, Er^pub := DH-GENERATE() */
1914 1.1 riastrad wg_algo_generate_keypair(pubkey, privkey);
1915 1.1 riastrad /* Cr := KDF1(Cr, Er^pub) */
1916 1.1 riastrad wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey));
1917 1.1 riastrad /* msg.ephemeral := Er^pub */
1918 1.1 riastrad memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral));
1919 1.1 riastrad /* Hr := HASH(Hr || msg.ephemeral) */
1920 1.1 riastrad wg_algo_hash(hash, pubkey, sizeof(pubkey));
1921 1.1 riastrad
1922 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
1923 1.1 riastrad WG_DUMP_HASH("hash", hash);
1924 1.1 riastrad
1925 1.1 riastrad /* [N] 2.2: "ee" */
1926 1.1 riastrad /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */
1927 1.1 riastrad wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer);
1928 1.1 riastrad
1929 1.1 riastrad /* [N] 2.2: "se" */
1930 1.1 riastrad /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */
1931 1.1 riastrad wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey);
1932 1.1 riastrad
1933 1.1 riastrad /* [N] 9.2: "psk" */
1934 1.1 riastrad {
1935 1.1 riastrad uint8_t kdfout[WG_KDF_OUTPUT_LEN];
1936 1.1 riastrad /* Cr, r, k := KDF3(Cr, Q) */
1937 1.1 riastrad wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk,
1938 1.1 riastrad sizeof(wgp->wgp_psk));
1939 1.1 riastrad /* Hr := HASH(Hr || r) */
1940 1.1 riastrad wg_algo_hash(hash, kdfout, sizeof(kdfout));
1941 1.1 riastrad }
1942 1.1 riastrad
1943 1.1 riastrad /* msg.empty := AEAD(k, 0, e, Hr) */
1944 1.14 riastrad wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty),
1945 1.14 riastrad cipher_key, 0, NULL, 0, hash, sizeof(hash));
1946 1.1 riastrad /* Hr := HASH(Hr || msg.empty) */
1947 1.1 riastrad wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty));
1948 1.1 riastrad
1949 1.1 riastrad WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty);
1950 1.1 riastrad
1951 1.1 riastrad /* [W] 5.4.4: Cookie MACs */
1952 1.1 riastrad /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */
1953 1.1 riastrad wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1),
1954 1.1 riastrad wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey),
1955 1.17 riastrad (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1));
1956 1.1 riastrad /* Need mac1 to decrypt a cookie from a cookie message */
1957 1.1 riastrad memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1,
1958 1.1 riastrad sizeof(wgp->wgp_last_sent_mac1));
1959 1.1 riastrad wgp->wgp_last_sent_mac1_valid = true;
1960 1.1 riastrad
1961 1.1 riastrad if (wgp->wgp_latest_cookie_time == 0 ||
1962 1.1 riastrad (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME)
1963 1.1 riastrad /* msg.mac2 := 0^16 */
1964 1.1 riastrad memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2));
1965 1.1 riastrad else {
1966 1.1 riastrad /* msg.mac2 := MAC(Lm, msg_b) */
1967 1.1 riastrad wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2),
1968 1.1 riastrad wgp->wgp_latest_cookie, WG_COOKIE_LEN,
1969 1.17 riastrad (const uint8_t *)wgmr,
1970 1.17 riastrad offsetof(struct wg_msg_resp, wgmr_mac2),
1971 1.1 riastrad NULL, 0);
1972 1.1 riastrad }
1973 1.1 riastrad
1974 1.1 riastrad memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
1975 1.1 riastrad memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
1976 1.1 riastrad memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
1977 1.1 riastrad memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
1978 1.49 riastrad wgs->wgs_remote_index = wgmi->wgmi_sender;
1979 1.49 riastrad WG_DLOG("sender=%x\n", wgs->wgs_local_index);
1980 1.49 riastrad WG_DLOG("receiver=%x\n", wgs->wgs_remote_index);
1981 1.1 riastrad }
1982 1.1 riastrad
1983 1.1 riastrad static void
1984 1.1 riastrad wg_swap_sessions(struct wg_peer *wgp)
1985 1.1 riastrad {
1986 1.49 riastrad struct wg_session *wgs, *wgs_prev;
1987 1.1 riastrad
1988 1.1 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
1989 1.1 riastrad
1990 1.49 riastrad wgs = wgp->wgp_session_unstable;
1991 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_ESTABLISHED, "state=%d",
1992 1.94 riastrad wgs->wgs_state);
1993 1.49 riastrad
1994 1.49 riastrad wgs_prev = wgp->wgp_session_stable;
1995 1.94 riastrad KASSERTMSG((wgs_prev->wgs_state == WGS_STATE_ESTABLISHED ||
1996 1.94 riastrad wgs_prev->wgs_state == WGS_STATE_UNKNOWN),
1997 1.94 riastrad "state=%d", wgs_prev->wgs_state);
1998 1.49 riastrad atomic_store_release(&wgp->wgp_session_stable, wgs);
1999 1.49 riastrad wgp->wgp_session_unstable = wgs_prev;
2000 1.1 riastrad }
2001 1.1 riastrad
2002 1.63 riastrad static void __noinline
2003 1.1 riastrad wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr,
2004 1.1 riastrad const struct sockaddr *src)
2005 1.1 riastrad {
2006 1.1 riastrad uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
2007 1.1 riastrad uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */
2008 1.1 riastrad uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
2009 1.1 riastrad struct wg_peer *wgp;
2010 1.1 riastrad struct wg_session *wgs;
2011 1.1 riastrad struct psref psref;
2012 1.1 riastrad int error;
2013 1.1 riastrad uint8_t mac1[WG_MAC_LEN];
2014 1.1 riastrad struct wg_session *wgs_prev;
2015 1.54 riastrad struct mbuf *m;
2016 1.1 riastrad
2017 1.1 riastrad wg_algo_mac_mac1(mac1, sizeof(mac1),
2018 1.1 riastrad wg->wg_pubkey, sizeof(wg->wg_pubkey),
2019 1.1 riastrad (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1));
2020 1.1 riastrad
2021 1.1 riastrad /*
2022 1.1 riastrad * [W] 5.3: Denial of Service Mitigation & Cookies
2023 1.1 riastrad * "the responder, ..., must always reject messages with an invalid
2024 1.1 riastrad * msg.mac1"
2025 1.1 riastrad */
2026 1.13 riastrad if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) {
2027 1.1 riastrad WG_DLOG("mac1 is invalid\n");
2028 1.44 riastrad return;
2029 1.44 riastrad }
2030 1.44 riastrad
2031 1.44 riastrad WG_TRACE("resp msg received");
2032 1.44 riastrad wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref);
2033 1.44 riastrad if (wgs == NULL) {
2034 1.44 riastrad WG_TRACE("No session found");
2035 1.44 riastrad return;
2036 1.1 riastrad }
2037 1.1 riastrad
2038 1.44 riastrad wgp = wgs->wgs_peer;
2039 1.44 riastrad
2040 1.49 riastrad mutex_enter(wgp->wgp_lock);
2041 1.49 riastrad
2042 1.49 riastrad /* If we weren't waiting for a handshake response, drop it. */
2043 1.49 riastrad if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) {
2044 1.49 riastrad WG_TRACE("peer sent spurious handshake response, ignoring");
2045 1.49 riastrad goto out;
2046 1.49 riastrad }
2047 1.49 riastrad
2048 1.1 riastrad if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) {
2049 1.1 riastrad WG_TRACE("under load");
2050 1.1 riastrad /*
2051 1.1 riastrad * [W] 5.3: Denial of Service Mitigation & Cookies
2052 1.1 riastrad * "the responder, ..., and when under load may reject messages
2053 1.1 riastrad * with an invalid msg.mac2. If the responder receives a
2054 1.1 riastrad * message with a valid msg.mac1 yet with an invalid msg.mac2,
2055 1.1 riastrad * and is under load, it may respond with a cookie reply
2056 1.1 riastrad * message"
2057 1.1 riastrad */
2058 1.1 riastrad uint8_t zero[WG_MAC_LEN] = {0};
2059 1.13 riastrad if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) {
2060 1.1 riastrad WG_TRACE("sending a cookie message: no cookie included");
2061 1.108 riastrad wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender,
2062 1.1 riastrad wgmr->wgmr_mac1, src);
2063 1.1 riastrad goto out;
2064 1.1 riastrad }
2065 1.1 riastrad if (!wgp->wgp_last_sent_cookie_valid) {
2066 1.1 riastrad WG_TRACE("sending a cookie message: no cookie sent ever");
2067 1.108 riastrad wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender,
2068 1.1 riastrad wgmr->wgmr_mac1, src);
2069 1.1 riastrad goto out;
2070 1.1 riastrad }
2071 1.1 riastrad uint8_t mac2[WG_MAC_LEN];
2072 1.1 riastrad wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie,
2073 1.1 riastrad WG_COOKIE_LEN, (const uint8_t *)wgmr,
2074 1.1 riastrad offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0);
2075 1.13 riastrad if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) {
2076 1.1 riastrad WG_DLOG("mac2 is invalid\n");
2077 1.1 riastrad goto out;
2078 1.1 riastrad }
2079 1.1 riastrad WG_TRACE("under load, but continue to sending");
2080 1.1 riastrad }
2081 1.1 riastrad
2082 1.1 riastrad memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash));
2083 1.1 riastrad memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey));
2084 1.1 riastrad
2085 1.1 riastrad /*
2086 1.1 riastrad * [W] 5.4.3 Second Message: Responder to Initiator
2087 1.1 riastrad * "When the initiator receives this message, it does the same
2088 1.1 riastrad * operations so that its final state variables are identical,
2089 1.1 riastrad * replacing the operands of the DH function to produce equivalent
2090 1.1 riastrad * values."
2091 1.1 riastrad * Note that the following comments of operations are just copies of
2092 1.1 riastrad * the initiator's ones.
2093 1.1 riastrad */
2094 1.1 riastrad
2095 1.1 riastrad /* [N] 2.2: "e" */
2096 1.1 riastrad /* Cr := KDF1(Cr, Er^pub) */
2097 1.1 riastrad wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral,
2098 1.1 riastrad sizeof(wgmr->wgmr_ephemeral));
2099 1.1 riastrad /* Hr := HASH(Hr || msg.ephemeral) */
2100 1.1 riastrad wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral));
2101 1.1 riastrad
2102 1.1 riastrad WG_DUMP_HASH("ckey", ckey);
2103 1.1 riastrad WG_DUMP_HASH("hash", hash);
2104 1.1 riastrad
2105 1.1 riastrad /* [N] 2.2: "ee" */
2106 1.1 riastrad /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */
2107 1.1 riastrad wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv,
2108 1.1 riastrad wgmr->wgmr_ephemeral);
2109 1.1 riastrad
2110 1.1 riastrad /* [N] 2.2: "se" */
2111 1.1 riastrad /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */
2112 1.1 riastrad wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral);
2113 1.1 riastrad
2114 1.1 riastrad /* [N] 9.2: "psk" */
2115 1.1 riastrad {
2116 1.1 riastrad uint8_t kdfout[WG_KDF_OUTPUT_LEN];
2117 1.1 riastrad /* Cr, r, k := KDF3(Cr, Q) */
2118 1.1 riastrad wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk,
2119 1.1 riastrad sizeof(wgp->wgp_psk));
2120 1.1 riastrad /* Hr := HASH(Hr || r) */
2121 1.1 riastrad wg_algo_hash(hash, kdfout, sizeof(kdfout));
2122 1.1 riastrad }
2123 1.1 riastrad
2124 1.1 riastrad {
2125 1.1 riastrad uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */
2126 1.1 riastrad /* msg.empty := AEAD(k, 0, e, Hr) */
2127 1.1 riastrad error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty,
2128 1.1 riastrad sizeof(wgmr->wgmr_empty), hash, sizeof(hash));
2129 1.1 riastrad WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty);
2130 1.1 riastrad if (error != 0) {
2131 1.1 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2132 1.76 jakllsch "%s: peer %s: wg_algo_aead_dec for empty message failed\n",
2133 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name);
2134 1.1 riastrad goto out;
2135 1.1 riastrad }
2136 1.1 riastrad /* Hr := HASH(Hr || msg.empty) */
2137 1.1 riastrad wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty));
2138 1.1 riastrad }
2139 1.1 riastrad
2140 1.1 riastrad memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash));
2141 1.1 riastrad memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key));
2142 1.49 riastrad wgs->wgs_remote_index = wgmr->wgmr_sender;
2143 1.49 riastrad WG_DLOG("receiver=%x\n", wgs->wgs_remote_index);
2144 1.1 riastrad
2145 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_INIT_ACTIVE, "state=%d",
2146 1.94 riastrad wgs->wgs_state);
2147 1.104 riastrad wgs->wgs_time_established = time_uptime32;
2148 1.100 riastrad wg_schedule_session_dtor_timer(wgp);
2149 1.1 riastrad wgs->wgs_time_last_data_sent = 0;
2150 1.1 riastrad wgs->wgs_is_initiator = true;
2151 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]:"
2152 1.94 riastrad " calculate keys as initiator\n",
2153 1.94 riastrad wgs->wgs_local_index, wgs->wgs_remote_index);
2154 1.1 riastrad wg_calculate_keys(wgs, true);
2155 1.1 riastrad wg_clear_states(wgs);
2156 1.94 riastrad
2157 1.94 riastrad /*
2158 1.94 riastrad * Session is ready to receive data now that we have received
2159 1.94 riastrad * the responder's response.
2160 1.94 riastrad *
2161 1.94 riastrad * Transition from INIT_ACTIVE to ESTABLISHED to publish it to
2162 1.94 riastrad * the data rx path, wg_handle_msg_data.
2163 1.94 riastrad */
2164 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32" -> WGS_STATE_ESTABLISHED\n",
2165 1.94 riastrad wgs->wgs_local_index, wgs->wgs_remote_index);
2166 1.94 riastrad atomic_store_release(&wgs->wgs_state, WGS_STATE_ESTABLISHED);
2167 1.1 riastrad WG_TRACE("WGS_STATE_ESTABLISHED");
2168 1.1 riastrad
2169 1.96 riastrad callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
2170 1.18 riastrad
2171 1.94 riastrad /*
2172 1.94 riastrad * Session is ready to send data now that we have received the
2173 1.94 riastrad * responder's response.
2174 1.94 riastrad *
2175 1.94 riastrad * Swap the sessions to publish the new one as the stable
2176 1.94 riastrad * session for the data tx path, wg_output.
2177 1.94 riastrad */
2178 1.1 riastrad wg_swap_sessions(wgp);
2179 1.49 riastrad KASSERT(wgs == wgp->wgp_session_stable);
2180 1.1 riastrad wgs_prev = wgp->wgp_session_unstable;
2181 1.1 riastrad getnanotime(&wgp->wgp_last_handshake_time);
2182 1.1 riastrad wgp->wgp_handshake_start_time = 0;
2183 1.1 riastrad wgp->wgp_last_sent_mac1_valid = false;
2184 1.1 riastrad wgp->wgp_last_sent_cookie_valid = false;
2185 1.1 riastrad
2186 1.1 riastrad wg_update_endpoint_if_necessary(wgp, src);
2187 1.1 riastrad
2188 1.1 riastrad /*
2189 1.54 riastrad * If we had a data packet queued up, send it; otherwise send a
2190 1.54 riastrad * keepalive message -- either way we have to send something
2191 1.54 riastrad * immediately or else the responder will never answer.
2192 1.1 riastrad */
2193 1.54 riastrad if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) {
2194 1.57 riastrad kpreempt_disable();
2195 1.54 riastrad const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m)
2196 1.54 riastrad M_SETCTX(m, wgp);
2197 1.54 riastrad if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) {
2198 1.76 jakllsch WGLOG(LOG_ERR, "%s: pktq full, dropping\n",
2199 1.76 jakllsch if_name(&wg->wg_if));
2200 1.54 riastrad m_freem(m);
2201 1.54 riastrad }
2202 1.57 riastrad kpreempt_enable();
2203 1.54 riastrad } else {
2204 1.54 riastrad wg_send_keepalive_msg(wgp, wgs);
2205 1.54 riastrad }
2206 1.1 riastrad
2207 1.1 riastrad if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
2208 1.94 riastrad /*
2209 1.94 riastrad * Transition ESTABLISHED->DESTROYING. The session
2210 1.94 riastrad * will remain usable for the data rx path to process
2211 1.94 riastrad * packets still in flight to us, but we won't use it
2212 1.94 riastrad * for data tx.
2213 1.94 riastrad */
2214 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]"
2215 1.94 riastrad " -> WGS_STATE_DESTROYING\n",
2216 1.94 riastrad wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index);
2217 1.94 riastrad atomic_store_relaxed(&wgs_prev->wgs_state,
2218 1.94 riastrad WGS_STATE_DESTROYING);
2219 1.49 riastrad } else {
2220 1.49 riastrad KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN,
2221 1.49 riastrad "state=%d", wgs_prev->wgs_state);
2222 1.1 riastrad }
2223 1.1 riastrad
2224 1.1 riastrad out:
2225 1.49 riastrad mutex_exit(wgp->wgp_lock);
2226 1.1 riastrad wg_put_session(wgs, &psref);
2227 1.1 riastrad }
2228 1.1 riastrad
2229 1.108 riastrad static void
2230 1.1 riastrad wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp,
2231 1.49 riastrad struct wg_session *wgs, const struct wg_msg_init *wgmi)
2232 1.1 riastrad {
2233 1.1 riastrad int error;
2234 1.1 riastrad struct mbuf *m;
2235 1.1 riastrad struct wg_msg_resp *wgmr;
2236 1.1 riastrad
2237 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
2238 1.49 riastrad KASSERT(wgs == wgp->wgp_session_unstable);
2239 1.94 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
2240 1.94 riastrad wgs->wgs_state);
2241 1.49 riastrad
2242 1.1 riastrad m = m_gethdr(M_WAIT, MT_DATA);
2243 1.69 hannken if (sizeof(*wgmr) > MHLEN) {
2244 1.69 hannken m_clget(m, M_WAIT);
2245 1.69 hannken CTASSERT(sizeof(*wgmr) <= MCLBYTES);
2246 1.69 hannken }
2247 1.1 riastrad m->m_pkthdr.len = m->m_len = sizeof(*wgmr);
2248 1.1 riastrad wgmr = mtod(m, struct wg_msg_resp *);
2249 1.49 riastrad wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi);
2250 1.1 riastrad
2251 1.108 riastrad error = wg->wg_ops->send_hs_msg(wgp, m); /* consumes m */
2252 1.108 riastrad if (error) {
2253 1.108 riastrad WG_DLOG("send_hs_msg failed, error=%d\n", error);
2254 1.108 riastrad return;
2255 1.108 riastrad }
2256 1.108 riastrad
2257 1.108 riastrad WG_TRACE("resp msg sent");
2258 1.1 riastrad }
2259 1.1 riastrad
2260 1.1 riastrad static struct wg_peer *
2261 1.1 riastrad wg_lookup_peer_by_pubkey(struct wg_softc *wg,
2262 1.114 riastrad const uint8_t pubkey[static WG_STATIC_KEY_LEN], struct psref *psref)
2263 1.1 riastrad {
2264 1.1 riastrad struct wg_peer *wgp;
2265 1.1 riastrad
2266 1.1 riastrad int s = pserialize_read_enter();
2267 1.37 riastrad wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN);
2268 1.1 riastrad if (wgp != NULL)
2269 1.1 riastrad wg_get_peer(wgp, psref);
2270 1.1 riastrad pserialize_read_exit(s);
2271 1.1 riastrad
2272 1.1 riastrad return wgp;
2273 1.1 riastrad }
2274 1.1 riastrad
2275 1.1 riastrad static void
2276 1.1 riastrad wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp,
2277 1.1 riastrad struct wg_msg_cookie *wgmc, const uint32_t sender,
2278 1.114 riastrad const uint8_t mac1[static WG_MAC_LEN], const struct sockaddr *src)
2279 1.1 riastrad {
2280 1.1 riastrad uint8_t cookie[WG_COOKIE_LEN];
2281 1.1 riastrad uint8_t key[WG_HASH_LEN];
2282 1.1 riastrad uint8_t addr[sizeof(struct in6_addr)];
2283 1.1 riastrad size_t addrlen;
2284 1.1 riastrad uint16_t uh_sport; /* be */
2285 1.1 riastrad
2286 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
2287 1.49 riastrad
2288 1.39 riastrad wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE);
2289 1.1 riastrad wgmc->wgmc_receiver = sender;
2290 1.1 riastrad cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt));
2291 1.1 riastrad
2292 1.1 riastrad /*
2293 1.1 riastrad * [W] 5.4.7: Under Load: Cookie Reply Message
2294 1.14 riastrad * "The secret variable, Rm, changes every two minutes to a
2295 1.14 riastrad * random value"
2296 1.1 riastrad */
2297 1.98 riastrad if ((time_uptime - wgp->wgp_last_cookiesecret_time) >
2298 1.98 riastrad WG_COOKIESECRET_TIME) {
2299 1.98 riastrad cprng_strong(kern_cprng, wgp->wgp_cookiesecret,
2300 1.98 riastrad sizeof(wgp->wgp_cookiesecret), 0);
2301 1.98 riastrad wgp->wgp_last_cookiesecret_time = time_uptime;
2302 1.1 riastrad }
2303 1.1 riastrad
2304 1.1 riastrad switch (src->sa_family) {
2305 1.109 riastrad #ifdef INET
2306 1.1 riastrad case AF_INET: {
2307 1.1 riastrad const struct sockaddr_in *sin = satocsin(src);
2308 1.1 riastrad addrlen = sizeof(sin->sin_addr);
2309 1.1 riastrad memcpy(addr, &sin->sin_addr, addrlen);
2310 1.1 riastrad uh_sport = sin->sin_port;
2311 1.1 riastrad break;
2312 1.1 riastrad }
2313 1.109 riastrad #endif
2314 1.1 riastrad #ifdef INET6
2315 1.1 riastrad case AF_INET6: {
2316 1.1 riastrad const struct sockaddr_in6 *sin6 = satocsin6(src);
2317 1.1 riastrad addrlen = sizeof(sin6->sin6_addr);
2318 1.1 riastrad memcpy(addr, &sin6->sin6_addr, addrlen);
2319 1.1 riastrad uh_sport = sin6->sin6_port;
2320 1.1 riastrad break;
2321 1.1 riastrad }
2322 1.1 riastrad #endif
2323 1.1 riastrad default:
2324 1.47 riastrad panic("invalid af=%d", src->sa_family);
2325 1.1 riastrad }
2326 1.1 riastrad
2327 1.1 riastrad wg_algo_mac(cookie, sizeof(cookie),
2328 1.98 riastrad wgp->wgp_cookiesecret, sizeof(wgp->wgp_cookiesecret),
2329 1.17 riastrad addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport));
2330 1.1 riastrad wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey,
2331 1.1 riastrad sizeof(wg->wg_pubkey));
2332 1.1 riastrad wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key,
2333 1.1 riastrad cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt);
2334 1.1 riastrad
2335 1.1 riastrad /* Need to store to calculate mac2 */
2336 1.1 riastrad memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie));
2337 1.1 riastrad wgp->wgp_last_sent_cookie_valid = true;
2338 1.1 riastrad }
2339 1.1 riastrad
2340 1.108 riastrad static void
2341 1.1 riastrad wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp,
2342 1.114 riastrad const uint32_t sender, const uint8_t mac1[static WG_MAC_LEN],
2343 1.1 riastrad const struct sockaddr *src)
2344 1.1 riastrad {
2345 1.1 riastrad int error;
2346 1.1 riastrad struct mbuf *m;
2347 1.1 riastrad struct wg_msg_cookie *wgmc;
2348 1.1 riastrad
2349 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
2350 1.49 riastrad
2351 1.1 riastrad m = m_gethdr(M_WAIT, MT_DATA);
2352 1.69 hannken if (sizeof(*wgmc) > MHLEN) {
2353 1.69 hannken m_clget(m, M_WAIT);
2354 1.69 hannken CTASSERT(sizeof(*wgmc) <= MCLBYTES);
2355 1.69 hannken }
2356 1.1 riastrad m->m_pkthdr.len = m->m_len = sizeof(*wgmc);
2357 1.1 riastrad wgmc = mtod(m, struct wg_msg_cookie *);
2358 1.1 riastrad wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src);
2359 1.1 riastrad
2360 1.108 riastrad error = wg->wg_ops->send_hs_msg(wgp, m); /* consumes m */
2361 1.108 riastrad if (error) {
2362 1.108 riastrad WG_DLOG("send_hs_msg failed, error=%d\n", error);
2363 1.108 riastrad return;
2364 1.108 riastrad }
2365 1.108 riastrad
2366 1.108 riastrad WG_TRACE("cookie msg sent");
2367 1.1 riastrad }
2368 1.1 riastrad
2369 1.1 riastrad static bool
2370 1.1 riastrad wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype)
2371 1.1 riastrad {
2372 1.1 riastrad #ifdef WG_DEBUG_PARAMS
2373 1.1 riastrad if (wg_force_underload)
2374 1.1 riastrad return true;
2375 1.1 riastrad #endif
2376 1.1 riastrad
2377 1.1 riastrad /*
2378 1.1 riastrad * XXX we don't have a means of a load estimation. The purpose of
2379 1.1 riastrad * the mechanism is a DoS mitigation, so we consider frequent handshake
2380 1.1 riastrad * messages as (a kind of) load; if a message of the same type comes
2381 1.1 riastrad * to a peer within 1 second, we consider we are under load.
2382 1.1 riastrad */
2383 1.1 riastrad time_t last = wgp->wgp_last_msg_received_time[msgtype];
2384 1.1 riastrad wgp->wgp_last_msg_received_time[msgtype] = time_uptime;
2385 1.1 riastrad return (time_uptime - last) == 0;
2386 1.1 riastrad }
2387 1.1 riastrad
2388 1.1 riastrad static void
2389 1.1 riastrad wg_calculate_keys(struct wg_session *wgs, const bool initiator)
2390 1.1 riastrad {
2391 1.1 riastrad
2392 1.49 riastrad KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock));
2393 1.49 riastrad
2394 1.14 riastrad /*
2395 1.14 riastrad * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e)
2396 1.14 riastrad */
2397 1.1 riastrad if (initiator) {
2398 1.1 riastrad wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL,
2399 1.1 riastrad wgs->wgs_chaining_key, NULL, 0);
2400 1.1 riastrad } else {
2401 1.1 riastrad wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL,
2402 1.1 riastrad wgs->wgs_chaining_key, NULL, 0);
2403 1.1 riastrad }
2404 1.1 riastrad WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send);
2405 1.1 riastrad WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv);
2406 1.1 riastrad }
2407 1.1 riastrad
2408 1.22 riastrad static uint64_t
2409 1.22 riastrad wg_session_get_send_counter(struct wg_session *wgs)
2410 1.22 riastrad {
2411 1.22 riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
2412 1.22 riastrad return atomic_load_relaxed(&wgs->wgs_send_counter);
2413 1.22 riastrad #else
2414 1.22 riastrad uint64_t send_counter;
2415 1.22 riastrad
2416 1.22 riastrad mutex_enter(&wgs->wgs_send_counter_lock);
2417 1.22 riastrad send_counter = wgs->wgs_send_counter;
2418 1.22 riastrad mutex_exit(&wgs->wgs_send_counter_lock);
2419 1.22 riastrad
2420 1.22 riastrad return send_counter;
2421 1.22 riastrad #endif
2422 1.22 riastrad }
2423 1.22 riastrad
2424 1.22 riastrad static uint64_t
2425 1.22 riastrad wg_session_inc_send_counter(struct wg_session *wgs)
2426 1.22 riastrad {
2427 1.22 riastrad #ifdef __HAVE_ATOMIC64_LOADSTORE
2428 1.22 riastrad return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1;
2429 1.22 riastrad #else
2430 1.22 riastrad uint64_t send_counter;
2431 1.22 riastrad
2432 1.22 riastrad mutex_enter(&wgs->wgs_send_counter_lock);
2433 1.22 riastrad send_counter = wgs->wgs_send_counter++;
2434 1.22 riastrad mutex_exit(&wgs->wgs_send_counter_lock);
2435 1.22 riastrad
2436 1.22 riastrad return send_counter;
2437 1.22 riastrad #endif
2438 1.22 riastrad }
2439 1.22 riastrad
2440 1.1 riastrad static void
2441 1.1 riastrad wg_clear_states(struct wg_session *wgs)
2442 1.1 riastrad {
2443 1.1 riastrad
2444 1.49 riastrad KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock));
2445 1.49 riastrad
2446 1.1 riastrad wgs->wgs_send_counter = 0;
2447 1.6 riastrad sliwin_reset(&wgs->wgs_recvwin->window);
2448 1.1 riastrad
2449 1.1 riastrad #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v))
2450 1.1 riastrad wgs_clear(handshake_hash);
2451 1.1 riastrad wgs_clear(chaining_key);
2452 1.1 riastrad wgs_clear(ephemeral_key_pub);
2453 1.1 riastrad wgs_clear(ephemeral_key_priv);
2454 1.1 riastrad wgs_clear(ephemeral_key_peer);
2455 1.1 riastrad #undef wgs_clear
2456 1.1 riastrad }
2457 1.1 riastrad
2458 1.1 riastrad static struct wg_session *
2459 1.1 riastrad wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index,
2460 1.1 riastrad struct psref *psref)
2461 1.1 riastrad {
2462 1.1 riastrad struct wg_session *wgs;
2463 1.1 riastrad
2464 1.1 riastrad int s = pserialize_read_enter();
2465 1.37 riastrad wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index);
2466 1.49 riastrad if (wgs != NULL) {
2467 1.94 riastrad uint32_t oindex __diagused =
2468 1.94 riastrad atomic_load_relaxed(&wgs->wgs_local_index);
2469 1.94 riastrad KASSERTMSG(index == oindex,
2470 1.94 riastrad "index=%"PRIx32" wgs->wgs_local_index=%"PRIx32,
2471 1.94 riastrad index, oindex);
2472 1.1 riastrad psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
2473 1.49 riastrad }
2474 1.1 riastrad pserialize_read_exit(s);
2475 1.1 riastrad
2476 1.1 riastrad return wgs;
2477 1.1 riastrad }
2478 1.1 riastrad
2479 1.1 riastrad static void
2480 1.1 riastrad wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs)
2481 1.1 riastrad {
2482 1.1 riastrad struct mbuf *m;
2483 1.1 riastrad
2484 1.1 riastrad /*
2485 1.1 riastrad * [W] 6.5 Passive Keepalive
2486 1.1 riastrad * "A keepalive message is simply a transport data message with
2487 1.1 riastrad * a zero-length encapsulated encrypted inner-packet."
2488 1.1 riastrad */
2489 1.80 christos WG_TRACE("");
2490 1.1 riastrad m = m_gethdr(M_WAIT, MT_DATA);
2491 1.1 riastrad wg_send_data_msg(wgp, wgs, m);
2492 1.1 riastrad }
2493 1.1 riastrad
2494 1.1 riastrad static bool
2495 1.1 riastrad wg_need_to_send_init_message(struct wg_session *wgs)
2496 1.1 riastrad {
2497 1.1 riastrad /*
2498 1.1 riastrad * [W] 6.2 Transport Message Limits
2499 1.1 riastrad * "if a peer is the initiator of a current secure session,
2500 1.1 riastrad * WireGuard will send a handshake initiation message to begin
2501 1.1 riastrad * a new secure session ... if after receiving a transport data
2502 1.1 riastrad * message, the current secure session is (REJECT-AFTER-TIME
2503 1.1 riastrad * KEEPALIVE-TIMEOUT REKEY-TIMEOUT) seconds old and it has
2504 1.1 riastrad * not yet acted upon this event."
2505 1.1 riastrad */
2506 1.104 riastrad return wgs->wgs_is_initiator &&
2507 1.104 riastrad atomic_load_relaxed(&wgs->wgs_time_last_data_sent) == 0 &&
2508 1.104 riastrad ((time_uptime32 -
2509 1.104 riastrad atomic_load_relaxed(&wgs->wgs_time_established)) >=
2510 1.104 riastrad (wg_reject_after_time - wg_keepalive_timeout -
2511 1.104 riastrad wg_rekey_timeout));
2512 1.1 riastrad }
2513 1.1 riastrad
2514 1.1 riastrad static void
2515 1.65 christos wg_schedule_peer_task(struct wg_peer *wgp, unsigned int task)
2516 1.1 riastrad {
2517 1.1 riastrad
2518 1.55 riastrad mutex_enter(wgp->wgp_intr_lock);
2519 1.1 riastrad WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task);
2520 1.55 riastrad if (wgp->wgp_tasks == 0)
2521 1.55 riastrad /*
2522 1.55 riastrad * XXX If the current CPU is already loaded -- e.g., if
2523 1.55 riastrad * there's already a bunch of handshakes queued up --
2524 1.55 riastrad * consider tossing this over to another CPU to
2525 1.55 riastrad * distribute the load.
2526 1.55 riastrad */
2527 1.55 riastrad workqueue_enqueue(wg_wq, &wgp->wgp_work, NULL);
2528 1.55 riastrad wgp->wgp_tasks |= task;
2529 1.55 riastrad mutex_exit(wgp->wgp_intr_lock);
2530 1.1 riastrad }
2531 1.1 riastrad
2532 1.1 riastrad static void
2533 1.1 riastrad wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new)
2534 1.1 riastrad {
2535 1.49 riastrad struct wg_sockaddr *wgsa_prev;
2536 1.1 riastrad
2537 1.1 riastrad WG_TRACE("Changing endpoint");
2538 1.1 riastrad
2539 1.1 riastrad memcpy(wgp->wgp_endpoint0, new, new->sa_len);
2540 1.49 riastrad wgsa_prev = wgp->wgp_endpoint;
2541 1.49 riastrad atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0);
2542 1.49 riastrad wgp->wgp_endpoint0 = wgsa_prev;
2543 1.49 riastrad atomic_store_release(&wgp->wgp_endpoint_available, true);
2544 1.49 riastrad
2545 1.1 riastrad wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED);
2546 1.1 riastrad }
2547 1.1 riastrad
2548 1.2 riastrad static bool
2549 1.17 riastrad wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af)
2550 1.1 riastrad {
2551 1.2 riastrad uint16_t packet_len;
2552 1.17 riastrad const struct ip *ip;
2553 1.2 riastrad
2554 1.81 christos if (__predict_false(decrypted_len < sizeof(*ip))) {
2555 1.81 christos WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len,
2556 1.81 christos sizeof(*ip));
2557 1.2 riastrad return false;
2558 1.81 christos }
2559 1.1 riastrad
2560 1.17 riastrad ip = (const struct ip *)packet;
2561 1.2 riastrad if (ip->ip_v == 4)
2562 1.2 riastrad *af = AF_INET;
2563 1.2 riastrad else if (ip->ip_v == 6)
2564 1.2 riastrad *af = AF_INET6;
2565 1.81 christos else {
2566 1.81 christos WG_DLOG("ip_v=%d\n", ip->ip_v);
2567 1.2 riastrad return false;
2568 1.81 christos }
2569 1.2 riastrad
2570 1.2 riastrad WG_DLOG("af=%d\n", *af);
2571 1.1 riastrad
2572 1.62 riastrad switch (*af) {
2573 1.62 riastrad #ifdef INET
2574 1.62 riastrad case AF_INET:
2575 1.2 riastrad packet_len = ntohs(ip->ip_len);
2576 1.62 riastrad break;
2577 1.62 riastrad #endif
2578 1.62 riastrad #ifdef INET6
2579 1.62 riastrad case AF_INET6: {
2580 1.17 riastrad const struct ip6_hdr *ip6;
2581 1.1 riastrad
2582 1.81 christos if (__predict_false(decrypted_len < sizeof(*ip6))) {
2583 1.81 christos WG_DLOG("decrypted_len=%zu < %zu\n", decrypted_len,
2584 1.81 christos sizeof(*ip6));
2585 1.2 riastrad return false;
2586 1.81 christos }
2587 1.1 riastrad
2588 1.17 riastrad ip6 = (const struct ip6_hdr *)packet;
2589 1.81 christos packet_len = sizeof(*ip6) + ntohs(ip6->ip6_plen);
2590 1.62 riastrad break;
2591 1.62 riastrad }
2592 1.62 riastrad #endif
2593 1.62 riastrad default:
2594 1.62 riastrad return false;
2595 1.1 riastrad }
2596 1.2 riastrad
2597 1.81 christos if (packet_len > decrypted_len) {
2598 1.81 christos WG_DLOG("packet_len %u > decrypted_len %zu\n", packet_len,
2599 1.81 christos decrypted_len);
2600 1.1 riastrad return false;
2601 1.81 christos }
2602 1.1 riastrad
2603 1.1 riastrad return true;
2604 1.1 riastrad }
2605 1.1 riastrad
2606 1.1 riastrad static bool
2607 1.1 riastrad wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected,
2608 1.1 riastrad int af, char *packet)
2609 1.1 riastrad {
2610 1.1 riastrad struct sockaddr_storage ss;
2611 1.1 riastrad struct sockaddr *sa;
2612 1.1 riastrad struct psref psref;
2613 1.1 riastrad struct wg_peer *wgp;
2614 1.1 riastrad bool ok;
2615 1.1 riastrad
2616 1.1 riastrad /*
2617 1.1 riastrad * II CRYPTOKEY ROUTING
2618 1.14 riastrad * "it will only accept it if its source IP resolves in the
2619 1.14 riastrad * table to the public key used in the secure session for
2620 1.14 riastrad * decrypting it."
2621 1.1 riastrad */
2622 1.1 riastrad
2623 1.109 riastrad switch (af) {
2624 1.109 riastrad #ifdef INET
2625 1.109 riastrad case AF_INET: {
2626 1.17 riastrad const struct ip *ip = (const struct ip *)packet;
2627 1.1 riastrad struct sockaddr_in *sin = (struct sockaddr_in *)&ss;
2628 1.1 riastrad sockaddr_in_init(sin, &ip->ip_src, 0);
2629 1.1 riastrad sa = sintosa(sin);
2630 1.109 riastrad break;
2631 1.109 riastrad }
2632 1.109 riastrad #endif
2633 1.1 riastrad #ifdef INET6
2634 1.109 riastrad case AF_INET6: {
2635 1.17 riastrad const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet;
2636 1.1 riastrad struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss;
2637 1.1 riastrad sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0);
2638 1.1 riastrad sa = sin6tosa(sin6);
2639 1.109 riastrad break;
2640 1.109 riastrad }
2641 1.1 riastrad #endif
2642 1.109 riastrad default:
2643 1.109 riastrad __USE(ss);
2644 1.109 riastrad return false;
2645 1.1 riastrad }
2646 1.1 riastrad
2647 1.1 riastrad wgp = wg_pick_peer_by_sa(wg, sa, &psref);
2648 1.1 riastrad ok = (wgp == wgp_expected);
2649 1.1 riastrad if (wgp != NULL)
2650 1.1 riastrad wg_put_peer(wgp, &psref);
2651 1.1 riastrad
2652 1.1 riastrad return ok;
2653 1.1 riastrad }
2654 1.1 riastrad
2655 1.1 riastrad static void
2656 1.1 riastrad wg_session_dtor_timer(void *arg)
2657 1.1 riastrad {
2658 1.1 riastrad struct wg_peer *wgp = arg;
2659 1.1 riastrad
2660 1.1 riastrad WG_TRACE("enter");
2661 1.1 riastrad
2662 1.100 riastrad wg_schedule_session_dtor_timer(wgp);
2663 1.1 riastrad wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION);
2664 1.1 riastrad }
2665 1.1 riastrad
2666 1.1 riastrad static void
2667 1.1 riastrad wg_schedule_session_dtor_timer(struct wg_peer *wgp)
2668 1.1 riastrad {
2669 1.1 riastrad
2670 1.100 riastrad /*
2671 1.100 riastrad * If the periodic session destructor is already pending to
2672 1.100 riastrad * handle the previous session, that's fine -- leave it in
2673 1.100 riastrad * place; it will be scheduled again.
2674 1.100 riastrad */
2675 1.100 riastrad if (callout_pending(&wgp->wgp_session_dtor_timer)) {
2676 1.100 riastrad WG_DLOG("session dtor already pending\n");
2677 1.100 riastrad return;
2678 1.100 riastrad }
2679 1.100 riastrad
2680 1.100 riastrad WG_DLOG("scheduling session dtor in %u secs\n", wg_reject_after_time);
2681 1.100 riastrad callout_schedule(&wgp->wgp_session_dtor_timer,
2682 1.100 riastrad wg_reject_after_time*hz);
2683 1.1 riastrad }
2684 1.1 riastrad
2685 1.1 riastrad static bool
2686 1.1 riastrad sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2)
2687 1.1 riastrad {
2688 1.1 riastrad if (sa1->sa_family != sa2->sa_family)
2689 1.1 riastrad return false;
2690 1.1 riastrad
2691 1.1 riastrad switch (sa1->sa_family) {
2692 1.62 riastrad #ifdef INET
2693 1.1 riastrad case AF_INET:
2694 1.1 riastrad return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port;
2695 1.62 riastrad #endif
2696 1.62 riastrad #ifdef INET6
2697 1.1 riastrad case AF_INET6:
2698 1.1 riastrad return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port;
2699 1.62 riastrad #endif
2700 1.1 riastrad default:
2701 1.62 riastrad return false;
2702 1.1 riastrad }
2703 1.1 riastrad }
2704 1.1 riastrad
2705 1.1 riastrad static void
2706 1.1 riastrad wg_update_endpoint_if_necessary(struct wg_peer *wgp,
2707 1.1 riastrad const struct sockaddr *src)
2708 1.1 riastrad {
2709 1.47 riastrad struct wg_sockaddr *wgsa;
2710 1.47 riastrad struct psref psref;
2711 1.47 riastrad
2712 1.47 riastrad wgsa = wg_get_endpoint_sa(wgp, &psref);
2713 1.1 riastrad
2714 1.1 riastrad #ifdef WG_DEBUG_LOG
2715 1.1 riastrad char oldaddr[128], newaddr[128];
2716 1.47 riastrad sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr));
2717 1.1 riastrad sockaddr_format(src, newaddr, sizeof(newaddr));
2718 1.1 riastrad WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr);
2719 1.1 riastrad #endif
2720 1.1 riastrad
2721 1.1 riastrad /*
2722 1.1 riastrad * III: "Since the packet has authenticated correctly, the source IP of
2723 1.1 riastrad * the outer UDP/IP packet is used to update the endpoint for peer..."
2724 1.1 riastrad */
2725 1.47 riastrad if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 ||
2726 1.47 riastrad !sockaddr_port_match(src, wgsatosa(wgsa)))) {
2727 1.1 riastrad /* XXX We can't change the endpoint twice in a short period */
2728 1.49 riastrad if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) {
2729 1.1 riastrad wg_change_endpoint(wgp, src);
2730 1.1 riastrad }
2731 1.1 riastrad }
2732 1.47 riastrad
2733 1.47 riastrad wg_put_sa(wgp, wgsa, &psref);
2734 1.1 riastrad }
2735 1.1 riastrad
2736 1.63 riastrad static void __noinline
2737 1.1 riastrad wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m,
2738 1.1 riastrad const struct sockaddr *src)
2739 1.1 riastrad {
2740 1.2 riastrad struct wg_msg_data *wgmd;
2741 1.1 riastrad char *encrypted_buf = NULL, *decrypted_buf;
2742 1.1 riastrad size_t encrypted_len, decrypted_len;
2743 1.1 riastrad struct wg_session *wgs;
2744 1.1 riastrad struct wg_peer *wgp;
2745 1.49 riastrad int state;
2746 1.104 riastrad uint32_t age;
2747 1.1 riastrad size_t mlen;
2748 1.1 riastrad struct psref psref;
2749 1.1 riastrad int error, af;
2750 1.1 riastrad bool success, free_encrypted_buf = false, ok;
2751 1.1 riastrad struct mbuf *n;
2752 1.1 riastrad
2753 1.26 riastrad KASSERT(m->m_len >= sizeof(struct wg_msg_data));
2754 1.2 riastrad wgmd = mtod(m, struct wg_msg_data *);
2755 1.2 riastrad
2756 1.39 riastrad KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA));
2757 1.1 riastrad WG_TRACE("data");
2758 1.1 riastrad
2759 1.49 riastrad /* Find the putative session, or drop. */
2760 1.1 riastrad wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref);
2761 1.1 riastrad if (wgs == NULL) {
2762 1.1 riastrad WG_TRACE("No session found");
2763 1.1 riastrad m_freem(m);
2764 1.1 riastrad return;
2765 1.1 riastrad }
2766 1.49 riastrad
2767 1.49 riastrad /*
2768 1.49 riastrad * We are only ready to handle data when in INIT_PASSIVE,
2769 1.49 riastrad * ESTABLISHED, or DESTROYING. All transitions out of that
2770 1.49 riastrad * state dissociate the session index and drain psrefs.
2771 1.94 riastrad *
2772 1.94 riastrad * atomic_load_acquire matches atomic_store_release in either
2773 1.94 riastrad * wg_handle_msg_init or wg_handle_msg_resp. (The transition
2774 1.94 riastrad * INIT_PASSIVE to ESTABLISHED in wg_task_establish_session
2775 1.94 riastrad * doesn't make a difference for this rx path.)
2776 1.49 riastrad */
2777 1.94 riastrad state = atomic_load_acquire(&wgs->wgs_state);
2778 1.49 riastrad switch (state) {
2779 1.49 riastrad case WGS_STATE_UNKNOWN:
2780 1.49 riastrad case WGS_STATE_INIT_ACTIVE:
2781 1.49 riastrad WG_TRACE("not yet ready for data");
2782 1.49 riastrad goto out;
2783 1.49 riastrad case WGS_STATE_INIT_PASSIVE:
2784 1.49 riastrad case WGS_STATE_ESTABLISHED:
2785 1.49 riastrad case WGS_STATE_DESTROYING:
2786 1.49 riastrad break;
2787 1.49 riastrad }
2788 1.49 riastrad
2789 1.49 riastrad /*
2790 1.101 riastrad * Reject if the session is too old.
2791 1.101 riastrad */
2792 1.104 riastrad age = time_uptime32 - atomic_load_relaxed(&wgs->wgs_time_established);
2793 1.101 riastrad if (__predict_false(age >= wg_reject_after_time)) {
2794 1.104 riastrad WG_DLOG("session %"PRIx32" too old, %"PRIu32" sec\n",
2795 1.104 riastrad wgmd->wgmd_receiver, age);
2796 1.101 riastrad goto out;
2797 1.101 riastrad }
2798 1.101 riastrad
2799 1.101 riastrad /*
2800 1.49 riastrad * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and
2801 1.49 riastrad * to update the endpoint if authentication succeeds.
2802 1.49 riastrad */
2803 1.1 riastrad wgp = wgs->wgs_peer;
2804 1.1 riastrad
2805 1.49 riastrad /*
2806 1.49 riastrad * Reject outrageously wrong sequence numbers before doing any
2807 1.49 riastrad * crypto work or taking any locks.
2808 1.49 riastrad */
2809 1.6 riastrad error = sliwin_check_fast(&wgs->wgs_recvwin->window,
2810 1.39 riastrad le64toh(wgmd->wgmd_counter));
2811 1.6 riastrad if (error) {
2812 1.6 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2813 1.76 jakllsch "%s: peer %s: out-of-window packet: %"PRIu64"\n",
2814 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name,
2815 1.39 riastrad le64toh(wgmd->wgmd_counter));
2816 1.6 riastrad goto out;
2817 1.6 riastrad }
2818 1.6 riastrad
2819 1.49 riastrad /* Ensure the payload and authenticator are contiguous. */
2820 1.1 riastrad mlen = m_length(m);
2821 1.1 riastrad encrypted_len = mlen - sizeof(*wgmd);
2822 1.2 riastrad if (encrypted_len < WG_AUTHTAG_LEN) {
2823 1.87 kre WG_DLOG("Short encrypted_len: %zu\n", encrypted_len);
2824 1.2 riastrad goto out;
2825 1.2 riastrad }
2826 1.1 riastrad success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len);
2827 1.1 riastrad if (success) {
2828 1.1 riastrad encrypted_buf = mtod(m, char *) + sizeof(*wgmd);
2829 1.1 riastrad } else {
2830 1.1 riastrad encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP);
2831 1.1 riastrad if (encrypted_buf == NULL) {
2832 1.1 riastrad WG_DLOG("failed to allocate encrypted_buf\n");
2833 1.1 riastrad goto out;
2834 1.1 riastrad }
2835 1.2 riastrad m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf);
2836 1.1 riastrad free_encrypted_buf = true;
2837 1.1 riastrad }
2838 1.1 riastrad /* m_ensure_contig may change m regardless of its result */
2839 1.27 riastrad KASSERT(m->m_len >= sizeof(*wgmd));
2840 1.1 riastrad wgmd = mtod(m, struct wg_msg_data *);
2841 1.1 riastrad
2842 1.49 riastrad /*
2843 1.49 riastrad * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid
2844 1.49 riastrad * a zero-length buffer (XXX). Drop if plaintext is longer
2845 1.49 riastrad * than MCLBYTES (XXX).
2846 1.49 riastrad */
2847 1.2 riastrad decrypted_len = encrypted_len - WG_AUTHTAG_LEN;
2848 1.2 riastrad if (decrypted_len > MCLBYTES) {
2849 1.2 riastrad /* FIXME handle larger data than MCLBYTES */
2850 1.2 riastrad WG_DLOG("couldn't handle larger data than MCLBYTES\n");
2851 1.2 riastrad goto out;
2852 1.2 riastrad }
2853 1.14 riastrad n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN);
2854 1.1 riastrad if (n == NULL) {
2855 1.1 riastrad WG_DLOG("wg_get_mbuf failed\n");
2856 1.1 riastrad goto out;
2857 1.1 riastrad }
2858 1.1 riastrad decrypted_buf = mtod(n, char *);
2859 1.1 riastrad
2860 1.49 riastrad /* Decrypt and verify the packet. */
2861 1.87 kre WG_DLOG("mlen=%zu, encrypted_len=%zu\n", mlen, encrypted_len);
2862 1.1 riastrad error = wg_algo_aead_dec(decrypted_buf,
2863 1.1 riastrad encrypted_len - WG_AUTHTAG_LEN /* can be 0 */,
2864 1.39 riastrad wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf,
2865 1.1 riastrad encrypted_len, NULL, 0);
2866 1.1 riastrad if (error != 0) {
2867 1.1 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2868 1.76 jakllsch "%s: peer %s: failed to wg_algo_aead_dec\n",
2869 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name);
2870 1.1 riastrad m_freem(n);
2871 1.1 riastrad goto out;
2872 1.1 riastrad }
2873 1.1 riastrad WG_DLOG("outsize=%u\n", (u_int)decrypted_len);
2874 1.1 riastrad
2875 1.49 riastrad /* Packet is genuine. Reject it if a replay or just too old. */
2876 1.6 riastrad mutex_enter(&wgs->wgs_recvwin->lock);
2877 1.6 riastrad error = sliwin_update(&wgs->wgs_recvwin->window,
2878 1.39 riastrad le64toh(wgmd->wgmd_counter));
2879 1.6 riastrad mutex_exit(&wgs->wgs_recvwin->lock);
2880 1.6 riastrad if (error) {
2881 1.1 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2882 1.76 jakllsch "%s: peer %s: replay or out-of-window packet: %"PRIu64"\n",
2883 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name,
2884 1.39 riastrad le64toh(wgmd->wgmd_counter));
2885 1.1 riastrad m_freem(n);
2886 1.1 riastrad goto out;
2887 1.1 riastrad }
2888 1.1 riastrad
2889 1.49 riastrad /* We're done with m now; free it and chuck the pointers. */
2890 1.1 riastrad m_freem(m);
2891 1.1 riastrad m = NULL;
2892 1.1 riastrad wgmd = NULL;
2893 1.1 riastrad
2894 1.49 riastrad /*
2895 1.103 riastrad * The packet is genuine. Update the peer's endpoint if the
2896 1.103 riastrad * source address changed.
2897 1.103 riastrad *
2898 1.103 riastrad * XXX How to prevent DoS by replaying genuine packets from the
2899 1.103 riastrad * wrong source address?
2900 1.103 riastrad */
2901 1.103 riastrad wg_update_endpoint_if_necessary(wgp, src);
2902 1.103 riastrad
2903 1.103 riastrad /*
2904 1.49 riastrad * Validate the encapsulated packet header and get the address
2905 1.49 riastrad * family, or drop.
2906 1.49 riastrad */
2907 1.2 riastrad ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af);
2908 1.1 riastrad if (!ok) {
2909 1.1 riastrad m_freem(n);
2910 1.102 riastrad goto update_state;
2911 1.1 riastrad }
2912 1.1 riastrad
2913 1.49 riastrad /* Submit it into our network stack if routable. */
2914 1.1 riastrad ok = wg_validate_route(wg, wgp, af, decrypted_buf);
2915 1.1 riastrad if (ok) {
2916 1.1 riastrad wg->wg_ops->input(&wg->wg_if, n, af);
2917 1.1 riastrad } else {
2918 1.76 jakllsch char addrstr[INET6_ADDRSTRLEN];
2919 1.76 jakllsch memset(addrstr, 0, sizeof(addrstr));
2920 1.109 riastrad switch (af) {
2921 1.109 riastrad #ifdef INET
2922 1.109 riastrad case AF_INET: {
2923 1.76 jakllsch const struct ip *ip = (const struct ip *)decrypted_buf;
2924 1.76 jakllsch IN_PRINT(addrstr, &ip->ip_src);
2925 1.109 riastrad break;
2926 1.109 riastrad }
2927 1.109 riastrad #endif
2928 1.76 jakllsch #ifdef INET6
2929 1.109 riastrad case AF_INET6: {
2930 1.76 jakllsch const struct ip6_hdr *ip6 =
2931 1.76 jakllsch (const struct ip6_hdr *)decrypted_buf;
2932 1.76 jakllsch IN6_PRINT(addrstr, &ip6->ip6_src);
2933 1.109 riastrad break;
2934 1.109 riastrad }
2935 1.76 jakllsch #endif
2936 1.109 riastrad default:
2937 1.109 riastrad panic("invalid af=%d", af);
2938 1.76 jakllsch }
2939 1.1 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2940 1.76 jakllsch "%s: peer %s: invalid source address (%s)\n",
2941 1.76 jakllsch if_name(&wg->wg_if), wgp->wgp_name, addrstr);
2942 1.1 riastrad m_freem(n);
2943 1.1 riastrad /*
2944 1.1 riastrad * The inner address is invalid however the session is valid
2945 1.1 riastrad * so continue the session processing below.
2946 1.1 riastrad */
2947 1.1 riastrad }
2948 1.1 riastrad n = NULL;
2949 1.1 riastrad
2950 1.102 riastrad update_state:
2951 1.49 riastrad /* Update the state machine if necessary. */
2952 1.49 riastrad if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) {
2953 1.49 riastrad /*
2954 1.49 riastrad * We were waiting for the initiator to send their
2955 1.49 riastrad * first data transport message, and that has happened.
2956 1.49 riastrad * Schedule a task to establish this session.
2957 1.49 riastrad */
2958 1.49 riastrad wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION);
2959 1.1 riastrad } else {
2960 1.1 riastrad if (__predict_false(wg_need_to_send_init_message(wgs))) {
2961 1.1 riastrad wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
2962 1.1 riastrad }
2963 1.1 riastrad /*
2964 1.1 riastrad * [W] 6.5 Passive Keepalive
2965 1.1 riastrad * "If a peer has received a validly-authenticated transport
2966 1.1 riastrad * data message (section 5.4.6), but does not have any packets
2967 1.1 riastrad * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends
2968 1.1 riastrad * a keepalive message."
2969 1.1 riastrad */
2970 1.104 riastrad const uint32_t now = time_uptime32;
2971 1.104 riastrad const uint32_t time_last_data_sent =
2972 1.104 riastrad atomic_load_relaxed(&wgs->wgs_time_last_data_sent);
2973 1.104 riastrad WG_DLOG("time_uptime32=%"PRIu32
2974 1.104 riastrad " wgs_time_last_data_sent=%"PRIu32"\n",
2975 1.104 riastrad now, time_last_data_sent);
2976 1.104 riastrad if ((now - time_last_data_sent) >= wg_keepalive_timeout) {
2977 1.1 riastrad WG_TRACE("Schedule sending keepalive message");
2978 1.1 riastrad /*
2979 1.1 riastrad * We can't send a keepalive message here to avoid
2980 1.1 riastrad * a deadlock; we already hold the solock of a socket
2981 1.1 riastrad * that is used to send the message.
2982 1.1 riastrad */
2983 1.14 riastrad wg_schedule_peer_task(wgp,
2984 1.14 riastrad WGP_TASK_SEND_KEEPALIVE_MESSAGE);
2985 1.1 riastrad }
2986 1.1 riastrad }
2987 1.1 riastrad out:
2988 1.1 riastrad wg_put_session(wgs, &psref);
2989 1.79 rin m_freem(m);
2990 1.1 riastrad if (free_encrypted_buf)
2991 1.1 riastrad kmem_intr_free(encrypted_buf, encrypted_len);
2992 1.1 riastrad }
2993 1.1 riastrad
2994 1.63 riastrad static void __noinline
2995 1.1 riastrad wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc)
2996 1.1 riastrad {
2997 1.1 riastrad struct wg_session *wgs;
2998 1.1 riastrad struct wg_peer *wgp;
2999 1.1 riastrad struct psref psref;
3000 1.1 riastrad int error;
3001 1.1 riastrad uint8_t key[WG_HASH_LEN];
3002 1.1 riastrad uint8_t cookie[WG_COOKIE_LEN];
3003 1.1 riastrad
3004 1.1 riastrad WG_TRACE("cookie msg received");
3005 1.49 riastrad
3006 1.49 riastrad /* Find the putative session. */
3007 1.1 riastrad wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref);
3008 1.1 riastrad if (wgs == NULL) {
3009 1.1 riastrad WG_TRACE("No session found");
3010 1.1 riastrad return;
3011 1.1 riastrad }
3012 1.49 riastrad
3013 1.49 riastrad /* Lock the peer so we can update the cookie state. */
3014 1.1 riastrad wgp = wgs->wgs_peer;
3015 1.49 riastrad mutex_enter(wgp->wgp_lock);
3016 1.1 riastrad
3017 1.1 riastrad if (!wgp->wgp_last_sent_mac1_valid) {
3018 1.1 riastrad WG_TRACE("No valid mac1 sent (or expired)");
3019 1.1 riastrad goto out;
3020 1.1 riastrad }
3021 1.1 riastrad
3022 1.94 riastrad /*
3023 1.94 riastrad * wgp_last_sent_mac1_valid is only set to true when we are
3024 1.94 riastrad * transitioning to INIT_ACTIVE or INIT_PASSIVE, and always
3025 1.94 riastrad * cleared on transition out of them.
3026 1.94 riastrad */
3027 1.94 riastrad KASSERTMSG((wgs->wgs_state == WGS_STATE_INIT_ACTIVE ||
3028 1.94 riastrad wgs->wgs_state == WGS_STATE_INIT_PASSIVE),
3029 1.94 riastrad "state=%d", wgs->wgs_state);
3030 1.94 riastrad
3031 1.49 riastrad /* Decrypt the cookie and store it for later handshake retry. */
3032 1.1 riastrad wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey,
3033 1.1 riastrad sizeof(wgp->wgp_pubkey));
3034 1.36 riastrad error = wg_algo_xaead_dec(cookie, sizeof(cookie), key,
3035 1.1 riastrad wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie),
3036 1.1 riastrad wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1),
3037 1.1 riastrad wgmc->wgmc_salt);
3038 1.1 riastrad if (error != 0) {
3039 1.1 riastrad WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
3040 1.76 jakllsch "%s: peer %s: wg_algo_aead_dec for cookie failed: "
3041 1.76 jakllsch "error=%d\n", if_name(&wg->wg_if), wgp->wgp_name, error);
3042 1.1 riastrad goto out;
3043 1.1 riastrad }
3044 1.1 riastrad /*
3045 1.1 riastrad * [W] 6.6: Interaction with Cookie Reply System
3046 1.1 riastrad * "it should simply store the decrypted cookie value from the cookie
3047 1.1 riastrad * reply message, and wait for the expiration of the REKEY-TIMEOUT
3048 1.1 riastrad * timer for retrying a handshake initiation message."
3049 1.1 riastrad */
3050 1.1 riastrad wgp->wgp_latest_cookie_time = time_uptime;
3051 1.1 riastrad memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie));
3052 1.1 riastrad out:
3053 1.49 riastrad mutex_exit(wgp->wgp_lock);
3054 1.1 riastrad wg_put_session(wgs, &psref);
3055 1.1 riastrad }
3056 1.1 riastrad
3057 1.26 riastrad static struct mbuf *
3058 1.26 riastrad wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m)
3059 1.2 riastrad {
3060 1.26 riastrad struct wg_msg wgm;
3061 1.26 riastrad size_t mbuflen;
3062 1.26 riastrad size_t msglen;
3063 1.2 riastrad
3064 1.26 riastrad /*
3065 1.26 riastrad * Get the mbuf chain length. It is already guaranteed, by
3066 1.26 riastrad * wg_overudp_cb, to be large enough for a struct wg_msg.
3067 1.26 riastrad */
3068 1.26 riastrad mbuflen = m_length(m);
3069 1.26 riastrad KASSERT(mbuflen >= sizeof(struct wg_msg));
3070 1.2 riastrad
3071 1.26 riastrad /*
3072 1.26 riastrad * Copy the message header (32-bit message type) out -- we'll
3073 1.26 riastrad * worry about contiguity and alignment later.
3074 1.26 riastrad */
3075 1.26 riastrad m_copydata(m, 0, sizeof(wgm), &wgm);
3076 1.39 riastrad switch (le32toh(wgm.wgm_type)) {
3077 1.2 riastrad case WG_MSG_TYPE_INIT:
3078 1.26 riastrad msglen = sizeof(struct wg_msg_init);
3079 1.2 riastrad break;
3080 1.2 riastrad case WG_MSG_TYPE_RESP:
3081 1.26 riastrad msglen = sizeof(struct wg_msg_resp);
3082 1.2 riastrad break;
3083 1.2 riastrad case WG_MSG_TYPE_COOKIE:
3084 1.26 riastrad msglen = sizeof(struct wg_msg_cookie);
3085 1.2 riastrad break;
3086 1.2 riastrad case WG_MSG_TYPE_DATA:
3087 1.26 riastrad msglen = sizeof(struct wg_msg_data);
3088 1.2 riastrad break;
3089 1.2 riastrad default:
3090 1.2 riastrad WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG,
3091 1.76 jakllsch "%s: Unexpected msg type: %u\n", if_name(&wg->wg_if),
3092 1.76 jakllsch le32toh(wgm.wgm_type));
3093 1.26 riastrad goto error;
3094 1.26 riastrad }
3095 1.26 riastrad
3096 1.26 riastrad /* Verify the mbuf chain is long enough for this type of message. */
3097 1.26 riastrad if (__predict_false(mbuflen < msglen)) {
3098 1.87 kre WG_DLOG("Invalid msg size: mbuflen=%zu type=%u\n", mbuflen,
3099 1.39 riastrad le32toh(wgm.wgm_type));
3100 1.26 riastrad goto error;
3101 1.26 riastrad }
3102 1.26 riastrad
3103 1.26 riastrad /* Make the message header contiguous if necessary. */
3104 1.26 riastrad if (__predict_false(m->m_len < msglen)) {
3105 1.26 riastrad m = m_pullup(m, msglen);
3106 1.26 riastrad if (m == NULL)
3107 1.26 riastrad return NULL;
3108 1.2 riastrad }
3109 1.2 riastrad
3110 1.26 riastrad return m;
3111 1.26 riastrad
3112 1.26 riastrad error:
3113 1.26 riastrad m_freem(m);
3114 1.26 riastrad return NULL;
3115 1.2 riastrad }
3116 1.2 riastrad
3117 1.1 riastrad static void
3118 1.14 riastrad wg_handle_packet(struct wg_softc *wg, struct mbuf *m,
3119 1.14 riastrad const struct sockaddr *src)
3120 1.1 riastrad {
3121 1.1 riastrad struct wg_msg *wgm;
3122 1.2 riastrad
3123 1.78 riastrad KASSERT(curlwp->l_pflag & LP_BOUND);
3124 1.78 riastrad
3125 1.26 riastrad m = wg_validate_msg_header(wg, m);
3126 1.26 riastrad if (__predict_false(m == NULL))
3127 1.2 riastrad return;
3128 1.1 riastrad
3129 1.26 riastrad KASSERT(m->m_len >= sizeof(struct wg_msg));
3130 1.1 riastrad wgm = mtod(m, struct wg_msg *);
3131 1.39 riastrad switch (le32toh(wgm->wgm_type)) {
3132 1.1 riastrad case WG_MSG_TYPE_INIT:
3133 1.1 riastrad wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src);
3134 1.1 riastrad break;
3135 1.1 riastrad case WG_MSG_TYPE_RESP:
3136 1.1 riastrad wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src);
3137 1.1 riastrad break;
3138 1.1 riastrad case WG_MSG_TYPE_COOKIE:
3139 1.1 riastrad wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm);
3140 1.1 riastrad break;
3141 1.1 riastrad case WG_MSG_TYPE_DATA:
3142 1.1 riastrad wg_handle_msg_data(wg, m, src);
3143 1.38 riastrad /* wg_handle_msg_data frees m for us */
3144 1.38 riastrad return;
3145 1.1 riastrad default:
3146 1.39 riastrad panic("invalid message type: %d", le32toh(wgm->wgm_type));
3147 1.1 riastrad }
3148 1.38 riastrad
3149 1.38 riastrad m_freem(m);
3150 1.1 riastrad }
3151 1.1 riastrad
3152 1.1 riastrad static void
3153 1.1 riastrad wg_receive_packets(struct wg_softc *wg, const int af)
3154 1.1 riastrad {
3155 1.1 riastrad
3156 1.14 riastrad for (;;) {
3157 1.1 riastrad int error, flags;
3158 1.1 riastrad struct socket *so;
3159 1.1 riastrad struct mbuf *m = NULL;
3160 1.1 riastrad struct uio dummy_uio;
3161 1.1 riastrad struct mbuf *paddr = NULL;
3162 1.1 riastrad struct sockaddr *src;
3163 1.1 riastrad
3164 1.55 riastrad so = wg_get_so_by_af(wg, af);
3165 1.1 riastrad flags = MSG_DONTWAIT;
3166 1.1 riastrad dummy_uio.uio_resid = 1000000000;
3167 1.1 riastrad
3168 1.14 riastrad error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL,
3169 1.14 riastrad &flags);
3170 1.1 riastrad if (error || m == NULL) {
3171 1.1 riastrad //if (error == EWOULDBLOCK)
3172 1.1 riastrad return;
3173 1.1 riastrad }
3174 1.1 riastrad
3175 1.1 riastrad KASSERT(paddr != NULL);
3176 1.27 riastrad KASSERT(paddr->m_len >= sizeof(struct sockaddr));
3177 1.1 riastrad src = mtod(paddr, struct sockaddr *);
3178 1.1 riastrad
3179 1.1 riastrad wg_handle_packet(wg, m, src);
3180 1.1 riastrad }
3181 1.1 riastrad }
3182 1.1 riastrad
3183 1.1 riastrad static void
3184 1.1 riastrad wg_get_peer(struct wg_peer *wgp, struct psref *psref)
3185 1.1 riastrad {
3186 1.1 riastrad
3187 1.1 riastrad psref_acquire(psref, &wgp->wgp_psref, wg_psref_class);
3188 1.1 riastrad }
3189 1.1 riastrad
3190 1.1 riastrad static void
3191 1.1 riastrad wg_put_peer(struct wg_peer *wgp, struct psref *psref)
3192 1.1 riastrad {
3193 1.1 riastrad
3194 1.1 riastrad psref_release(psref, &wgp->wgp_psref, wg_psref_class);
3195 1.1 riastrad }
3196 1.1 riastrad
3197 1.1 riastrad static void
3198 1.11 riastrad wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp)
3199 1.11 riastrad {
3200 1.11 riastrad struct wg_session *wgs;
3201 1.11 riastrad
3202 1.11 riastrad WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE");
3203 1.11 riastrad
3204 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
3205 1.49 riastrad
3206 1.49 riastrad if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) {
3207 1.76 jakllsch WGLOG(LOG_DEBUG, "%s: No endpoint available\n",
3208 1.76 jakllsch if_name(&wg->wg_if));
3209 1.11 riastrad /* XXX should do something? */
3210 1.11 riastrad return;
3211 1.11 riastrad }
3212 1.11 riastrad
3213 1.95 riastrad /*
3214 1.95 riastrad * If we already have an established session, there's no need
3215 1.95 riastrad * to initiate a new one -- unless the rekey-after-time or
3216 1.95 riastrad * rekey-after-messages limits have passed.
3217 1.95 riastrad */
3218 1.49 riastrad wgs = wgp->wgp_session_stable;
3219 1.95 riastrad if (wgs->wgs_state == WGS_STATE_ESTABLISHED &&
3220 1.113 riastrad !atomic_load_relaxed(&wgs->wgs_force_rekey))
3221 1.95 riastrad return;
3222 1.95 riastrad
3223 1.95 riastrad /*
3224 1.95 riastrad * Ensure we're initiating a new session. If the unstable
3225 1.95 riastrad * session is already INIT_ACTIVE or INIT_PASSIVE, this does
3226 1.95 riastrad * nothing.
3227 1.95 riastrad */
3228 1.95 riastrad wg_send_handshake_msg_init(wg, wgp);
3229 1.11 riastrad }
3230 1.11 riastrad
3231 1.11 riastrad static void
3232 1.49 riastrad wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp)
3233 1.49 riastrad {
3234 1.49 riastrad struct wg_session *wgs;
3235 1.49 riastrad
3236 1.49 riastrad WG_TRACE("WGP_TASK_RETRY_HANDSHAKE");
3237 1.49 riastrad
3238 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
3239 1.49 riastrad KASSERT(wgp->wgp_handshake_start_time != 0);
3240 1.49 riastrad
3241 1.49 riastrad wgs = wgp->wgp_session_unstable;
3242 1.49 riastrad if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE)
3243 1.49 riastrad return;
3244 1.49 riastrad
3245 1.49 riastrad /*
3246 1.49 riastrad * XXX no real need to assign a new index here, but we do need
3247 1.49 riastrad * to transition to UNKNOWN temporarily
3248 1.49 riastrad */
3249 1.49 riastrad wg_put_session_index(wg, wgs);
3250 1.49 riastrad
3251 1.49 riastrad /* [W] 6.4 Handshake Initiation Retransmission */
3252 1.49 riastrad if ((time_uptime - wgp->wgp_handshake_start_time) >
3253 1.49 riastrad wg_rekey_attempt_time) {
3254 1.49 riastrad /* Give up handshaking */
3255 1.49 riastrad wgp->wgp_handshake_start_time = 0;
3256 1.49 riastrad WG_TRACE("give up");
3257 1.49 riastrad
3258 1.49 riastrad /*
3259 1.49 riastrad * If a new data packet comes, handshaking will be retried
3260 1.49 riastrad * and a new session would be established at that time,
3261 1.49 riastrad * however we don't want to send pending packets then.
3262 1.49 riastrad */
3263 1.49 riastrad wg_purge_pending_packets(wgp);
3264 1.49 riastrad return;
3265 1.49 riastrad }
3266 1.49 riastrad
3267 1.49 riastrad wg_task_send_init_message(wg, wgp);
3268 1.49 riastrad }
3269 1.49 riastrad
3270 1.49 riastrad static void
3271 1.49 riastrad wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp)
3272 1.49 riastrad {
3273 1.49 riastrad struct wg_session *wgs, *wgs_prev;
3274 1.54 riastrad struct mbuf *m;
3275 1.49 riastrad
3276 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
3277 1.49 riastrad
3278 1.49 riastrad wgs = wgp->wgp_session_unstable;
3279 1.49 riastrad if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE)
3280 1.49 riastrad /* XXX Can this happen? */
3281 1.49 riastrad return;
3282 1.49 riastrad
3283 1.49 riastrad wgs->wgs_time_last_data_sent = 0;
3284 1.49 riastrad wgs->wgs_is_initiator = false;
3285 1.94 riastrad
3286 1.94 riastrad /*
3287 1.94 riastrad * Session was already ready to receive data. Transition from
3288 1.94 riastrad * INIT_PASSIVE to ESTABLISHED just so we can swap the
3289 1.94 riastrad * sessions.
3290 1.94 riastrad *
3291 1.94 riastrad * atomic_store_relaxed because this doesn't affect the data rx
3292 1.94 riastrad * path, wg_handle_msg_data -- changing from INIT_PASSIVE to
3293 1.94 riastrad * ESTABLISHED makes no difference to the data rx path, and the
3294 1.94 riastrad * transition to INIT_PASSIVE with store-release already
3295 1.94 riastrad * published the state needed by the data rx path.
3296 1.94 riastrad */
3297 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"] -> WGS_STATE_ESTABLISHED\n",
3298 1.94 riastrad wgs->wgs_local_index, wgs->wgs_remote_index);
3299 1.94 riastrad atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_ESTABLISHED);
3300 1.49 riastrad WG_TRACE("WGS_STATE_ESTABLISHED");
3301 1.49 riastrad
3302 1.94 riastrad /*
3303 1.94 riastrad * Session is ready to send data too now that we have received
3304 1.94 riastrad * the peer initiator's first data packet.
3305 1.94 riastrad *
3306 1.94 riastrad * Swap the sessions to publish the new one as the stable
3307 1.94 riastrad * session for the data tx path, wg_output.
3308 1.94 riastrad */
3309 1.49 riastrad wg_swap_sessions(wgp);
3310 1.49 riastrad KASSERT(wgs == wgp->wgp_session_stable);
3311 1.49 riastrad wgs_prev = wgp->wgp_session_unstable;
3312 1.49 riastrad getnanotime(&wgp->wgp_last_handshake_time);
3313 1.49 riastrad wgp->wgp_handshake_start_time = 0;
3314 1.49 riastrad wgp->wgp_last_sent_mac1_valid = false;
3315 1.49 riastrad wgp->wgp_last_sent_cookie_valid = false;
3316 1.49 riastrad
3317 1.54 riastrad /* If we had a data packet queued up, send it. */
3318 1.54 riastrad if ((m = atomic_swap_ptr(&wgp->wgp_pending, NULL)) != NULL) {
3319 1.57 riastrad kpreempt_disable();
3320 1.54 riastrad const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m)
3321 1.54 riastrad M_SETCTX(m, wgp);
3322 1.54 riastrad if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) {
3323 1.76 jakllsch WGLOG(LOG_ERR, "%s: pktq full, dropping\n",
3324 1.76 jakllsch if_name(&wg->wg_if));
3325 1.54 riastrad m_freem(m);
3326 1.54 riastrad }
3327 1.57 riastrad kpreempt_enable();
3328 1.54 riastrad }
3329 1.54 riastrad
3330 1.49 riastrad if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
3331 1.94 riastrad /*
3332 1.94 riastrad * Transition ESTABLISHED->DESTROYING. The session
3333 1.94 riastrad * will remain usable for the data rx path to process
3334 1.94 riastrad * packets still in flight to us, but we won't use it
3335 1.94 riastrad * for data tx.
3336 1.94 riastrad */
3337 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]"
3338 1.94 riastrad " -> WGS_STATE_DESTROYING\n",
3339 1.94 riastrad wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index);
3340 1.94 riastrad atomic_store_relaxed(&wgs_prev->wgs_state,
3341 1.94 riastrad WGS_STATE_DESTROYING);
3342 1.49 riastrad } else {
3343 1.49 riastrad KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN,
3344 1.49 riastrad "state=%d", wgs_prev->wgs_state);
3345 1.94 riastrad WG_DLOG("session[L=%"PRIx32" R=%"PRIx32"]"
3346 1.94 riastrad " -> WGS_STATE_UNKNOWN\n",
3347 1.94 riastrad wgs_prev->wgs_local_index, wgs_prev->wgs_remote_index);
3348 1.94 riastrad wgs_prev->wgs_local_index = 0; /* paranoia */
3349 1.94 riastrad wgs_prev->wgs_remote_index = 0; /* paranoia */
3350 1.94 riastrad wg_clear_states(wgs_prev); /* paranoia */
3351 1.49 riastrad wgs_prev->wgs_state = WGS_STATE_UNKNOWN;
3352 1.49 riastrad }
3353 1.49 riastrad }
3354 1.49 riastrad
3355 1.49 riastrad static void
3356 1.11 riastrad wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp)
3357 1.11 riastrad {
3358 1.11 riastrad
3359 1.11 riastrad WG_TRACE("WGP_TASK_ENDPOINT_CHANGED");
3360 1.11 riastrad
3361 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
3362 1.49 riastrad
3363 1.49 riastrad if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) {
3364 1.11 riastrad pserialize_perform(wgp->wgp_psz);
3365 1.56 riastrad mutex_exit(wgp->wgp_lock);
3366 1.11 riastrad psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref,
3367 1.11 riastrad wg_psref_class);
3368 1.11 riastrad psref_target_init(&wgp->wgp_endpoint0->wgsa_psref,
3369 1.11 riastrad wg_psref_class);
3370 1.56 riastrad mutex_enter(wgp->wgp_lock);
3371 1.49 riastrad atomic_store_release(&wgp->wgp_endpoint_changing, 0);
3372 1.11 riastrad }
3373 1.11 riastrad }
3374 1.11 riastrad
3375 1.11 riastrad static void
3376 1.11 riastrad wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp)
3377 1.11 riastrad {
3378 1.11 riastrad struct wg_session *wgs;
3379 1.11 riastrad
3380 1.11 riastrad WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE");
3381 1.11 riastrad
3382 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
3383 1.49 riastrad
3384 1.49 riastrad wgs = wgp->wgp_session_stable;
3385 1.49 riastrad if (wgs->wgs_state != WGS_STATE_ESTABLISHED)
3386 1.49 riastrad return;
3387 1.49 riastrad
3388 1.11 riastrad wg_send_keepalive_msg(wgp, wgs);
3389 1.11 riastrad }
3390 1.11 riastrad
3391 1.11 riastrad static void
3392 1.11 riastrad wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp)
3393 1.11 riastrad {
3394 1.11 riastrad struct wg_session *wgs;
3395 1.104 riastrad uint32_t age;
3396 1.11 riastrad
3397 1.11 riastrad WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION");
3398 1.11 riastrad
3399 1.49 riastrad KASSERT(mutex_owned(wgp->wgp_lock));
3400 1.49 riastrad
3401 1.100 riastrad /*
3402 1.100 riastrad * If theres's any previous unstable session, i.e., one that
3403 1.100 riastrad * was ESTABLISHED and is now DESTROYING, older than
3404 1.100 riastrad * reject-after-time, destroy it. Upcoming sessions are still
3405 1.100 riastrad * in INIT_ACTIVE or INIT_PASSIVE -- we don't touch those here.
3406 1.104 riastrad *
3407 1.104 riastrad * No atomic for access to wgs_time_established because it is
3408 1.104 riastrad * only updated under wgp_lock.
3409 1.100 riastrad */
3410 1.11 riastrad wgs = wgp->wgp_session_unstable;
3411 1.100 riastrad KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED);
3412 1.100 riastrad if (wgs->wgs_state == WGS_STATE_DESTROYING &&
3413 1.104 riastrad ((age = (time_uptime32 - wgs->wgs_time_established)) >=
3414 1.100 riastrad wg_reject_after_time)) {
3415 1.104 riastrad WG_DLOG("destroying past session %"PRIu32" sec old\n", age);
3416 1.49 riastrad wg_put_session_index(wg, wgs);
3417 1.100 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
3418 1.100 riastrad wgs->wgs_state);
3419 1.100 riastrad }
3420 1.100 riastrad
3421 1.100 riastrad /*
3422 1.100 riastrad * If theres's any ESTABLISHED stable session older than
3423 1.100 riastrad * reject-after-time, destroy it. (The stable session can also
3424 1.100 riastrad * be in UNKNOWN state -- nothing to do in that case)
3425 1.100 riastrad */
3426 1.100 riastrad wgs = wgp->wgp_session_stable;
3427 1.100 riastrad KASSERT(wgs->wgs_state != WGS_STATE_INIT_ACTIVE);
3428 1.100 riastrad KASSERT(wgs->wgs_state != WGS_STATE_INIT_PASSIVE);
3429 1.100 riastrad KASSERT(wgs->wgs_state != WGS_STATE_DESTROYING);
3430 1.100 riastrad if (wgs->wgs_state == WGS_STATE_ESTABLISHED &&
3431 1.104 riastrad ((age = (time_uptime32 - wgs->wgs_time_established)) >=
3432 1.100 riastrad wg_reject_after_time)) {
3433 1.104 riastrad WG_DLOG("destroying current session %"PRIu32" sec old\n", age);
3434 1.100 riastrad atomic_store_relaxed(&wgs->wgs_state, WGS_STATE_DESTROYING);
3435 1.100 riastrad wg_put_session_index(wg, wgs);
3436 1.100 riastrad KASSERTMSG(wgs->wgs_state == WGS_STATE_UNKNOWN, "state=%d",
3437 1.100 riastrad wgs->wgs_state);
3438 1.11 riastrad }
3439 1.100 riastrad
3440 1.100 riastrad /*
3441 1.100 riastrad * If there's no sessions left, no need to have the timer run
3442 1.100 riastrad * until the next time around -- halt it.
3443 1.100 riastrad *
3444 1.100 riastrad * It is only ever scheduled with wgp_lock held or in the
3445 1.100 riastrad * callout itself, and callout_halt prevents rescheudling
3446 1.100 riastrad * itself, so this never races with rescheduling.
3447 1.100 riastrad */
3448 1.100 riastrad if (wgp->wgp_session_unstable->wgs_state == WGS_STATE_UNKNOWN &&
3449 1.100 riastrad wgp->wgp_session_stable->wgs_state == WGS_STATE_UNKNOWN)
3450 1.100 riastrad callout_halt(&wgp->wgp_session_dtor_timer, NULL);
3451 1.11 riastrad }
3452 1.11 riastrad
3453 1.11 riastrad static void
3454 1.55 riastrad wg_peer_work(struct work *wk, void *cookie)
3455 1.1 riastrad {
3456 1.55 riastrad struct wg_peer *wgp = container_of(wk, struct wg_peer, wgp_work);
3457 1.55 riastrad struct wg_softc *wg = wgp->wgp_sc;
3458 1.65 christos unsigned int tasks;
3459 1.1 riastrad
3460 1.55 riastrad mutex_enter(wgp->wgp_intr_lock);
3461 1.55 riastrad while ((tasks = wgp->wgp_tasks) != 0) {
3462 1.55 riastrad wgp->wgp_tasks = 0;
3463 1.55 riastrad mutex_exit(wgp->wgp_intr_lock);
3464 1.1 riastrad
3465 1.49 riastrad mutex_enter(wgp->wgp_lock);
3466 1.11 riastrad if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE))
3467 1.11 riastrad wg_task_send_init_message(wg, wgp);
3468 1.49 riastrad if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE))
3469 1.49 riastrad wg_task_retry_handshake(wg, wgp);
3470 1.49 riastrad if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION))
3471 1.49 riastrad wg_task_establish_session(wg, wgp);
3472 1.11 riastrad if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED))
3473 1.11 riastrad wg_task_endpoint_changed(wg, wgp);
3474 1.11 riastrad if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE))
3475 1.11 riastrad wg_task_send_keepalive_message(wg, wgp);
3476 1.11 riastrad if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION))
3477 1.11 riastrad wg_task_destroy_prev_session(wg, wgp);
3478 1.49 riastrad mutex_exit(wgp->wgp_lock);
3479 1.1 riastrad
3480 1.55 riastrad mutex_enter(wgp->wgp_intr_lock);
3481 1.1 riastrad }
3482 1.55 riastrad mutex_exit(wgp->wgp_intr_lock);
3483 1.1 riastrad }
3484 1.1 riastrad
3485 1.1 riastrad static void
3486 1.55 riastrad wg_job(struct threadpool_job *job)
3487 1.1 riastrad {
3488 1.55 riastrad struct wg_softc *wg = container_of(job, struct wg_softc, wg_job);
3489 1.55 riastrad int bound, upcalls;
3490 1.1 riastrad
3491 1.55 riastrad mutex_enter(wg->wg_intr_lock);
3492 1.55 riastrad while ((upcalls = wg->wg_upcalls) != 0) {
3493 1.55 riastrad wg->wg_upcalls = 0;
3494 1.55 riastrad mutex_exit(wg->wg_intr_lock);
3495 1.10 riastrad bound = curlwp_bind();
3496 1.55 riastrad if (ISSET(upcalls, WG_UPCALL_INET))
3497 1.1 riastrad wg_receive_packets(wg, AF_INET);
3498 1.55 riastrad if (ISSET(upcalls, WG_UPCALL_INET6))
3499 1.1 riastrad wg_receive_packets(wg, AF_INET6);
3500 1.10 riastrad curlwp_bindx(bound);
3501 1.55 riastrad mutex_enter(wg->wg_intr_lock);
3502 1.1 riastrad }
3503 1.55 riastrad threadpool_job_done(job);
3504 1.55 riastrad mutex_exit(wg->wg_intr_lock);
3505 1.1 riastrad }
3506 1.1 riastrad
3507 1.1 riastrad static int
3508 1.1 riastrad wg_bind_port(struct wg_softc *wg, const uint16_t port)
3509 1.1 riastrad {
3510 1.109 riastrad int error = 0;
3511 1.1 riastrad uint16_t old_port = wg->wg_listen_port;
3512 1.1 riastrad
3513 1.1 riastrad if (port != 0 && old_port == port)
3514 1.1 riastrad return 0;
3515 1.1 riastrad
3516 1.109 riastrad #ifdef INET
3517 1.1 riastrad struct sockaddr_in _sin, *sin = &_sin;
3518 1.1 riastrad sin->sin_len = sizeof(*sin);
3519 1.1 riastrad sin->sin_family = AF_INET;
3520 1.1 riastrad sin->sin_addr.s_addr = INADDR_ANY;
3521 1.1 riastrad sin->sin_port = htons(port);
3522 1.1 riastrad
3523 1.55 riastrad error = sobind(wg->wg_so4, sintosa(sin), curlwp);
3524 1.109 riastrad if (error)
3525 1.1 riastrad return error;
3526 1.109 riastrad #endif
3527 1.1 riastrad
3528 1.1 riastrad #ifdef INET6
3529 1.1 riastrad struct sockaddr_in6 _sin6, *sin6 = &_sin6;
3530 1.1 riastrad sin6->sin6_len = sizeof(*sin6);
3531 1.1 riastrad sin6->sin6_family = AF_INET6;
3532 1.1 riastrad sin6->sin6_addr = in6addr_any;
3533 1.1 riastrad sin6->sin6_port = htons(port);
3534 1.1 riastrad
3535 1.55 riastrad error = sobind(wg->wg_so6, sin6tosa(sin6), curlwp);
3536 1.109 riastrad if (error)
3537 1.1 riastrad return error;
3538 1.1 riastrad #endif
3539 1.1 riastrad
3540 1.1 riastrad wg->wg_listen_port = port;
3541 1.1 riastrad
3542 1.109 riastrad return error;
3543 1.1 riastrad }
3544 1.1 riastrad
3545 1.1 riastrad static void
3546 1.55 riastrad wg_so_upcall(struct socket *so, void *cookie, int events, int waitflag)
3547 1.1 riastrad {
3548 1.55 riastrad struct wg_softc *wg = cookie;
3549 1.1 riastrad int reason;
3550 1.1 riastrad
3551 1.1 riastrad reason = (so->so_proto->pr_domain->dom_family == AF_INET) ?
3552 1.55 riastrad WG_UPCALL_INET :
3553 1.55 riastrad WG_UPCALL_INET6;
3554 1.55 riastrad
3555 1.55 riastrad mutex_enter(wg->wg_intr_lock);
3556 1.55 riastrad wg->wg_upcalls |= reason;
3557 1.55 riastrad threadpool_schedule_job(wg->wg_threadpool, &wg->wg_job);
3558 1.55 riastrad mutex_exit(wg->wg_intr_lock);
3559 1.1 riastrad }
3560 1.1 riastrad
3561 1.1 riastrad static int
3562 1.1 riastrad wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so,
3563 1.1 riastrad struct sockaddr *src, void *arg)
3564 1.1 riastrad {
3565 1.1 riastrad struct wg_softc *wg = arg;
3566 1.2 riastrad struct wg_msg wgm;
3567 1.1 riastrad struct mbuf *m = *mp;
3568 1.1 riastrad
3569 1.1 riastrad WG_TRACE("enter");
3570 1.1 riastrad
3571 1.25 riastrad /* Verify the mbuf chain is long enough to have a wg msg header. */
3572 1.25 riastrad KASSERT(offset <= m_length(m));
3573 1.25 riastrad if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) {
3574 1.28 riastrad /* drop on the floor */
3575 1.25 riastrad m_freem(m);
3576 1.25 riastrad return -1;
3577 1.25 riastrad }
3578 1.25 riastrad
3579 1.25 riastrad /*
3580 1.25 riastrad * Copy the message header (32-bit message type) out -- we'll
3581 1.25 riastrad * worry about contiguity and alignment later.
3582 1.25 riastrad */
3583 1.2 riastrad m_copydata(m, offset, sizeof(struct wg_msg), &wgm);
3584 1.39 riastrad WG_DLOG("type=%d\n", le32toh(wgm.wgm_type));
3585 1.2 riastrad
3586 1.25 riastrad /*
3587 1.94 riastrad * Handle DATA packets promptly as they arrive, if they are in
3588 1.94 riastrad * an active session. Other packets may require expensive
3589 1.94 riastrad * public-key crypto and are not as sensitive to latency, so
3590 1.94 riastrad * defer them to the worker thread.
3591 1.25 riastrad */
3592 1.39 riastrad switch (le32toh(wgm.wgm_type)) {
3593 1.1 riastrad case WG_MSG_TYPE_DATA:
3594 1.28 riastrad /* handle immediately */
3595 1.1 riastrad m_adj(m, offset);
3596 1.29 riastrad if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) {
3597 1.29 riastrad m = m_pullup(m, sizeof(struct wg_msg_data));
3598 1.29 riastrad if (m == NULL)
3599 1.29 riastrad return -1;
3600 1.29 riastrad }
3601 1.1 riastrad wg_handle_msg_data(wg, m, src);
3602 1.1 riastrad *mp = NULL;
3603 1.1 riastrad return 1;
3604 1.28 riastrad case WG_MSG_TYPE_INIT:
3605 1.28 riastrad case WG_MSG_TYPE_RESP:
3606 1.28 riastrad case WG_MSG_TYPE_COOKIE:
3607 1.28 riastrad /* pass through to so_receive in wg_receive_packets */
3608 1.28 riastrad return 0;
3609 1.1 riastrad default:
3610 1.28 riastrad /* drop on the floor */
3611 1.28 riastrad m_freem(m);
3612 1.28 riastrad return -1;
3613 1.1 riastrad }
3614 1.1 riastrad }
3615 1.1 riastrad
3616 1.1 riastrad static int
3617 1.55 riastrad wg_socreate(struct wg_softc *wg, int af, struct socket **sop)
3618 1.1 riastrad {
3619 1.1 riastrad int error;
3620 1.1 riastrad struct socket *so;
3621 1.1 riastrad
3622 1.1 riastrad error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL);
3623 1.1 riastrad if (error != 0)
3624 1.1 riastrad return error;
3625 1.1 riastrad
3626 1.1 riastrad solock(so);
3627 1.55 riastrad so->so_upcallarg = wg;
3628 1.1 riastrad so->so_upcall = wg_so_upcall;
3629 1.1 riastrad so->so_rcv.sb_flags |= SB_UPCALL;
3630 1.71 ozaki inpcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg);
3631 1.1 riastrad sounlock(so);
3632 1.1 riastrad
3633 1.1 riastrad *sop = so;
3634 1.1 riastrad
3635 1.1 riastrad return 0;
3636 1.1 riastrad }
3637 1.1 riastrad
3638 1.1 riastrad static bool
3639 1.1 riastrad wg_session_hit_limits(struct wg_session *wgs)
3640 1.1 riastrad {
3641 1.104 riastrad uint32_t time_established =
3642 1.104 riastrad atomic_load_relaxed(&wgs->wgs_time_established);
3643 1.1 riastrad
3644 1.1 riastrad /*
3645 1.1 riastrad * [W] 6.2: Transport Message Limits
3646 1.1 riastrad * "After REJECT-AFTER-MESSAGES transport data messages or after the
3647 1.1 riastrad * current secure session is REJECT-AFTER-TIME seconds old, whichever
3648 1.106 riastrad * comes first, WireGuard will refuse to send or receive any more
3649 1.106 riastrad * transport data messages using the current secure session, ..."
3650 1.1 riastrad */
3651 1.104 riastrad KASSERT(time_established != 0 || time_uptime > UINT32_MAX);
3652 1.104 riastrad if ((time_uptime32 - time_established) > wg_reject_after_time) {
3653 1.1 riastrad WG_DLOG("The session hits REJECT_AFTER_TIME\n");
3654 1.1 riastrad return true;
3655 1.22 riastrad } else if (wg_session_get_send_counter(wgs) >
3656 1.22 riastrad wg_reject_after_messages) {
3657 1.1 riastrad WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n");
3658 1.1 riastrad return true;
3659 1.1 riastrad }
3660 1.1 riastrad
3661 1.1 riastrad return false;
3662 1.1 riastrad }
3663 1.1 riastrad
3664 1.1 riastrad static void
3665 1.54 riastrad wgintr(void *cookie)
3666 1.1 riastrad {
3667 1.54 riastrad struct wg_peer *wgp;
3668 1.1 riastrad struct wg_session *wgs;
3669 1.1 riastrad struct mbuf *m;
3670 1.1 riastrad struct psref psref;
3671 1.1 riastrad
3672 1.54 riastrad while ((m = pktq_dequeue(wg_pktq)) != NULL) {
3673 1.54 riastrad wgp = M_GETCTX(m, struct wg_peer *);
3674 1.54 riastrad if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) {
3675 1.54 riastrad WG_TRACE("no stable session");
3676 1.54 riastrad wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3677 1.54 riastrad goto next0;
3678 1.54 riastrad }
3679 1.54 riastrad if (__predict_false(wg_session_hit_limits(wgs))) {
3680 1.54 riastrad WG_TRACE("stable session hit limits");
3681 1.54 riastrad wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3682 1.54 riastrad goto next1;
3683 1.54 riastrad }
3684 1.1 riastrad wg_send_data_msg(wgp, wgs, m);
3685 1.54 riastrad m = NULL; /* consumed */
3686 1.54 riastrad next1: wg_put_session(wgs, &psref);
3687 1.79 rin next0: m_freem(m);
3688 1.54 riastrad /* XXX Yield to avoid userland starvation? */
3689 1.1 riastrad }
3690 1.1 riastrad }
3691 1.1 riastrad
3692 1.1 riastrad static void
3693 1.1 riastrad wg_purge_pending_packets(struct wg_peer *wgp)
3694 1.1 riastrad {
3695 1.1 riastrad struct mbuf *m;
3696 1.1 riastrad
3697 1.79 rin m = atomic_swap_ptr(&wgp->wgp_pending, NULL);
3698 1.79 rin m_freem(m);
3699 1.107 riastrad #ifdef ALTQ
3700 1.107 riastrad wg_start(&wgp->wgp_sc->wg_if);
3701 1.107 riastrad #endif
3702 1.54 riastrad pktq_barrier(wg_pktq);
3703 1.1 riastrad }
3704 1.1 riastrad
3705 1.1 riastrad static void
3706 1.1 riastrad wg_handshake_timeout_timer(void *arg)
3707 1.1 riastrad {
3708 1.1 riastrad struct wg_peer *wgp = arg;
3709 1.1 riastrad
3710 1.1 riastrad WG_TRACE("enter");
3711 1.1 riastrad
3712 1.49 riastrad wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE);
3713 1.1 riastrad }
3714 1.1 riastrad
3715 1.1 riastrad static struct wg_peer *
3716 1.1 riastrad wg_alloc_peer(struct wg_softc *wg)
3717 1.1 riastrad {
3718 1.1 riastrad struct wg_peer *wgp;
3719 1.1 riastrad
3720 1.1 riastrad wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP);
3721 1.1 riastrad
3722 1.1 riastrad wgp->wgp_sc = wg;
3723 1.1 riastrad callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE);
3724 1.1 riastrad callout_setfunc(&wgp->wgp_handshake_timeout_timer,
3725 1.1 riastrad wg_handshake_timeout_timer, wgp);
3726 1.1 riastrad callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE);
3727 1.1 riastrad callout_setfunc(&wgp->wgp_session_dtor_timer,
3728 1.1 riastrad wg_session_dtor_timer, wgp);
3729 1.1 riastrad PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry);
3730 1.1 riastrad wgp->wgp_endpoint_changing = false;
3731 1.1 riastrad wgp->wgp_endpoint_available = false;
3732 1.1 riastrad wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
3733 1.55 riastrad wgp->wgp_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
3734 1.1 riastrad wgp->wgp_psz = pserialize_create();
3735 1.1 riastrad psref_target_init(&wgp->wgp_psref, wg_psref_class);
3736 1.1 riastrad
3737 1.1 riastrad wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP);
3738 1.1 riastrad wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP);
3739 1.1 riastrad psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
3740 1.1 riastrad psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
3741 1.1 riastrad
3742 1.1 riastrad struct wg_session *wgs;
3743 1.14 riastrad wgp->wgp_session_stable =
3744 1.14 riastrad kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP);
3745 1.14 riastrad wgp->wgp_session_unstable =
3746 1.14 riastrad kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP);
3747 1.1 riastrad wgs = wgp->wgp_session_stable;
3748 1.1 riastrad wgs->wgs_peer = wgp;
3749 1.1 riastrad wgs->wgs_state = WGS_STATE_UNKNOWN;
3750 1.1 riastrad psref_target_init(&wgs->wgs_psref, wg_psref_class);
3751 1.22 riastrad #ifndef __HAVE_ATOMIC64_LOADSTORE
3752 1.22 riastrad mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
3753 1.22 riastrad #endif
3754 1.6 riastrad wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
3755 1.49 riastrad mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET);
3756 1.6 riastrad
3757 1.1 riastrad wgs = wgp->wgp_session_unstable;
3758 1.1 riastrad wgs->wgs_peer = wgp;
3759 1.1 riastrad wgs->wgs_state = WGS_STATE_UNKNOWN;
3760 1.1 riastrad psref_target_init(&wgs->wgs_psref, wg_psref_class);
3761 1.22 riastrad #ifndef __HAVE_ATOMIC64_LOADSTORE
3762 1.22 riastrad mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
3763 1.22 riastrad #endif
3764 1.6 riastrad wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
3765 1.49 riastrad mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET);
3766 1.1 riastrad
3767 1.1 riastrad return wgp;
3768 1.1 riastrad }
3769 1.1 riastrad
3770 1.1 riastrad static void
3771 1.1 riastrad wg_destroy_peer(struct wg_peer *wgp)
3772 1.1 riastrad {
3773 1.1 riastrad struct wg_session *wgs;
3774 1.1 riastrad struct wg_softc *wg = wgp->wgp_sc;
3775 1.1 riastrad
3776 1.37 riastrad /* Prevent new packets from this peer on any source address. */
3777 1.1 riastrad rw_enter(wg->wg_rwlock, RW_WRITER);
3778 1.1 riastrad for (int i = 0; i < wgp->wgp_n_allowedips; i++) {
3779 1.1 riastrad struct wg_allowedip *wga = &wgp->wgp_allowedips[i];
3780 1.1 riastrad struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family);
3781 1.1 riastrad struct radix_node *rn;
3782 1.1 riastrad
3783 1.1 riastrad KASSERT(rnh != NULL);
3784 1.1 riastrad rn = rnh->rnh_deladdr(&wga->wga_sa_addr,
3785 1.1 riastrad &wga->wga_sa_mask, rnh);
3786 1.1 riastrad if (rn == NULL) {
3787 1.1 riastrad char addrstr[128];
3788 1.1 riastrad sockaddr_format(&wga->wga_sa_addr, addrstr,
3789 1.1 riastrad sizeof(addrstr));
3790 1.76 jakllsch WGLOG(LOG_WARNING, "%s: Couldn't delete %s",
3791 1.76 jakllsch if_name(&wg->wg_if), addrstr);
3792 1.1 riastrad }
3793 1.1 riastrad }
3794 1.1 riastrad rw_exit(wg->wg_rwlock);
3795 1.1 riastrad
3796 1.38 riastrad /* Purge pending packets. */
3797 1.38 riastrad wg_purge_pending_packets(wgp);
3798 1.38 riastrad
3799 1.37 riastrad /* Halt all packet processing and timeouts. */
3800 1.1 riastrad callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
3801 1.1 riastrad callout_halt(&wgp->wgp_session_dtor_timer, NULL);
3802 1.1 riastrad
3803 1.55 riastrad /* Wait for any queued work to complete. */
3804 1.55 riastrad workqueue_wait(wg_wq, &wgp->wgp_work);
3805 1.55 riastrad
3806 1.49 riastrad wgs = wgp->wgp_session_unstable;
3807 1.49 riastrad if (wgs->wgs_state != WGS_STATE_UNKNOWN) {
3808 1.49 riastrad mutex_enter(wgp->wgp_lock);
3809 1.49 riastrad wg_destroy_session(wg, wgs);
3810 1.49 riastrad mutex_exit(wgp->wgp_lock);
3811 1.37 riastrad }
3812 1.6 riastrad mutex_destroy(&wgs->wgs_recvwin->lock);
3813 1.6 riastrad kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
3814 1.22 riastrad #ifndef __HAVE_ATOMIC64_LOADSTORE
3815 1.22 riastrad mutex_destroy(&wgs->wgs_send_counter_lock);
3816 1.22 riastrad #endif
3817 1.1 riastrad kmem_free(wgs, sizeof(*wgs));
3818 1.37 riastrad
3819 1.1 riastrad wgs = wgp->wgp_session_stable;
3820 1.49 riastrad if (wgs->wgs_state != WGS_STATE_UNKNOWN) {
3821 1.49 riastrad mutex_enter(wgp->wgp_lock);
3822 1.49 riastrad wg_destroy_session(wg, wgs);
3823 1.49 riastrad mutex_exit(wgp->wgp_lock);
3824 1.49 riastrad }
3825 1.6 riastrad mutex_destroy(&wgs->wgs_recvwin->lock);
3826 1.6 riastrad kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
3827 1.22 riastrad #ifndef __HAVE_ATOMIC64_LOADSTORE
3828 1.22 riastrad mutex_destroy(&wgs->wgs_send_counter_lock);
3829 1.22 riastrad #endif
3830 1.1 riastrad kmem_free(wgs, sizeof(*wgs));
3831 1.1 riastrad
3832 1.1 riastrad psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
3833 1.1 riastrad psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
3834 1.1 riastrad kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint));
3835 1.1 riastrad kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0));
3836 1.1 riastrad
3837 1.1 riastrad pserialize_destroy(wgp->wgp_psz);
3838 1.55 riastrad mutex_obj_free(wgp->wgp_intr_lock);
3839 1.1 riastrad mutex_obj_free(wgp->wgp_lock);
3840 1.1 riastrad
3841 1.1 riastrad kmem_free(wgp, sizeof(*wgp));
3842 1.1 riastrad }
3843 1.1 riastrad
3844 1.1 riastrad static void
3845 1.1 riastrad wg_destroy_all_peers(struct wg_softc *wg)
3846 1.1 riastrad {
3847 1.37 riastrad struct wg_peer *wgp, *wgp0 __diagused;
3848 1.37 riastrad void *garbage_byname, *garbage_bypubkey;
3849 1.1 riastrad
3850 1.1 riastrad restart:
3851 1.37 riastrad garbage_byname = garbage_bypubkey = NULL;
3852 1.1 riastrad mutex_enter(wg->wg_lock);
3853 1.1 riastrad WG_PEER_WRITER_FOREACH(wgp, wg) {
3854 1.37 riastrad if (wgp->wgp_name[0]) {
3855 1.37 riastrad wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name,
3856 1.37 riastrad strlen(wgp->wgp_name));
3857 1.37 riastrad KASSERT(wgp0 == wgp);
3858 1.37 riastrad garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
3859 1.37 riastrad }
3860 1.37 riastrad wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
3861 1.37 riastrad sizeof(wgp->wgp_pubkey));
3862 1.37 riastrad KASSERT(wgp0 == wgp);
3863 1.37 riastrad garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
3864 1.1 riastrad WG_PEER_WRITER_REMOVE(wgp);
3865 1.35 riastrad wg->wg_npeers--;
3866 1.1 riastrad mutex_enter(wgp->wgp_lock);
3867 1.1 riastrad pserialize_perform(wgp->wgp_psz);
3868 1.1 riastrad mutex_exit(wgp->wgp_lock);
3869 1.1 riastrad PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
3870 1.1 riastrad break;
3871 1.1 riastrad }
3872 1.1 riastrad mutex_exit(wg->wg_lock);
3873 1.1 riastrad
3874 1.1 riastrad if (wgp == NULL)
3875 1.1 riastrad return;
3876 1.1 riastrad
3877 1.1 riastrad psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
3878 1.1 riastrad
3879 1.1 riastrad wg_destroy_peer(wgp);
3880 1.37 riastrad thmap_gc(wg->wg_peers_byname, garbage_byname);
3881 1.37 riastrad thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey);
3882 1.1 riastrad
3883 1.1 riastrad goto restart;
3884 1.1 riastrad }
3885 1.1 riastrad
3886 1.1 riastrad static int
3887 1.1 riastrad wg_destroy_peer_name(struct wg_softc *wg, const char *name)
3888 1.1 riastrad {
3889 1.37 riastrad struct wg_peer *wgp, *wgp0 __diagused;
3890 1.37 riastrad void *garbage_byname, *garbage_bypubkey;
3891 1.1 riastrad
3892 1.1 riastrad mutex_enter(wg->wg_lock);
3893 1.37 riastrad wgp = thmap_del(wg->wg_peers_byname, name, strlen(name));
3894 1.1 riastrad if (wgp != NULL) {
3895 1.37 riastrad wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
3896 1.37 riastrad sizeof(wgp->wgp_pubkey));
3897 1.37 riastrad KASSERT(wgp0 == wgp);
3898 1.37 riastrad garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
3899 1.37 riastrad garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
3900 1.1 riastrad WG_PEER_WRITER_REMOVE(wgp);
3901 1.1 riastrad wg->wg_npeers--;
3902 1.61 roy if (wg->wg_npeers == 0)
3903 1.61 roy if_link_state_change(&wg->wg_if, LINK_STATE_DOWN);
3904 1.1 riastrad mutex_enter(wgp->wgp_lock);
3905 1.1 riastrad pserialize_perform(wgp->wgp_psz);
3906 1.1 riastrad mutex_exit(wgp->wgp_lock);
3907 1.1 riastrad PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
3908 1.1 riastrad }
3909 1.1 riastrad mutex_exit(wg->wg_lock);
3910 1.1 riastrad
3911 1.1 riastrad if (wgp == NULL)
3912 1.1 riastrad return ENOENT;
3913 1.1 riastrad
3914 1.1 riastrad psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
3915 1.1 riastrad
3916 1.1 riastrad wg_destroy_peer(wgp);
3917 1.37 riastrad thmap_gc(wg->wg_peers_byname, garbage_byname);
3918 1.37 riastrad thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey);
3919 1.1 riastrad
3920 1.1 riastrad return 0;
3921 1.1 riastrad }
3922 1.1 riastrad
3923 1.1 riastrad static int
3924 1.1 riastrad wg_if_attach(struct wg_softc *wg)
3925 1.1 riastrad {
3926 1.1 riastrad
3927 1.1 riastrad wg->wg_if.if_addrlen = 0;
3928 1.1 riastrad wg->wg_if.if_mtu = WG_MTU;
3929 1.33 riastrad wg->wg_if.if_flags = IFF_MULTICAST;
3930 1.61 roy wg->wg_if.if_extflags = IFEF_MPSAFE;
3931 1.1 riastrad wg->wg_if.if_ioctl = wg_ioctl;
3932 1.1 riastrad wg->wg_if.if_output = wg_output;
3933 1.1 riastrad wg->wg_if.if_init = wg_init;
3934 1.60 riastrad #ifdef ALTQ
3935 1.60 riastrad wg->wg_if.if_start = wg_start;
3936 1.60 riastrad #endif
3937 1.1 riastrad wg->wg_if.if_stop = wg_stop;
3938 1.24 riastrad wg->wg_if.if_type = IFT_OTHER;
3939 1.1 riastrad wg->wg_if.if_dlt = DLT_NULL;
3940 1.1 riastrad wg->wg_if.if_softc = wg;
3941 1.60 riastrad #ifdef ALTQ
3942 1.1 riastrad IFQ_SET_READY(&wg->wg_if.if_snd);
3943 1.60 riastrad #endif
3944 1.64 riastrad if_initialize(&wg->wg_if);
3945 1.1 riastrad
3946 1.61 roy wg->wg_if.if_link_state = LINK_STATE_DOWN;
3947 1.1 riastrad if_alloc_sadl(&wg->wg_if);
3948 1.1 riastrad if_register(&wg->wg_if);
3949 1.1 riastrad
3950 1.1 riastrad bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t));
3951 1.1 riastrad
3952 1.1 riastrad return 0;
3953 1.1 riastrad }
3954 1.1 riastrad
3955 1.54 riastrad static void
3956 1.54 riastrad wg_if_detach(struct wg_softc *wg)
3957 1.54 riastrad {
3958 1.54 riastrad struct ifnet *ifp = &wg->wg_if;
3959 1.54 riastrad
3960 1.54 riastrad bpf_detach(ifp);
3961 1.54 riastrad if_detach(ifp);
3962 1.54 riastrad }
3963 1.54 riastrad
3964 1.1 riastrad static int
3965 1.1 riastrad wg_clone_create(struct if_clone *ifc, int unit)
3966 1.1 riastrad {
3967 1.1 riastrad struct wg_softc *wg;
3968 1.1 riastrad int error;
3969 1.1 riastrad
3970 1.58 riastrad wg_guarantee_initialized();
3971 1.58 riastrad
3972 1.59 riastrad error = wg_count_inc();
3973 1.59 riastrad if (error)
3974 1.59 riastrad return error;
3975 1.59 riastrad
3976 1.54 riastrad wg = kmem_zalloc(sizeof(*wg), KM_SLEEP);
3977 1.1 riastrad
3978 1.1 riastrad if_initname(&wg->wg_if, ifc->ifc_name, unit);
3979 1.1 riastrad
3980 1.55 riastrad PSLIST_INIT(&wg->wg_peers);
3981 1.55 riastrad wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY);
3982 1.55 riastrad wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY);
3983 1.55 riastrad wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY);
3984 1.55 riastrad wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
3985 1.55 riastrad wg->wg_intr_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
3986 1.55 riastrad wg->wg_rwlock = rw_obj_alloc();
3987 1.55 riastrad threadpool_job_init(&wg->wg_job, wg_job, wg->wg_intr_lock,
3988 1.55 riastrad "%s", if_name(&wg->wg_if));
3989 1.55 riastrad wg->wg_ops = &wg_ops_rumpkernel;
3990 1.55 riastrad
3991 1.55 riastrad error = threadpool_get(&wg->wg_threadpool, PRI_NONE);
3992 1.54 riastrad if (error)
3993 1.54 riastrad goto fail0;
3994 1.1 riastrad
3995 1.55 riastrad #ifdef INET
3996 1.55 riastrad error = wg_socreate(wg, AF_INET, &wg->wg_so4);
3997 1.55 riastrad if (error)
3998 1.55 riastrad goto fail1;
3999 1.1 riastrad rn_inithead((void **)&wg->wg_rtable_ipv4,
4000 1.1 riastrad offsetof(struct sockaddr_in, sin_addr) * NBBY);
4001 1.55 riastrad #endif
4002 1.1 riastrad #ifdef INET6
4003 1.55 riastrad error = wg_socreate(wg, AF_INET6, &wg->wg_so6);
4004 1.55 riastrad if (error)
4005 1.55 riastrad goto fail2;
4006 1.1 riastrad rn_inithead((void **)&wg->wg_rtable_ipv6,
4007 1.1 riastrad offsetof(struct sockaddr_in6, sin6_addr) * NBBY);
4008 1.1 riastrad #endif
4009 1.1 riastrad
4010 1.1 riastrad error = wg_if_attach(wg);
4011 1.54 riastrad if (error)
4012 1.55 riastrad goto fail3;
4013 1.1 riastrad
4014 1.1 riastrad return 0;
4015 1.54 riastrad
4016 1.55 riastrad fail4: __unused
4017 1.107 riastrad wg_destroy_all_peers(wg);
4018 1.54 riastrad wg_if_detach(wg);
4019 1.107 riastrad fail3:
4020 1.55 riastrad #ifdef INET6
4021 1.55 riastrad solock(wg->wg_so6);
4022 1.55 riastrad wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL;
4023 1.55 riastrad sounlock(wg->wg_so6);
4024 1.55 riastrad #endif
4025 1.55 riastrad #ifdef INET
4026 1.55 riastrad solock(wg->wg_so4);
4027 1.55 riastrad wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL;
4028 1.55 riastrad sounlock(wg->wg_so4);
4029 1.55 riastrad #endif
4030 1.55 riastrad mutex_enter(wg->wg_intr_lock);
4031 1.55 riastrad threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job);
4032 1.55 riastrad mutex_exit(wg->wg_intr_lock);
4033 1.55 riastrad #ifdef INET6
4034 1.55 riastrad if (wg->wg_rtable_ipv6 != NULL)
4035 1.55 riastrad free(wg->wg_rtable_ipv6, M_RTABLE);
4036 1.55 riastrad soclose(wg->wg_so6);
4037 1.55 riastrad fail2:
4038 1.55 riastrad #endif
4039 1.55 riastrad #ifdef INET
4040 1.55 riastrad if (wg->wg_rtable_ipv4 != NULL)
4041 1.55 riastrad free(wg->wg_rtable_ipv4, M_RTABLE);
4042 1.55 riastrad soclose(wg->wg_so4);
4043 1.55 riastrad fail1:
4044 1.55 riastrad #endif
4045 1.55 riastrad threadpool_put(wg->wg_threadpool, PRI_NONE);
4046 1.55 riastrad fail0: threadpool_job_destroy(&wg->wg_job);
4047 1.54 riastrad rw_obj_free(wg->wg_rwlock);
4048 1.55 riastrad mutex_obj_free(wg->wg_intr_lock);
4049 1.54 riastrad mutex_obj_free(wg->wg_lock);
4050 1.54 riastrad thmap_destroy(wg->wg_sessions_byindex);
4051 1.54 riastrad thmap_destroy(wg->wg_peers_byname);
4052 1.54 riastrad thmap_destroy(wg->wg_peers_bypubkey);
4053 1.54 riastrad PSLIST_DESTROY(&wg->wg_peers);
4054 1.55 riastrad kmem_free(wg, sizeof(*wg));
4055 1.59 riastrad wg_count_dec();
4056 1.54 riastrad return error;
4057 1.1 riastrad }
4058 1.1 riastrad
4059 1.1 riastrad static int
4060 1.1 riastrad wg_clone_destroy(struct ifnet *ifp)
4061 1.1 riastrad {
4062 1.16 riastrad struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if);
4063 1.1 riastrad
4064 1.1 riastrad #ifdef WG_RUMPKERNEL
4065 1.1 riastrad if (wg_user_mode(wg)) {
4066 1.1 riastrad rumpuser_wg_destroy(wg->wg_user);
4067 1.1 riastrad wg->wg_user = NULL;
4068 1.1 riastrad }
4069 1.1 riastrad #endif
4070 1.1 riastrad
4071 1.107 riastrad wg_destroy_all_peers(wg);
4072 1.54 riastrad wg_if_detach(wg);
4073 1.55 riastrad #ifdef INET6
4074 1.55 riastrad solock(wg->wg_so6);
4075 1.55 riastrad wg->wg_so6->so_rcv.sb_flags &= ~SB_UPCALL;
4076 1.55 riastrad sounlock(wg->wg_so6);
4077 1.55 riastrad #endif
4078 1.55 riastrad #ifdef INET
4079 1.55 riastrad solock(wg->wg_so4);
4080 1.55 riastrad wg->wg_so4->so_rcv.sb_flags &= ~SB_UPCALL;
4081 1.55 riastrad sounlock(wg->wg_so4);
4082 1.55 riastrad #endif
4083 1.55 riastrad mutex_enter(wg->wg_intr_lock);
4084 1.55 riastrad threadpool_cancel_job(wg->wg_threadpool, &wg->wg_job);
4085 1.55 riastrad mutex_exit(wg->wg_intr_lock);
4086 1.55 riastrad #ifdef INET6
4087 1.55 riastrad if (wg->wg_rtable_ipv6 != NULL)
4088 1.55 riastrad free(wg->wg_rtable_ipv6, M_RTABLE);
4089 1.55 riastrad soclose(wg->wg_so6);
4090 1.55 riastrad #endif
4091 1.55 riastrad #ifdef INET
4092 1.55 riastrad if (wg->wg_rtable_ipv4 != NULL)
4093 1.55 riastrad free(wg->wg_rtable_ipv4, M_RTABLE);
4094 1.55 riastrad soclose(wg->wg_so4);
4095 1.55 riastrad #endif
4096 1.55 riastrad threadpool_put(wg->wg_threadpool, PRI_NONE);
4097 1.55 riastrad threadpool_job_destroy(&wg->wg_job);
4098 1.54 riastrad rw_obj_free(wg->wg_rwlock);
4099 1.55 riastrad mutex_obj_free(wg->wg_intr_lock);
4100 1.54 riastrad mutex_obj_free(wg->wg_lock);
4101 1.54 riastrad thmap_destroy(wg->wg_sessions_byindex);
4102 1.54 riastrad thmap_destroy(wg->wg_peers_byname);
4103 1.54 riastrad thmap_destroy(wg->wg_peers_bypubkey);
4104 1.54 riastrad PSLIST_DESTROY(&wg->wg_peers);
4105 1.54 riastrad kmem_free(wg, sizeof(*wg));
4106 1.59 riastrad wg_count_dec();
4107 1.1 riastrad
4108 1.1 riastrad return 0;
4109 1.1 riastrad }
4110 1.1 riastrad
4111 1.1 riastrad static struct wg_peer *
4112 1.1 riastrad wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa,
4113 1.1 riastrad struct psref *psref)
4114 1.1 riastrad {
4115 1.1 riastrad struct radix_node_head *rnh;
4116 1.1 riastrad struct radix_node *rn;
4117 1.1 riastrad struct wg_peer *wgp = NULL;
4118 1.1 riastrad struct wg_allowedip *wga;
4119 1.1 riastrad
4120 1.1 riastrad #ifdef WG_DEBUG_LOG
4121 1.1 riastrad char addrstr[128];
4122 1.1 riastrad sockaddr_format(sa, addrstr, sizeof(addrstr));
4123 1.1 riastrad WG_DLOG("sa=%s\n", addrstr);
4124 1.1 riastrad #endif
4125 1.1 riastrad
4126 1.1 riastrad rw_enter(wg->wg_rwlock, RW_READER);
4127 1.1 riastrad
4128 1.1 riastrad rnh = wg_rnh(wg, sa->sa_family);
4129 1.1 riastrad if (rnh == NULL)
4130 1.1 riastrad goto out;
4131 1.1 riastrad
4132 1.1 riastrad rn = rnh->rnh_matchaddr(sa, rnh);
4133 1.1 riastrad if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0)
4134 1.1 riastrad goto out;
4135 1.1 riastrad
4136 1.1 riastrad WG_TRACE("success");
4137 1.1 riastrad
4138 1.16 riastrad wga = container_of(rn, struct wg_allowedip, wga_nodes[0]);
4139 1.1 riastrad wgp = wga->wga_peer;
4140 1.1 riastrad wg_get_peer(wgp, psref);
4141 1.1 riastrad
4142 1.1 riastrad out:
4143 1.1 riastrad rw_exit(wg->wg_rwlock);
4144 1.1 riastrad return wgp;
4145 1.1 riastrad }
4146 1.1 riastrad
4147 1.1 riastrad static void
4148 1.1 riastrad wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp,
4149 1.1 riastrad struct wg_session *wgs, struct wg_msg_data *wgmd)
4150 1.1 riastrad {
4151 1.1 riastrad
4152 1.1 riastrad memset(wgmd, 0, sizeof(*wgmd));
4153 1.39 riastrad wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA);
4154 1.49 riastrad wgmd->wgmd_receiver = wgs->wgs_remote_index;
4155 1.1 riastrad /* [W] 5.4.6: msg.counter := Nm^send */
4156 1.1 riastrad /* [W] 5.4.6: Nm^send := Nm^send + 1 */
4157 1.39 riastrad wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs));
4158 1.39 riastrad WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter));
4159 1.1 riastrad }
4160 1.1 riastrad
4161 1.1 riastrad static int
4162 1.1 riastrad wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
4163 1.1 riastrad const struct rtentry *rt)
4164 1.1 riastrad {
4165 1.1 riastrad struct wg_softc *wg = ifp->if_softc;
4166 1.49 riastrad struct wg_peer *wgp = NULL;
4167 1.49 riastrad struct wg_session *wgs = NULL;
4168 1.49 riastrad struct psref wgp_psref, wgs_psref;
4169 1.1 riastrad int bound;
4170 1.49 riastrad int error;
4171 1.49 riastrad
4172 1.49 riastrad bound = curlwp_bind();
4173 1.1 riastrad
4174 1.1 riastrad /* TODO make the nest limit configurable via sysctl */
4175 1.1 riastrad error = if_tunnel_check_nesting(ifp, m, 1);
4176 1.49 riastrad if (error) {
4177 1.76 jakllsch WGLOG(LOG_ERR,
4178 1.76 jakllsch "%s: tunneling loop detected and packet dropped\n",
4179 1.76 jakllsch if_name(&wg->wg_if));
4180 1.54 riastrad goto out0;
4181 1.1 riastrad }
4182 1.1 riastrad
4183 1.60 riastrad #ifdef ALTQ
4184 1.60 riastrad bool altq = atomic_load_relaxed(&ifp->if_snd.altq_flags)
4185 1.60 riastrad & ALTQF_ENABLED;
4186 1.60 riastrad if (altq)
4187 1.60 riastrad IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family);
4188 1.60 riastrad #endif
4189 1.1 riastrad
4190 1.1 riastrad bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT);
4191 1.1 riastrad
4192 1.1 riastrad m->m_flags &= ~(M_BCAST|M_MCAST);
4193 1.1 riastrad
4194 1.49 riastrad wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref);
4195 1.1 riastrad if (wgp == NULL) {
4196 1.1 riastrad WG_TRACE("peer not found");
4197 1.1 riastrad error = EHOSTUNREACH;
4198 1.54 riastrad goto out0;
4199 1.1 riastrad }
4200 1.1 riastrad
4201 1.1 riastrad /* Clear checksum-offload flags. */
4202 1.1 riastrad m->m_pkthdr.csum_flags = 0;
4203 1.1 riastrad m->m_pkthdr.csum_data = 0;
4204 1.1 riastrad
4205 1.54 riastrad /* Check whether there's an established session. */
4206 1.54 riastrad wgs = wg_get_stable_session(wgp, &wgs_psref);
4207 1.54 riastrad if (wgs == NULL) {
4208 1.54 riastrad /*
4209 1.54 riastrad * No established session. If we're the first to try
4210 1.54 riastrad * sending data, schedule a handshake and queue the
4211 1.54 riastrad * packet for when the handshake is done; otherwise
4212 1.54 riastrad * just drop the packet and let the ongoing handshake
4213 1.54 riastrad * attempt continue. We could queue more data packets
4214 1.54 riastrad * but it's not clear that's worthwhile.
4215 1.54 riastrad */
4216 1.115 riastrad if ((m = atomic_swap_ptr(&wgp->wgp_pending, m)) == NULL) {
4217 1.54 riastrad WG_TRACE("queued first packet; init handshake");
4218 1.54 riastrad wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
4219 1.54 riastrad } else {
4220 1.54 riastrad WG_TRACE("first packet already queued, dropping");
4221 1.54 riastrad }
4222 1.54 riastrad goto out1;
4223 1.54 riastrad }
4224 1.54 riastrad
4225 1.54 riastrad /* There's an established session. Toss it in the queue. */
4226 1.60 riastrad #ifdef ALTQ
4227 1.60 riastrad if (altq) {
4228 1.60 riastrad mutex_enter(ifp->if_snd.ifq_lock);
4229 1.60 riastrad if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
4230 1.60 riastrad M_SETCTX(m, wgp);
4231 1.60 riastrad ALTQ_ENQUEUE(&ifp->if_snd, m, error);
4232 1.60 riastrad m = NULL; /* consume */
4233 1.60 riastrad }
4234 1.60 riastrad mutex_exit(ifp->if_snd.ifq_lock);
4235 1.60 riastrad if (m == NULL) {
4236 1.60 riastrad wg_start(ifp);
4237 1.60 riastrad goto out2;
4238 1.60 riastrad }
4239 1.60 riastrad }
4240 1.60 riastrad #endif
4241 1.54 riastrad kpreempt_disable();
4242 1.54 riastrad const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m)
4243 1.54 riastrad M_SETCTX(m, wgp);
4244 1.54 riastrad if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) {
4245 1.76 jakllsch WGLOG(LOG_ERR, "%s: pktq full, dropping\n",
4246 1.76 jakllsch if_name(&wg->wg_if));
4247 1.1 riastrad error = ENOBUFS;
4248 1.60 riastrad goto out3;
4249 1.1 riastrad }
4250 1.49 riastrad m = NULL; /* consumed */
4251 1.49 riastrad error = 0;
4252 1.60 riastrad out3: kpreempt_enable();
4253 1.1 riastrad
4254 1.60 riastrad #ifdef ALTQ
4255 1.60 riastrad out2:
4256 1.60 riastrad #endif
4257 1.54 riastrad wg_put_session(wgs, &wgs_psref);
4258 1.54 riastrad out1: wg_put_peer(wgp, &wgp_psref);
4259 1.79 rin out0: m_freem(m);
4260 1.1 riastrad curlwp_bindx(bound);
4261 1.1 riastrad return error;
4262 1.1 riastrad }
4263 1.1 riastrad
4264 1.1 riastrad static int
4265 1.1 riastrad wg_send_udp(struct wg_peer *wgp, struct mbuf *m)
4266 1.1 riastrad {
4267 1.1 riastrad struct psref psref;
4268 1.1 riastrad struct wg_sockaddr *wgsa;
4269 1.1 riastrad int error;
4270 1.47 riastrad struct socket *so;
4271 1.1 riastrad
4272 1.47 riastrad wgsa = wg_get_endpoint_sa(wgp, &psref);
4273 1.47 riastrad so = wg_get_so_by_peer(wgp, wgsa);
4274 1.1 riastrad solock(so);
4275 1.109 riastrad switch (wgsatosa(wgsa)->sa_family) {
4276 1.109 riastrad #ifdef INET
4277 1.109 riastrad case AF_INET:
4278 1.1 riastrad error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp);
4279 1.109 riastrad break;
4280 1.109 riastrad #endif
4281 1.1 riastrad #ifdef INET6
4282 1.109 riastrad case AF_INET6:
4283 1.70 ozaki error = udp6_output(sotoinpcb(so), m, wgsatosin6(wgsa),
4284 1.1 riastrad NULL, curlwp);
4285 1.109 riastrad break;
4286 1.109 riastrad #endif
4287 1.109 riastrad default:
4288 1.38 riastrad m_freem(m);
4289 1.47 riastrad error = EPFNOSUPPORT;
4290 1.1 riastrad }
4291 1.47 riastrad sounlock(so);
4292 1.1 riastrad wg_put_sa(wgp, wgsa, &psref);
4293 1.1 riastrad
4294 1.1 riastrad return error;
4295 1.1 riastrad }
4296 1.1 riastrad
4297 1.1 riastrad /* Inspired by pppoe_get_mbuf */
4298 1.1 riastrad static struct mbuf *
4299 1.1 riastrad wg_get_mbuf(size_t leading_len, size_t len)
4300 1.1 riastrad {
4301 1.1 riastrad struct mbuf *m;
4302 1.1 riastrad
4303 1.30 riastrad KASSERT(leading_len <= MCLBYTES);
4304 1.30 riastrad KASSERT(len <= MCLBYTES - leading_len);
4305 1.30 riastrad
4306 1.1 riastrad m = m_gethdr(M_DONTWAIT, MT_DATA);
4307 1.1 riastrad if (m == NULL)
4308 1.1 riastrad return NULL;
4309 1.1 riastrad if (len + leading_len > MHLEN) {
4310 1.1 riastrad m_clget(m, M_DONTWAIT);
4311 1.1 riastrad if ((m->m_flags & M_EXT) == 0) {
4312 1.1 riastrad m_free(m);
4313 1.1 riastrad return NULL;
4314 1.1 riastrad }
4315 1.1 riastrad }
4316 1.1 riastrad m->m_data += leading_len;
4317 1.1 riastrad m->m_pkthdr.len = m->m_len = len;
4318 1.1 riastrad
4319 1.1 riastrad return m;
4320 1.1 riastrad }
4321 1.1 riastrad
4322 1.108 riastrad static void
4323 1.108 riastrad wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, struct mbuf *m)
4324 1.1 riastrad {
4325 1.1 riastrad struct wg_softc *wg = wgp->wgp_sc;
4326 1.1 riastrad int error;
4327 1.1 riastrad size_t inner_len, padded_len, encrypted_len;
4328 1.1 riastrad char *padded_buf = NULL;
4329 1.1 riastrad size_t mlen;
4330 1.1 riastrad struct wg_msg_data *wgmd;
4331 1.1 riastrad bool free_padded_buf = false;
4332 1.1 riastrad struct mbuf *n;
4333 1.62 riastrad size_t leading_len = max_hdr + sizeof(struct udphdr);
4334 1.1 riastrad
4335 1.1 riastrad mlen = m_length(m);
4336 1.1 riastrad inner_len = mlen;
4337 1.2 riastrad padded_len = roundup(mlen, 16);
4338 1.2 riastrad encrypted_len = padded_len + WG_AUTHTAG_LEN;
4339 1.87 kre WG_DLOG("inner=%zu, padded=%zu, encrypted_len=%zu\n",
4340 1.1 riastrad inner_len, padded_len, encrypted_len);
4341 1.1 riastrad if (mlen != 0) {
4342 1.1 riastrad bool success;
4343 1.1 riastrad success = m_ensure_contig(&m, padded_len);
4344 1.1 riastrad if (success) {
4345 1.1 riastrad padded_buf = mtod(m, char *);
4346 1.1 riastrad } else {
4347 1.1 riastrad padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP);
4348 1.1 riastrad if (padded_buf == NULL) {
4349 1.1 riastrad error = ENOBUFS;
4350 1.108 riastrad goto out;
4351 1.1 riastrad }
4352 1.1 riastrad free_padded_buf = true;
4353 1.1 riastrad m_copydata(m, 0, mlen, padded_buf);
4354 1.1 riastrad }
4355 1.1 riastrad memset(padded_buf + mlen, 0, padded_len - inner_len);
4356 1.1 riastrad }
4357 1.1 riastrad
4358 1.1 riastrad n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len);
4359 1.1 riastrad if (n == NULL) {
4360 1.1 riastrad error = ENOBUFS;
4361 1.108 riastrad goto out;
4362 1.1 riastrad }
4363 1.27 riastrad KASSERT(n->m_len >= sizeof(*wgmd));
4364 1.1 riastrad wgmd = mtod(n, struct wg_msg_data *);
4365 1.1 riastrad wg_fill_msg_data(wg, wgp, wgs, wgmd);
4366 1.111 riastrad
4367 1.1 riastrad /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */
4368 1.1 riastrad wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len,
4369 1.39 riastrad wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter),
4370 1.39 riastrad padded_buf, padded_len,
4371 1.1 riastrad NULL, 0);
4372 1.1 riastrad
4373 1.108 riastrad error = wg->wg_ops->send_data_msg(wgp, n); /* consumes n */
4374 1.108 riastrad if (error) {
4375 1.108 riastrad WG_DLOG("send_data_msg failed, error=%d\n", error);
4376 1.108 riastrad goto out;
4377 1.108 riastrad }
4378 1.108 riastrad
4379 1.108 riastrad /*
4380 1.108 riastrad * Packet was sent out -- count it in the interface statistics.
4381 1.108 riastrad */
4382 1.108 riastrad if_statadd(&wg->wg_if, if_obytes, mlen);
4383 1.108 riastrad if_statinc(&wg->wg_if, if_opackets);
4384 1.108 riastrad
4385 1.108 riastrad /*
4386 1.108 riastrad * Record when we last sent data, for determining when we need
4387 1.108 riastrad * to send a passive keepalive.
4388 1.108 riastrad *
4389 1.108 riastrad * Other logic assumes that wgs_time_last_data_sent is zero iff
4390 1.108 riastrad * we have never sent data on this session. Early at boot, if
4391 1.108 riastrad * wg(4) starts operating within <1sec, or after 136 years of
4392 1.108 riastrad * uptime, we may observe time_uptime32 = 0. In that case,
4393 1.108 riastrad * pretend we observed 1 instead. That way, we correctly
4394 1.108 riastrad * indicate we have sent data on this session; the only logic
4395 1.108 riastrad * this might adversely affect is the keepalive timeout
4396 1.108 riastrad * detection, which might spuriously send a keepalive during
4397 1.108 riastrad * one second every 136 years. All of this is very silly, of
4398 1.108 riastrad * course, but the cost to guaranteeing wgs_time_last_data_sent
4399 1.108 riastrad * is nonzero is negligible here.
4400 1.108 riastrad */
4401 1.108 riastrad const uint32_t now = time_uptime32;
4402 1.108 riastrad atomic_store_relaxed(&wgs->wgs_time_last_data_sent, MAX(now, 1));
4403 1.108 riastrad
4404 1.108 riastrad /*
4405 1.108 riastrad * Check rekey-after-time.
4406 1.108 riastrad */
4407 1.108 riastrad if (wgs->wgs_is_initiator &&
4408 1.108 riastrad ((time_uptime32 -
4409 1.108 riastrad atomic_load_relaxed(&wgs->wgs_time_established)) >=
4410 1.108 riastrad wg_rekey_after_time)) {
4411 1.108 riastrad /*
4412 1.108 riastrad * [W] 6.2 Transport Message Limits
4413 1.108 riastrad * "if a peer is the initiator of a current secure
4414 1.108 riastrad * session, WireGuard will send a handshake initiation
4415 1.108 riastrad * message to begin a new secure session if, after
4416 1.108 riastrad * transmitting a transport data message, the current
4417 1.108 riastrad * secure session is REKEY-AFTER-TIME seconds old,"
4418 1.108 riastrad */
4419 1.108 riastrad WG_TRACE("rekey after time");
4420 1.113 riastrad atomic_store_relaxed(&wgs->wgs_force_rekey, true);
4421 1.108 riastrad wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
4422 1.108 riastrad }
4423 1.108 riastrad
4424 1.108 riastrad /*
4425 1.108 riastrad * Check rekey-after-messages.
4426 1.108 riastrad */
4427 1.108 riastrad if (wg_session_get_send_counter(wgs) >= wg_rekey_after_messages) {
4428 1.108 riastrad /*
4429 1.108 riastrad * [W] 6.2 Transport Message Limits
4430 1.108 riastrad * "WireGuard will try to create a new session, by
4431 1.108 riastrad * sending a handshake initiation message (section
4432 1.108 riastrad * 5.4.2), after it has sent REKEY-AFTER-MESSAGES
4433 1.108 riastrad * transport data messages..."
4434 1.108 riastrad */
4435 1.108 riastrad WG_TRACE("rekey after messages");
4436 1.113 riastrad atomic_store_relaxed(&wgs->wgs_force_rekey, true);
4437 1.108 riastrad wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
4438 1.1 riastrad }
4439 1.108 riastrad
4440 1.108 riastrad out: m_freem(m);
4441 1.1 riastrad if (free_padded_buf)
4442 1.1 riastrad kmem_intr_free(padded_buf, padded_len);
4443 1.1 riastrad }
4444 1.1 riastrad
4445 1.1 riastrad static void
4446 1.1 riastrad wg_input(struct ifnet *ifp, struct mbuf *m, const int af)
4447 1.1 riastrad {
4448 1.1 riastrad pktqueue_t *pktq;
4449 1.1 riastrad size_t pktlen;
4450 1.1 riastrad
4451 1.1 riastrad KASSERT(af == AF_INET || af == AF_INET6);
4452 1.1 riastrad
4453 1.1 riastrad WG_TRACE("");
4454 1.1 riastrad
4455 1.1 riastrad m_set_rcvif(m, ifp);
4456 1.1 riastrad pktlen = m->m_pkthdr.len;
4457 1.1 riastrad
4458 1.1 riastrad bpf_mtap_af(ifp, af, m, BPF_D_IN);
4459 1.1 riastrad
4460 1.1 riastrad switch (af) {
4461 1.109 riastrad #ifdef INET
4462 1.1 riastrad case AF_INET:
4463 1.1 riastrad pktq = ip_pktq;
4464 1.1 riastrad break;
4465 1.109 riastrad #endif
4466 1.1 riastrad #ifdef INET6
4467 1.1 riastrad case AF_INET6:
4468 1.1 riastrad pktq = ip6_pktq;
4469 1.1 riastrad break;
4470 1.1 riastrad #endif
4471 1.1 riastrad default:
4472 1.1 riastrad panic("invalid af=%d", af);
4473 1.1 riastrad }
4474 1.1 riastrad
4475 1.57 riastrad kpreempt_disable();
4476 1.1 riastrad const u_int h = curcpu()->ci_index;
4477 1.1 riastrad if (__predict_true(pktq_enqueue(pktq, m, h))) {
4478 1.4 riastrad if_statadd(ifp, if_ibytes, pktlen);
4479 1.4 riastrad if_statinc(ifp, if_ipackets);
4480 1.1 riastrad } else {
4481 1.1 riastrad m_freem(m);
4482 1.1 riastrad }
4483 1.57 riastrad kpreempt_enable();
4484 1.1 riastrad }
4485 1.1 riastrad
4486 1.1 riastrad static void
4487 1.114 riastrad wg_calc_pubkey(uint8_t pubkey[static WG_STATIC_KEY_LEN],
4488 1.114 riastrad const uint8_t privkey[static WG_STATIC_KEY_LEN])
4489 1.1 riastrad {
4490 1.1 riastrad
4491 1.1 riastrad crypto_scalarmult_base(pubkey, privkey);
4492 1.1 riastrad }
4493 1.1 riastrad
4494 1.1 riastrad static int
4495 1.1 riastrad wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga)
4496 1.1 riastrad {
4497 1.1 riastrad struct radix_node_head *rnh;
4498 1.1 riastrad struct radix_node *rn;
4499 1.1 riastrad int error = 0;
4500 1.1 riastrad
4501 1.1 riastrad rw_enter(wg->wg_rwlock, RW_WRITER);
4502 1.1 riastrad rnh = wg_rnh(wg, wga->wga_family);
4503 1.1 riastrad KASSERT(rnh != NULL);
4504 1.1 riastrad rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh,
4505 1.1 riastrad wga->wga_nodes);
4506 1.1 riastrad rw_exit(wg->wg_rwlock);
4507 1.1 riastrad
4508 1.1 riastrad if (rn == NULL)
4509 1.1 riastrad error = EEXIST;
4510 1.1 riastrad
4511 1.1 riastrad return error;
4512 1.1 riastrad }
4513 1.1 riastrad
4514 1.1 riastrad static int
4515 1.1 riastrad wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer,
4516 1.1 riastrad struct wg_peer **wgpp)
4517 1.1 riastrad {
4518 1.1 riastrad int error = 0;
4519 1.12 riastrad const void *pubkey;
4520 1.1 riastrad size_t pubkey_len;
4521 1.12 riastrad const void *psk;
4522 1.12 riastrad size_t psk_len;
4523 1.1 riastrad const char *name = NULL;
4524 1.1 riastrad
4525 1.12 riastrad if (prop_dictionary_get_string(peer, "name", &name)) {
4526 1.1 riastrad if (strlen(name) > WG_PEER_NAME_MAXLEN) {
4527 1.1 riastrad error = EINVAL;
4528 1.1 riastrad goto out;
4529 1.1 riastrad }
4530 1.1 riastrad }
4531 1.1 riastrad
4532 1.12 riastrad if (!prop_dictionary_get_data(peer, "public_key",
4533 1.12 riastrad &pubkey, &pubkey_len)) {
4534 1.1 riastrad error = EINVAL;
4535 1.1 riastrad goto out;
4536 1.1 riastrad }
4537 1.1 riastrad #ifdef WG_DEBUG_DUMP
4538 1.80 christos if (wg_debug & WG_DEBUG_FLAGS_DUMP) {
4539 1.80 christos char *hex = gethexdump(pubkey, pubkey_len);
4540 1.87 kre log(LOG_DEBUG, "pubkey=%p, pubkey_len=%zu\n%s\n",
4541 1.80 christos pubkey, pubkey_len, hex);
4542 1.80 christos puthexdump(hex, pubkey, pubkey_len);
4543 1.80 christos }
4544 1.1 riastrad #endif
4545 1.1 riastrad
4546 1.1 riastrad struct wg_peer *wgp = wg_alloc_peer(wg);
4547 1.1 riastrad memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey));
4548 1.1 riastrad if (name != NULL)
4549 1.1 riastrad strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name));
4550 1.1 riastrad
4551 1.12 riastrad if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) {
4552 1.1 riastrad if (psk_len != sizeof(wgp->wgp_psk)) {
4553 1.1 riastrad error = EINVAL;
4554 1.1 riastrad goto out;
4555 1.1 riastrad }
4556 1.1 riastrad memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk));
4557 1.1 riastrad }
4558 1.1 riastrad
4559 1.12 riastrad const void *addr;
4560 1.1 riastrad size_t addr_len;
4561 1.47 riastrad struct wg_sockaddr *wgsa = wgp->wgp_endpoint;
4562 1.1 riastrad
4563 1.12 riastrad if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len))
4564 1.1 riastrad goto skip_endpoint;
4565 1.47 riastrad if (addr_len < sizeof(*wgsatosa(wgsa)) ||
4566 1.47 riastrad addr_len > sizeof(*wgsatoss(wgsa))) {
4567 1.47 riastrad error = EINVAL;
4568 1.47 riastrad goto out;
4569 1.47 riastrad }
4570 1.47 riastrad memcpy(wgsatoss(wgsa), addr, addr_len);
4571 1.47 riastrad switch (wgsa_family(wgsa)) {
4572 1.109 riastrad #ifdef INET
4573 1.47 riastrad case AF_INET:
4574 1.109 riastrad break;
4575 1.109 riastrad #endif
4576 1.1 riastrad #ifdef INET6
4577 1.47 riastrad case AF_INET6:
4578 1.109 riastrad break;
4579 1.47 riastrad #endif
4580 1.1 riastrad default:
4581 1.47 riastrad error = EPFNOSUPPORT;
4582 1.47 riastrad goto out;
4583 1.47 riastrad }
4584 1.47 riastrad if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) {
4585 1.47 riastrad error = EINVAL;
4586 1.47 riastrad goto out;
4587 1.1 riastrad }
4588 1.47 riastrad {
4589 1.47 riastrad char addrstr[128];
4590 1.47 riastrad sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr));
4591 1.47 riastrad WG_DLOG("addr=%s\n", addrstr);
4592 1.47 riastrad }
4593 1.1 riastrad wgp->wgp_endpoint_available = true;
4594 1.1 riastrad
4595 1.1 riastrad prop_array_t allowedips;
4596 1.1 riastrad skip_endpoint:
4597 1.1 riastrad allowedips = prop_dictionary_get(peer, "allowedips");
4598 1.1 riastrad if (allowedips == NULL)
4599 1.1 riastrad goto skip;
4600 1.1 riastrad
4601 1.1 riastrad prop_object_iterator_t _it = prop_array_iterator(allowedips);
4602 1.1 riastrad prop_dictionary_t prop_allowedip;
4603 1.1 riastrad int j = 0;
4604 1.1 riastrad while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) {
4605 1.1 riastrad struct wg_allowedip *wga = &wgp->wgp_allowedips[j];
4606 1.1 riastrad
4607 1.12 riastrad if (!prop_dictionary_get_int(prop_allowedip, "family",
4608 1.12 riastrad &wga->wga_family))
4609 1.1 riastrad continue;
4610 1.12 riastrad if (!prop_dictionary_get_data(prop_allowedip, "ip",
4611 1.12 riastrad &addr, &addr_len))
4612 1.1 riastrad continue;
4613 1.12 riastrad if (!prop_dictionary_get_uint8(prop_allowedip, "cidr",
4614 1.12 riastrad &wga->wga_cidr))
4615 1.1 riastrad continue;
4616 1.1 riastrad
4617 1.1 riastrad switch (wga->wga_family) {
4618 1.109 riastrad #ifdef INET
4619 1.1 riastrad case AF_INET: {
4620 1.1 riastrad struct sockaddr_in sin;
4621 1.1 riastrad char addrstr[128];
4622 1.1 riastrad struct in_addr mask;
4623 1.1 riastrad struct sockaddr_in sin_mask;
4624 1.1 riastrad
4625 1.1 riastrad if (addr_len != sizeof(struct in_addr))
4626 1.1 riastrad return EINVAL;
4627 1.1 riastrad memcpy(&wga->wga_addr4, addr, addr_len);
4628 1.1 riastrad
4629 1.9 riastrad sockaddr_in_init(&sin, (const struct in_addr *)addr,
4630 1.9 riastrad 0);
4631 1.1 riastrad sockaddr_copy(&wga->wga_sa_addr,
4632 1.1 riastrad sizeof(sin), sintosa(&sin));
4633 1.1 riastrad
4634 1.9 riastrad sockaddr_format(sintosa(&sin),
4635 1.9 riastrad addrstr, sizeof(addrstr));
4636 1.1 riastrad WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr);
4637 1.1 riastrad
4638 1.1 riastrad in_len2mask(&mask, wga->wga_cidr);
4639 1.1 riastrad sockaddr_in_init(&sin_mask, &mask, 0);
4640 1.1 riastrad sockaddr_copy(&wga->wga_sa_mask,
4641 1.1 riastrad sizeof(sin_mask), sintosa(&sin_mask));
4642 1.1 riastrad
4643 1.1 riastrad break;
4644 1.1 riastrad }
4645 1.109 riastrad #endif
4646 1.1 riastrad #ifdef INET6
4647 1.1 riastrad case AF_INET6: {
4648 1.1 riastrad struct sockaddr_in6 sin6;
4649 1.1 riastrad char addrstr[128];
4650 1.1 riastrad struct in6_addr mask;
4651 1.1 riastrad struct sockaddr_in6 sin6_mask;
4652 1.1 riastrad
4653 1.1 riastrad if (addr_len != sizeof(struct in6_addr))
4654 1.1 riastrad return EINVAL;
4655 1.1 riastrad memcpy(&wga->wga_addr6, addr, addr_len);
4656 1.1 riastrad
4657 1.9 riastrad sockaddr_in6_init(&sin6, (const struct in6_addr *)addr,
4658 1.9 riastrad 0, 0, 0);
4659 1.1 riastrad sockaddr_copy(&wga->wga_sa_addr,
4660 1.1 riastrad sizeof(sin6), sin6tosa(&sin6));
4661 1.1 riastrad
4662 1.9 riastrad sockaddr_format(sin6tosa(&sin6),
4663 1.9 riastrad addrstr, sizeof(addrstr));
4664 1.1 riastrad WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr);
4665 1.1 riastrad
4666 1.1 riastrad in6_prefixlen2mask(&mask, wga->wga_cidr);
4667 1.1 riastrad sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0);
4668 1.1 riastrad sockaddr_copy(&wga->wga_sa_mask,
4669 1.1 riastrad sizeof(sin6_mask), sin6tosa(&sin6_mask));
4670 1.1 riastrad
4671 1.1 riastrad break;
4672 1.1 riastrad }
4673 1.1 riastrad #endif
4674 1.1 riastrad default:
4675 1.1 riastrad error = EINVAL;
4676 1.1 riastrad goto out;
4677 1.1 riastrad }
4678 1.1 riastrad wga->wga_peer = wgp;
4679 1.1 riastrad
4680 1.1 riastrad error = wg_rtable_add_route(wg, wga);
4681 1.1 riastrad if (error != 0)
4682 1.1 riastrad goto out;
4683 1.1 riastrad
4684 1.1 riastrad j++;
4685 1.1 riastrad }
4686 1.1 riastrad wgp->wgp_n_allowedips = j;
4687 1.1 riastrad skip:
4688 1.1 riastrad *wgpp = wgp;
4689 1.1 riastrad out:
4690 1.1 riastrad return error;
4691 1.1 riastrad }
4692 1.1 riastrad
4693 1.1 riastrad static int
4694 1.1 riastrad wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd)
4695 1.1 riastrad {
4696 1.1 riastrad int error;
4697 1.1 riastrad char *buf;
4698 1.1 riastrad
4699 1.87 kre WG_DLOG("buf=%p, len=%zu\n", ifd->ifd_data, ifd->ifd_len);
4700 1.68 riastrad if (ifd->ifd_len >= WG_MAX_PROPLEN)
4701 1.68 riastrad return E2BIG;
4702 1.1 riastrad buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP);
4703 1.1 riastrad error = copyin(ifd->ifd_data, buf, ifd->ifd_len);
4704 1.1 riastrad if (error != 0)
4705 1.1 riastrad return error;
4706 1.1 riastrad buf[ifd->ifd_len] = '\0';
4707 1.1 riastrad #ifdef WG_DEBUG_DUMP
4708 1.80 christos if (wg_debug & WG_DEBUG_FLAGS_DUMP) {
4709 1.80 christos log(LOG_DEBUG, "%.*s\n", (int)MIN(INT_MAX, ifd->ifd_len),
4710 1.80 christos (const char *)buf);
4711 1.80 christos }
4712 1.1 riastrad #endif
4713 1.1 riastrad *_buf = buf;
4714 1.1 riastrad return 0;
4715 1.1 riastrad }
4716 1.1 riastrad
4717 1.1 riastrad static int
4718 1.1 riastrad wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd)
4719 1.1 riastrad {
4720 1.1 riastrad int error;
4721 1.1 riastrad prop_dictionary_t prop_dict;
4722 1.1 riastrad char *buf = NULL;
4723 1.12 riastrad const void *privkey;
4724 1.1 riastrad size_t privkey_len;
4725 1.1 riastrad
4726 1.1 riastrad error = wg_alloc_prop_buf(&buf, ifd);
4727 1.1 riastrad if (error != 0)
4728 1.1 riastrad return error;
4729 1.1 riastrad error = EINVAL;
4730 1.1 riastrad prop_dict = prop_dictionary_internalize(buf);
4731 1.1 riastrad if (prop_dict == NULL)
4732 1.1 riastrad goto out;
4733 1.12 riastrad if (!prop_dictionary_get_data(prop_dict, "private_key",
4734 1.12 riastrad &privkey, &privkey_len))
4735 1.1 riastrad goto out;
4736 1.1 riastrad #ifdef WG_DEBUG_DUMP
4737 1.80 christos if (wg_debug & WG_DEBUG_FLAGS_DUMP) {
4738 1.80 christos char *hex = gethexdump(privkey, privkey_len);
4739 1.87 kre log(LOG_DEBUG, "privkey=%p, privkey_len=%zu\n%s\n",
4740 1.80 christos privkey, privkey_len, hex);
4741 1.80 christos puthexdump(hex, privkey, privkey_len);
4742 1.80 christos }
4743 1.1 riastrad #endif
4744 1.1 riastrad if (privkey_len != WG_STATIC_KEY_LEN)
4745 1.1 riastrad goto out;
4746 1.1 riastrad memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN);
4747 1.1 riastrad wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey);
4748 1.1 riastrad error = 0;
4749 1.1 riastrad
4750 1.1 riastrad out:
4751 1.1 riastrad kmem_free(buf, ifd->ifd_len + 1);
4752 1.1 riastrad return error;
4753 1.1 riastrad }
4754 1.1 riastrad
4755 1.1 riastrad static int
4756 1.1 riastrad wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd)
4757 1.1 riastrad {
4758 1.1 riastrad int error;
4759 1.1 riastrad prop_dictionary_t prop_dict;
4760 1.1 riastrad char *buf = NULL;
4761 1.12 riastrad uint16_t port;
4762 1.1 riastrad
4763 1.1 riastrad error = wg_alloc_prop_buf(&buf, ifd);
4764 1.1 riastrad if (error != 0)
4765 1.1 riastrad return error;
4766 1.1 riastrad error = EINVAL;
4767 1.1 riastrad prop_dict = prop_dictionary_internalize(buf);
4768 1.1 riastrad if (prop_dict == NULL)
4769 1.1 riastrad goto out;
4770 1.12 riastrad if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port))
4771 1.1 riastrad goto out;
4772 1.1 riastrad
4773 1.1 riastrad error = wg->wg_ops->bind_port(wg, (uint16_t)port);
4774 1.1 riastrad
4775 1.1 riastrad out:
4776 1.1 riastrad kmem_free(buf, ifd->ifd_len + 1);
4777 1.1 riastrad return error;
4778 1.1 riastrad }
4779 1.1 riastrad
4780 1.1 riastrad static int
4781 1.1 riastrad wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd)
4782 1.1 riastrad {
4783 1.1 riastrad int error;
4784 1.1 riastrad prop_dictionary_t prop_dict;
4785 1.1 riastrad char *buf = NULL;
4786 1.37 riastrad struct wg_peer *wgp = NULL, *wgp0 __diagused;
4787 1.1 riastrad
4788 1.1 riastrad error = wg_alloc_prop_buf(&buf, ifd);
4789 1.1 riastrad if (error != 0)
4790 1.1 riastrad return error;
4791 1.1 riastrad error = EINVAL;
4792 1.1 riastrad prop_dict = prop_dictionary_internalize(buf);
4793 1.1 riastrad if (prop_dict == NULL)
4794 1.1 riastrad goto out;
4795 1.1 riastrad
4796 1.1 riastrad error = wg_handle_prop_peer(wg, prop_dict, &wgp);
4797 1.1 riastrad if (error != 0)
4798 1.1 riastrad goto out;
4799 1.1 riastrad
4800 1.1 riastrad mutex_enter(wg->wg_lock);
4801 1.37 riastrad if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
4802 1.37 riastrad sizeof(wgp->wgp_pubkey)) != NULL ||
4803 1.37 riastrad (wgp->wgp_name[0] &&
4804 1.37 riastrad thmap_get(wg->wg_peers_byname, wgp->wgp_name,
4805 1.37 riastrad strlen(wgp->wgp_name)) != NULL)) {
4806 1.37 riastrad mutex_exit(wg->wg_lock);
4807 1.37 riastrad wg_destroy_peer(wgp);
4808 1.37 riastrad error = EEXIST;
4809 1.37 riastrad goto out;
4810 1.37 riastrad }
4811 1.37 riastrad wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
4812 1.37 riastrad sizeof(wgp->wgp_pubkey), wgp);
4813 1.37 riastrad KASSERT(wgp0 == wgp);
4814 1.37 riastrad if (wgp->wgp_name[0]) {
4815 1.37 riastrad wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name,
4816 1.37 riastrad strlen(wgp->wgp_name), wgp);
4817 1.37 riastrad KASSERT(wgp0 == wgp);
4818 1.37 riastrad }
4819 1.1 riastrad WG_PEER_WRITER_INSERT_HEAD(wgp, wg);
4820 1.1 riastrad wg->wg_npeers++;
4821 1.1 riastrad mutex_exit(wg->wg_lock);
4822 1.1 riastrad
4823 1.61 roy if_link_state_change(&wg->wg_if, LINK_STATE_UP);
4824 1.61 roy
4825 1.1 riastrad out:
4826 1.1 riastrad kmem_free(buf, ifd->ifd_len + 1);
4827 1.1 riastrad return error;
4828 1.1 riastrad }
4829 1.1 riastrad
4830 1.1 riastrad static int
4831 1.1 riastrad wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd)
4832 1.1 riastrad {
4833 1.1 riastrad int error;
4834 1.1 riastrad prop_dictionary_t prop_dict;
4835 1.1 riastrad char *buf = NULL;
4836 1.1 riastrad const char *name;
4837 1.1 riastrad
4838 1.1 riastrad error = wg_alloc_prop_buf(&buf, ifd);
4839 1.1 riastrad if (error != 0)
4840 1.1 riastrad return error;
4841 1.1 riastrad error = EINVAL;
4842 1.1 riastrad prop_dict = prop_dictionary_internalize(buf);
4843 1.1 riastrad if (prop_dict == NULL)
4844 1.1 riastrad goto out;
4845 1.1 riastrad
4846 1.12 riastrad if (!prop_dictionary_get_string(prop_dict, "name", &name))
4847 1.1 riastrad goto out;
4848 1.1 riastrad if (strlen(name) > WG_PEER_NAME_MAXLEN)
4849 1.1 riastrad goto out;
4850 1.1 riastrad
4851 1.1 riastrad error = wg_destroy_peer_name(wg, name);
4852 1.1 riastrad out:
4853 1.1 riastrad kmem_free(buf, ifd->ifd_len + 1);
4854 1.1 riastrad return error;
4855 1.1 riastrad }
4856 1.1 riastrad
4857 1.74 christos static bool
4858 1.74 christos wg_is_authorized(struct wg_softc *wg, u_long cmd)
4859 1.74 christos {
4860 1.74 christos int au = cmd == SIOCGDRVSPEC ?
4861 1.74 christos KAUTH_REQ_NETWORK_INTERFACE_WG_GETPRIV :
4862 1.74 christos KAUTH_REQ_NETWORK_INTERFACE_WG_SETPRIV;
4863 1.74 christos return kauth_authorize_network(kauth_cred_get(),
4864 1.74 christos KAUTH_NETWORK_INTERFACE_WG, au, &wg->wg_if,
4865 1.74 christos (void *)cmd, NULL) == 0;
4866 1.74 christos }
4867 1.74 christos
4868 1.1 riastrad static int
4869 1.1 riastrad wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd)
4870 1.1 riastrad {
4871 1.1 riastrad int error = ENOMEM;
4872 1.1 riastrad prop_dictionary_t prop_dict;
4873 1.23 riastrad prop_array_t peers = NULL;
4874 1.1 riastrad char *buf;
4875 1.1 riastrad struct wg_peer *wgp;
4876 1.1 riastrad int s, i;
4877 1.1 riastrad
4878 1.1 riastrad prop_dict = prop_dictionary_create();
4879 1.1 riastrad if (prop_dict == NULL)
4880 1.1 riastrad goto error;
4881 1.1 riastrad
4882 1.74 christos if (wg_is_authorized(wg, SIOCGDRVSPEC)) {
4883 1.73 jakllsch if (!prop_dictionary_set_data(prop_dict, "private_key",
4884 1.73 jakllsch wg->wg_privkey, WG_STATIC_KEY_LEN))
4885 1.73 jakllsch goto error;
4886 1.73 jakllsch }
4887 1.1 riastrad
4888 1.1 riastrad if (wg->wg_listen_port != 0) {
4889 1.12 riastrad if (!prop_dictionary_set_uint16(prop_dict, "listen_port",
4890 1.12 riastrad wg->wg_listen_port))
4891 1.1 riastrad goto error;
4892 1.1 riastrad }
4893 1.1 riastrad
4894 1.1 riastrad if (wg->wg_npeers == 0)
4895 1.1 riastrad goto skip_peers;
4896 1.1 riastrad
4897 1.1 riastrad peers = prop_array_create();
4898 1.12 riastrad if (peers == NULL)
4899 1.12 riastrad goto error;
4900 1.12 riastrad
4901 1.1 riastrad s = pserialize_read_enter();
4902 1.1 riastrad i = 0;
4903 1.1 riastrad WG_PEER_READER_FOREACH(wgp, wg) {
4904 1.47 riastrad struct wg_sockaddr *wgsa;
4905 1.47 riastrad struct psref wgp_psref, wgsa_psref;
4906 1.1 riastrad prop_dictionary_t prop_peer;
4907 1.1 riastrad
4908 1.47 riastrad wg_get_peer(wgp, &wgp_psref);
4909 1.1 riastrad pserialize_read_exit(s);
4910 1.1 riastrad
4911 1.1 riastrad prop_peer = prop_dictionary_create();
4912 1.12 riastrad if (prop_peer == NULL)
4913 1.12 riastrad goto next;
4914 1.1 riastrad
4915 1.1 riastrad if (strlen(wgp->wgp_name) > 0) {
4916 1.12 riastrad if (!prop_dictionary_set_string(prop_peer, "name",
4917 1.12 riastrad wgp->wgp_name))
4918 1.12 riastrad goto next;
4919 1.1 riastrad }
4920 1.1 riastrad
4921 1.12 riastrad if (!prop_dictionary_set_data(prop_peer, "public_key",
4922 1.12 riastrad wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)))
4923 1.1 riastrad goto next;
4924 1.1 riastrad
4925 1.1 riastrad uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0};
4926 1.13 riastrad if (!consttime_memequal(wgp->wgp_psk, psk_zero,
4927 1.13 riastrad sizeof(wgp->wgp_psk))) {
4928 1.74 christos if (wg_is_authorized(wg, SIOCGDRVSPEC)) {
4929 1.73 jakllsch if (!prop_dictionary_set_data(prop_peer,
4930 1.73 jakllsch "preshared_key",
4931 1.73 jakllsch wgp->wgp_psk, sizeof(wgp->wgp_psk)))
4932 1.73 jakllsch goto next;
4933 1.73 jakllsch }
4934 1.1 riastrad }
4935 1.1 riastrad
4936 1.47 riastrad wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref);
4937 1.47 riastrad CTASSERT(AF_UNSPEC == 0);
4938 1.47 riastrad if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ &&
4939 1.47 riastrad !prop_dictionary_set_data(prop_peer, "endpoint",
4940 1.47 riastrad wgsatoss(wgsa),
4941 1.47 riastrad sockaddr_getsize_by_family(wgsa_family(wgsa)))) {
4942 1.47 riastrad wg_put_sa(wgp, wgsa, &wgsa_psref);
4943 1.47 riastrad goto next;
4944 1.1 riastrad }
4945 1.47 riastrad wg_put_sa(wgp, wgsa, &wgsa_psref);
4946 1.1 riastrad
4947 1.9 riastrad const struct timespec *t = &wgp->wgp_last_handshake_time;
4948 1.9 riastrad
4949 1.12 riastrad if (!prop_dictionary_set_uint64(prop_peer,
4950 1.65 christos "last_handshake_time_sec", (uint64_t)t->tv_sec))
4951 1.1 riastrad goto next;
4952 1.12 riastrad if (!prop_dictionary_set_uint32(prop_peer,
4953 1.65 christos "last_handshake_time_nsec", (uint32_t)t->tv_nsec))
4954 1.1 riastrad goto next;
4955 1.1 riastrad
4956 1.1 riastrad if (wgp->wgp_n_allowedips == 0)
4957 1.1 riastrad goto skip_allowedips;
4958 1.1 riastrad
4959 1.1 riastrad prop_array_t allowedips = prop_array_create();
4960 1.12 riastrad if (allowedips == NULL)
4961 1.12 riastrad goto next;
4962 1.1 riastrad for (int j = 0; j < wgp->wgp_n_allowedips; j++) {
4963 1.1 riastrad struct wg_allowedip *wga = &wgp->wgp_allowedips[j];
4964 1.1 riastrad prop_dictionary_t prop_allowedip;
4965 1.1 riastrad
4966 1.1 riastrad prop_allowedip = prop_dictionary_create();
4967 1.1 riastrad if (prop_allowedip == NULL)
4968 1.1 riastrad break;
4969 1.1 riastrad
4970 1.12 riastrad if (!prop_dictionary_set_int(prop_allowedip, "family",
4971 1.12 riastrad wga->wga_family))
4972 1.1 riastrad goto _next;
4973 1.12 riastrad if (!prop_dictionary_set_uint8(prop_allowedip, "cidr",
4974 1.12 riastrad wga->wga_cidr))
4975 1.1 riastrad goto _next;
4976 1.1 riastrad
4977 1.1 riastrad switch (wga->wga_family) {
4978 1.109 riastrad #ifdef INET
4979 1.1 riastrad case AF_INET:
4980 1.12 riastrad if (!prop_dictionary_set_data(prop_allowedip,
4981 1.12 riastrad "ip", &wga->wga_addr4,
4982 1.12 riastrad sizeof(wga->wga_addr4)))
4983 1.1 riastrad goto _next;
4984 1.1 riastrad break;
4985 1.109 riastrad #endif
4986 1.1 riastrad #ifdef INET6
4987 1.1 riastrad case AF_INET6:
4988 1.12 riastrad if (!prop_dictionary_set_data(prop_allowedip,
4989 1.12 riastrad "ip", &wga->wga_addr6,
4990 1.12 riastrad sizeof(wga->wga_addr6)))
4991 1.1 riastrad goto _next;
4992 1.1 riastrad break;
4993 1.1 riastrad #endif
4994 1.1 riastrad default:
4995 1.109 riastrad panic("invalid af=%d", wga->wga_family);
4996 1.1 riastrad }
4997 1.1 riastrad prop_array_set(allowedips, j, prop_allowedip);
4998 1.1 riastrad _next:
4999 1.1 riastrad prop_object_release(prop_allowedip);
5000 1.1 riastrad }
5001 1.1 riastrad prop_dictionary_set(prop_peer, "allowedips", allowedips);
5002 1.1 riastrad prop_object_release(allowedips);
5003 1.1 riastrad
5004 1.1 riastrad skip_allowedips:
5005 1.1 riastrad
5006 1.1 riastrad prop_array_set(peers, i, prop_peer);
5007 1.1 riastrad next:
5008 1.12 riastrad if (prop_peer)
5009 1.12 riastrad prop_object_release(prop_peer);
5010 1.1 riastrad i++;
5011 1.1 riastrad
5012 1.1 riastrad s = pserialize_read_enter();
5013 1.47 riastrad wg_put_peer(wgp, &wgp_psref);
5014 1.1 riastrad }
5015 1.1 riastrad pserialize_read_exit(s);
5016 1.1 riastrad
5017 1.1 riastrad prop_dictionary_set(prop_dict, "peers", peers);
5018 1.1 riastrad prop_object_release(peers);
5019 1.1 riastrad peers = NULL;
5020 1.1 riastrad
5021 1.1 riastrad skip_peers:
5022 1.1 riastrad buf = prop_dictionary_externalize(prop_dict);
5023 1.1 riastrad if (buf == NULL)
5024 1.1 riastrad goto error;
5025 1.1 riastrad if (ifd->ifd_len < (strlen(buf) + 1)) {
5026 1.1 riastrad error = EINVAL;
5027 1.1 riastrad goto error;
5028 1.1 riastrad }
5029 1.1 riastrad error = copyout(buf, ifd->ifd_data, strlen(buf) + 1);
5030 1.1 riastrad
5031 1.1 riastrad free(buf, 0);
5032 1.1 riastrad error:
5033 1.1 riastrad if (peers != NULL)
5034 1.1 riastrad prop_object_release(peers);
5035 1.1 riastrad if (prop_dict != NULL)
5036 1.1 riastrad prop_object_release(prop_dict);
5037 1.1 riastrad
5038 1.1 riastrad return error;
5039 1.1 riastrad }
5040 1.1 riastrad
5041 1.1 riastrad static int
5042 1.1 riastrad wg_ioctl(struct ifnet *ifp, u_long cmd, void *data)
5043 1.1 riastrad {
5044 1.1 riastrad struct wg_softc *wg = ifp->if_softc;
5045 1.1 riastrad struct ifreq *ifr = data;
5046 1.1 riastrad struct ifaddr *ifa = data;
5047 1.1 riastrad struct ifdrv *ifd = data;
5048 1.1 riastrad int error = 0;
5049 1.1 riastrad
5050 1.1 riastrad switch (cmd) {
5051 1.1 riastrad case SIOCINITIFADDR:
5052 1.1 riastrad if (ifa->ifa_addr->sa_family != AF_LINK &&
5053 1.1 riastrad (ifp->if_flags & (IFF_UP | IFF_RUNNING)) !=
5054 1.1 riastrad (IFF_UP | IFF_RUNNING)) {
5055 1.1 riastrad ifp->if_flags |= IFF_UP;
5056 1.67 riastrad error = if_init(ifp);
5057 1.1 riastrad }
5058 1.14 riastrad return error;
5059 1.1 riastrad case SIOCADDMULTI:
5060 1.1 riastrad case SIOCDELMULTI:
5061 1.1 riastrad switch (ifr->ifr_addr.sa_family) {
5062 1.109 riastrad #ifdef INET
5063 1.1 riastrad case AF_INET: /* IP supports Multicast */
5064 1.1 riastrad break;
5065 1.109 riastrad #endif
5066 1.1 riastrad #ifdef INET6
5067 1.1 riastrad case AF_INET6: /* IP6 supports Multicast */
5068 1.1 riastrad break;
5069 1.1 riastrad #endif
5070 1.1 riastrad default: /* Other protocols doesn't support Multicast */
5071 1.1 riastrad error = EAFNOSUPPORT;
5072 1.1 riastrad break;
5073 1.1 riastrad }
5074 1.14 riastrad return error;
5075 1.1 riastrad case SIOCSDRVSPEC:
5076 1.74 christos if (!wg_is_authorized(wg, cmd)) {
5077 1.72 jakllsch return EPERM;
5078 1.72 jakllsch }
5079 1.1 riastrad switch (ifd->ifd_cmd) {
5080 1.1 riastrad case WG_IOCTL_SET_PRIVATE_KEY:
5081 1.1 riastrad error = wg_ioctl_set_private_key(wg, ifd);
5082 1.1 riastrad break;
5083 1.1 riastrad case WG_IOCTL_SET_LISTEN_PORT:
5084 1.1 riastrad error = wg_ioctl_set_listen_port(wg, ifd);
5085 1.1 riastrad break;
5086 1.1 riastrad case WG_IOCTL_ADD_PEER:
5087 1.1 riastrad error = wg_ioctl_add_peer(wg, ifd);
5088 1.1 riastrad break;
5089 1.1 riastrad case WG_IOCTL_DELETE_PEER:
5090 1.1 riastrad error = wg_ioctl_delete_peer(wg, ifd);
5091 1.1 riastrad break;
5092 1.1 riastrad default:
5093 1.1 riastrad error = EINVAL;
5094 1.1 riastrad break;
5095 1.1 riastrad }
5096 1.14 riastrad return error;
5097 1.1 riastrad case SIOCGDRVSPEC:
5098 1.14 riastrad return wg_ioctl_get(wg, ifd);
5099 1.1 riastrad case SIOCSIFFLAGS:
5100 1.1 riastrad if ((error = ifioctl_common(ifp, cmd, data)) != 0)
5101 1.1 riastrad break;
5102 1.1 riastrad switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) {
5103 1.1 riastrad case IFF_RUNNING:
5104 1.1 riastrad /*
5105 1.1 riastrad * If interface is marked down and it is running,
5106 1.1 riastrad * then stop and disable it.
5107 1.1 riastrad */
5108 1.66 riastrad if_stop(ifp, 1);
5109 1.1 riastrad break;
5110 1.1 riastrad case IFF_UP:
5111 1.1 riastrad /*
5112 1.1 riastrad * If interface is marked up and it is stopped, then
5113 1.1 riastrad * start it.
5114 1.1 riastrad */
5115 1.67 riastrad error = if_init(ifp);
5116 1.1 riastrad break;
5117 1.1 riastrad default:
5118 1.1 riastrad break;
5119 1.1 riastrad }
5120 1.14 riastrad return error;
5121 1.1 riastrad #ifdef WG_RUMPKERNEL
5122 1.1 riastrad case SIOCSLINKSTR:
5123 1.1 riastrad error = wg_ioctl_linkstr(wg, ifd);
5124 1.108 riastrad if (error)
5125 1.108 riastrad return error;
5126 1.108 riastrad wg->wg_ops = &wg_ops_rumpuser;
5127 1.108 riastrad return 0;
5128 1.14 riastrad #endif
5129 1.14 riastrad default:
5130 1.1 riastrad break;
5131 1.14 riastrad }
5132 1.1 riastrad
5133 1.14 riastrad error = ifioctl_common(ifp, cmd, data);
5134 1.1 riastrad
5135 1.1 riastrad #ifdef WG_RUMPKERNEL
5136 1.14 riastrad if (!wg_user_mode(wg))
5137 1.14 riastrad return error;
5138 1.14 riastrad
5139 1.14 riastrad /* Do the same to the corresponding tun device on the host */
5140 1.14 riastrad /*
5141 1.14 riastrad * XXX Actually the command has not been handled yet. It
5142 1.14 riastrad * will be handled via pr_ioctl form doifioctl later.
5143 1.14 riastrad */
5144 1.14 riastrad switch (cmd) {
5145 1.109 riastrad #ifdef INET
5146 1.14 riastrad case SIOCAIFADDR:
5147 1.14 riastrad case SIOCDIFADDR: {
5148 1.17 riastrad struct in_aliasreq _ifra = *(const struct in_aliasreq *)data;
5149 1.14 riastrad struct in_aliasreq *ifra = &_ifra;
5150 1.14 riastrad KASSERT(error == ENOTTY);
5151 1.14 riastrad strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user),
5152 1.14 riastrad IFNAMSIZ);
5153 1.14 riastrad error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET);
5154 1.14 riastrad if (error == 0)
5155 1.14 riastrad error = ENOTTY;
5156 1.14 riastrad break;
5157 1.14 riastrad }
5158 1.109 riastrad #endif
5159 1.1 riastrad #ifdef INET6
5160 1.14 riastrad case SIOCAIFADDR_IN6:
5161 1.14 riastrad case SIOCDIFADDR_IN6: {
5162 1.17 riastrad struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data;
5163 1.14 riastrad struct in6_aliasreq *ifra = &_ifra;
5164 1.14 riastrad KASSERT(error == ENOTTY);
5165 1.14 riastrad strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user),
5166 1.14 riastrad IFNAMSIZ);
5167 1.14 riastrad error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6);
5168 1.14 riastrad if (error == 0)
5169 1.14 riastrad error = ENOTTY;
5170 1.14 riastrad break;
5171 1.14 riastrad }
5172 1.1 riastrad #endif
5173 1.109 riastrad default:
5174 1.109 riastrad break;
5175 1.14 riastrad }
5176 1.1 riastrad #endif /* WG_RUMPKERNEL */
5177 1.1 riastrad
5178 1.1 riastrad return error;
5179 1.1 riastrad }
5180 1.1 riastrad
5181 1.1 riastrad static int
5182 1.1 riastrad wg_init(struct ifnet *ifp)
5183 1.1 riastrad {
5184 1.1 riastrad
5185 1.1 riastrad ifp->if_flags |= IFF_RUNNING;
5186 1.1 riastrad
5187 1.1 riastrad /* TODO flush pending packets. */
5188 1.1 riastrad return 0;
5189 1.1 riastrad }
5190 1.1 riastrad
5191 1.60 riastrad #ifdef ALTQ
5192 1.60 riastrad static void
5193 1.60 riastrad wg_start(struct ifnet *ifp)
5194 1.60 riastrad {
5195 1.60 riastrad struct mbuf *m;
5196 1.60 riastrad
5197 1.60 riastrad for (;;) {
5198 1.60 riastrad IFQ_DEQUEUE(&ifp->if_snd, m);
5199 1.60 riastrad if (m == NULL)
5200 1.60 riastrad break;
5201 1.60 riastrad
5202 1.60 riastrad kpreempt_disable();
5203 1.60 riastrad const uint32_t h = curcpu()->ci_index; // pktq_rps_hash(m)
5204 1.60 riastrad if (__predict_false(!pktq_enqueue(wg_pktq, m, h))) {
5205 1.76 jakllsch WGLOG(LOG_ERR, "%s: pktq full, dropping\n",
5206 1.76 jakllsch if_name(ifp));
5207 1.60 riastrad m_freem(m);
5208 1.60 riastrad }
5209 1.60 riastrad kpreempt_enable();
5210 1.60 riastrad }
5211 1.60 riastrad }
5212 1.60 riastrad #endif
5213 1.60 riastrad
5214 1.1 riastrad static void
5215 1.1 riastrad wg_stop(struct ifnet *ifp, int disable)
5216 1.1 riastrad {
5217 1.1 riastrad
5218 1.1 riastrad KASSERT((ifp->if_flags & IFF_RUNNING) != 0);
5219 1.1 riastrad ifp->if_flags &= ~IFF_RUNNING;
5220 1.1 riastrad
5221 1.1 riastrad /* Need to do something? */
5222 1.1 riastrad }
5223 1.1 riastrad
5224 1.8 riastrad #ifdef WG_DEBUG_PARAMS
5225 1.24 riastrad SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup")
5226 1.1 riastrad {
5227 1.1 riastrad const struct sysctlnode *node = NULL;
5228 1.1 riastrad
5229 1.8 riastrad sysctl_createv(clog, 0, NULL, &node,
5230 1.8 riastrad CTLFLAG_PERMANENT,
5231 1.24 riastrad CTLTYPE_NODE, "wg",
5232 1.24 riastrad SYSCTL_DESCR("wg(4)"),
5233 1.8 riastrad NULL, 0, NULL, 0,
5234 1.8 riastrad CTL_NET, CTL_CREATE, CTL_EOL);
5235 1.8 riastrad sysctl_createv(clog, 0, &node, NULL,
5236 1.8 riastrad CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5237 1.21 riastrad CTLTYPE_QUAD, "rekey_after_messages",
5238 1.8 riastrad SYSCTL_DESCR("session liftime by messages"),
5239 1.8 riastrad NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL);
5240 1.8 riastrad sysctl_createv(clog, 0, &node, NULL,
5241 1.8 riastrad CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5242 1.21 riastrad CTLTYPE_INT, "rekey_after_time",
5243 1.8 riastrad SYSCTL_DESCR("session liftime"),
5244 1.8 riastrad NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL);
5245 1.8 riastrad sysctl_createv(clog, 0, &node, NULL,
5246 1.8 riastrad CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5247 1.21 riastrad CTLTYPE_INT, "rekey_timeout",
5248 1.8 riastrad SYSCTL_DESCR("session handshake retry time"),
5249 1.8 riastrad NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL);
5250 1.8 riastrad sysctl_createv(clog, 0, &node, NULL,
5251 1.8 riastrad CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5252 1.21 riastrad CTLTYPE_INT, "rekey_attempt_time",
5253 1.8 riastrad SYSCTL_DESCR("session handshake timeout"),
5254 1.8 riastrad NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL);
5255 1.8 riastrad sysctl_createv(clog, 0, &node, NULL,
5256 1.8 riastrad CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5257 1.21 riastrad CTLTYPE_INT, "keepalive_timeout",
5258 1.8 riastrad SYSCTL_DESCR("keepalive timeout"),
5259 1.8 riastrad NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL);
5260 1.8 riastrad sysctl_createv(clog, 0, &node, NULL,
5261 1.8 riastrad CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5262 1.8 riastrad CTLTYPE_BOOL, "force_underload",
5263 1.8 riastrad SYSCTL_DESCR("force to detemine under load"),
5264 1.8 riastrad NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL);
5265 1.80 christos sysctl_createv(clog, 0, &node, NULL,
5266 1.80 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5267 1.80 christos CTLTYPE_INT, "debug",
5268 1.84 christos SYSCTL_DESCR("set debug flags 1=log 2=trace 4=dump 8=packet"),
5269 1.80 christos NULL, 0, &wg_debug, 0, CTL_CREATE, CTL_EOL);
5270 1.8 riastrad }
5271 1.1 riastrad #endif
5272 1.1 riastrad
5273 1.1 riastrad #ifdef WG_RUMPKERNEL
5274 1.1 riastrad static bool
5275 1.1 riastrad wg_user_mode(struct wg_softc *wg)
5276 1.1 riastrad {
5277 1.1 riastrad
5278 1.1 riastrad return wg->wg_user != NULL;
5279 1.1 riastrad }
5280 1.1 riastrad
5281 1.1 riastrad static int
5282 1.1 riastrad wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd)
5283 1.1 riastrad {
5284 1.1 riastrad struct ifnet *ifp = &wg->wg_if;
5285 1.1 riastrad int error;
5286 1.1 riastrad
5287 1.1 riastrad if (ifp->if_flags & IFF_UP)
5288 1.1 riastrad return EBUSY;
5289 1.1 riastrad
5290 1.1 riastrad if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
5291 1.1 riastrad /* XXX do nothing */
5292 1.1 riastrad return 0;
5293 1.1 riastrad } else if (ifd->ifd_cmd != 0) {
5294 1.1 riastrad return EINVAL;
5295 1.1 riastrad } else if (wg->wg_user != NULL) {
5296 1.1 riastrad return EBUSY;
5297 1.1 riastrad }
5298 1.1 riastrad
5299 1.1 riastrad /* Assume \0 included */
5300 1.1 riastrad if (ifd->ifd_len > IFNAMSIZ) {
5301 1.1 riastrad return E2BIG;
5302 1.1 riastrad } else if (ifd->ifd_len < 1) {
5303 1.1 riastrad return EINVAL;
5304 1.1 riastrad }
5305 1.1 riastrad
5306 1.1 riastrad char tun_name[IFNAMSIZ];
5307 1.1 riastrad error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL);
5308 1.1 riastrad if (error != 0)
5309 1.1 riastrad return error;
5310 1.1 riastrad
5311 1.1 riastrad if (strncmp(tun_name, "tun", 3) != 0)
5312 1.1 riastrad return EINVAL;
5313 1.1 riastrad
5314 1.1 riastrad error = rumpuser_wg_create(tun_name, wg, &wg->wg_user);
5315 1.1 riastrad
5316 1.1 riastrad return error;
5317 1.1 riastrad }
5318 1.1 riastrad
5319 1.1 riastrad static int
5320 1.1 riastrad wg_send_user(struct wg_peer *wgp, struct mbuf *m)
5321 1.1 riastrad {
5322 1.1 riastrad int error;
5323 1.1 riastrad struct psref psref;
5324 1.1 riastrad struct wg_sockaddr *wgsa;
5325 1.1 riastrad struct wg_softc *wg = wgp->wgp_sc;
5326 1.1 riastrad struct iovec iov[1];
5327 1.1 riastrad
5328 1.1 riastrad wgsa = wg_get_endpoint_sa(wgp, &psref);
5329 1.1 riastrad
5330 1.1 riastrad iov[0].iov_base = mtod(m, void *);
5331 1.1 riastrad iov[0].iov_len = m->m_len;
5332 1.1 riastrad
5333 1.1 riastrad /* Send messages to a peer via an ordinary socket. */
5334 1.1 riastrad error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1);
5335 1.1 riastrad
5336 1.1 riastrad wg_put_sa(wgp, wgsa, &psref);
5337 1.1 riastrad
5338 1.38 riastrad m_freem(m);
5339 1.38 riastrad
5340 1.1 riastrad return error;
5341 1.1 riastrad }
5342 1.1 riastrad
5343 1.1 riastrad static void
5344 1.1 riastrad wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af)
5345 1.1 riastrad {
5346 1.1 riastrad struct wg_softc *wg = ifp->if_softc;
5347 1.1 riastrad struct iovec iov[2];
5348 1.1 riastrad struct sockaddr_storage ss;
5349 1.1 riastrad
5350 1.1 riastrad KASSERT(af == AF_INET || af == AF_INET6);
5351 1.1 riastrad
5352 1.1 riastrad WG_TRACE("");
5353 1.1 riastrad
5354 1.109 riastrad switch (af) {
5355 1.109 riastrad #ifdef INET
5356 1.109 riastrad case AF_INET: {
5357 1.1 riastrad struct sockaddr_in *sin = (struct sockaddr_in *)&ss;
5358 1.1 riastrad struct ip *ip;
5359 1.27 riastrad
5360 1.27 riastrad KASSERT(m->m_len >= sizeof(struct ip));
5361 1.1 riastrad ip = mtod(m, struct ip *);
5362 1.1 riastrad sockaddr_in_init(sin, &ip->ip_dst, 0);
5363 1.109 riastrad break;
5364 1.109 riastrad }
5365 1.109 riastrad #endif
5366 1.109 riastrad #ifdef INET6
5367 1.109 riastrad case AF_INET6: {
5368 1.1 riastrad struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss;
5369 1.1 riastrad struct ip6_hdr *ip6;
5370 1.27 riastrad
5371 1.27 riastrad KASSERT(m->m_len >= sizeof(struct ip6_hdr));
5372 1.1 riastrad ip6 = mtod(m, struct ip6_hdr *);
5373 1.1 riastrad sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0);
5374 1.109 riastrad break;
5375 1.109 riastrad }
5376 1.109 riastrad #endif
5377 1.109 riastrad default:
5378 1.109 riastrad goto out;
5379 1.1 riastrad }
5380 1.1 riastrad
5381 1.1 riastrad iov[0].iov_base = &ss;
5382 1.1 riastrad iov[0].iov_len = ss.ss_len;
5383 1.1 riastrad iov[1].iov_base = mtod(m, void *);
5384 1.1 riastrad iov[1].iov_len = m->m_len;
5385 1.1 riastrad
5386 1.1 riastrad WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len);
5387 1.1 riastrad
5388 1.1 riastrad /* Send decrypted packets to users via a tun. */
5389 1.1 riastrad rumpuser_wg_send_user(wg->wg_user, iov, 2);
5390 1.38 riastrad
5391 1.109 riastrad out: m_freem(m);
5392 1.1 riastrad }
5393 1.1 riastrad
5394 1.1 riastrad static int
5395 1.1 riastrad wg_bind_port_user(struct wg_softc *wg, const uint16_t port)
5396 1.1 riastrad {
5397 1.1 riastrad int error;
5398 1.1 riastrad uint16_t old_port = wg->wg_listen_port;
5399 1.1 riastrad
5400 1.1 riastrad if (port != 0 && old_port == port)
5401 1.1 riastrad return 0;
5402 1.1 riastrad
5403 1.1 riastrad error = rumpuser_wg_sock_bind(wg->wg_user, port);
5404 1.108 riastrad if (error)
5405 1.108 riastrad return error;
5406 1.108 riastrad
5407 1.108 riastrad wg->wg_listen_port = port;
5408 1.108 riastrad return 0;
5409 1.1 riastrad }
5410 1.1 riastrad
5411 1.1 riastrad /*
5412 1.1 riastrad * Receive user packets.
5413 1.1 riastrad */
5414 1.1 riastrad void
5415 1.1 riastrad rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen)
5416 1.1 riastrad {
5417 1.1 riastrad struct ifnet *ifp = &wg->wg_if;
5418 1.1 riastrad struct mbuf *m;
5419 1.1 riastrad const struct sockaddr *dst;
5420 1.108 riastrad int error;
5421 1.1 riastrad
5422 1.1 riastrad WG_TRACE("");
5423 1.1 riastrad
5424 1.1 riastrad dst = iov[0].iov_base;
5425 1.1 riastrad
5426 1.48 riastrad m = m_gethdr(M_DONTWAIT, MT_DATA);
5427 1.1 riastrad if (m == NULL)
5428 1.1 riastrad return;
5429 1.1 riastrad m->m_len = m->m_pkthdr.len = 0;
5430 1.1 riastrad m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base);
5431 1.1 riastrad
5432 1.87 kre WG_DLOG("iov_len=%zu\n", iov[1].iov_len);
5433 1.1 riastrad WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len);
5434 1.1 riastrad
5435 1.108 riastrad error = wg_output(ifp, m, dst, NULL); /* consumes m */
5436 1.108 riastrad if (error)
5437 1.108 riastrad WG_DLOG("wg_output failed, error=%d\n", error);
5438 1.1 riastrad }
5439 1.1 riastrad
5440 1.1 riastrad /*
5441 1.1 riastrad * Receive packets from a peer.
5442 1.1 riastrad */
5443 1.1 riastrad void
5444 1.1 riastrad rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen)
5445 1.1 riastrad {
5446 1.1 riastrad struct mbuf *m;
5447 1.1 riastrad const struct sockaddr *src;
5448 1.78 riastrad int bound;
5449 1.1 riastrad
5450 1.1 riastrad WG_TRACE("");
5451 1.1 riastrad
5452 1.1 riastrad src = iov[0].iov_base;
5453 1.1 riastrad
5454 1.48 riastrad m = m_gethdr(M_DONTWAIT, MT_DATA);
5455 1.1 riastrad if (m == NULL)
5456 1.1 riastrad return;
5457 1.1 riastrad m->m_len = m->m_pkthdr.len = 0;
5458 1.1 riastrad m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base);
5459 1.1 riastrad
5460 1.87 kre WG_DLOG("iov_len=%zu\n", iov[1].iov_len);
5461 1.1 riastrad WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len);
5462 1.1 riastrad
5463 1.78 riastrad bound = curlwp_bind();
5464 1.1 riastrad wg_handle_packet(wg, m, src);
5465 1.78 riastrad curlwp_bindx(bound);
5466 1.1 riastrad }
5467 1.1 riastrad #endif /* WG_RUMPKERNEL */
5468 1.1 riastrad
5469 1.1 riastrad /*
5470 1.1 riastrad * Module infrastructure
5471 1.1 riastrad */
5472 1.1 riastrad #include "if_module.h"
5473 1.1 riastrad
5474 1.65 christos IF_MODULE(MODULE_CLASS_DRIVER, wg, "sodium,blake2s")
5475