if_shmem.c revision 1.76 1 1.76 rin /* $NetBSD: if_shmem.c,v 1.76 2018/12/12 01:51:32 rin Exp $ */
2 1.1 pooka
3 1.1 pooka /*
4 1.39 pooka * Copyright (c) 2009, 2010 Antti Kantee. All Rights Reserved.
5 1.1 pooka *
6 1.1 pooka * Development of this software was supported by The Nokia Foundation.
7 1.1 pooka *
8 1.1 pooka * Redistribution and use in source and binary forms, with or without
9 1.1 pooka * modification, are permitted provided that the following conditions
10 1.1 pooka * are met:
11 1.1 pooka * 1. Redistributions of source code must retain the above copyright
12 1.1 pooka * notice, this list of conditions and the following disclaimer.
13 1.1 pooka * 2. Redistributions in binary form must reproduce the above copyright
14 1.1 pooka * notice, this list of conditions and the following disclaimer in the
15 1.1 pooka * documentation and/or other materials provided with the distribution.
16 1.1 pooka *
17 1.1 pooka * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
18 1.1 pooka * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 1.1 pooka * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20 1.1 pooka * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 1.1 pooka * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 1.1 pooka * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23 1.1 pooka * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 1.1 pooka * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 1.1 pooka * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 1.1 pooka * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 1.1 pooka * SUCH DAMAGE.
28 1.1 pooka */
29 1.1 pooka
30 1.1 pooka #include <sys/cdefs.h>
31 1.76 rin __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.76 2018/12/12 01:51:32 rin Exp $");
32 1.1 pooka
33 1.1 pooka #include <sys/param.h>
34 1.13 pooka #include <sys/atomic.h>
35 1.1 pooka #include <sys/fcntl.h>
36 1.1 pooka #include <sys/kmem.h>
37 1.1 pooka #include <sys/kthread.h>
38 1.1 pooka #include <sys/lock.h>
39 1.31 pooka #include <sys/vmem.h>
40 1.44 tls #include <sys/cprng.h>
41 1.1 pooka
42 1.35 pooka #include <net/bpf.h>
43 1.1 pooka #include <net/if.h>
44 1.37 pooka #include <net/if_dl.h>
45 1.1 pooka #include <net/if_ether.h>
46 1.76 rin #include <net/ether_sw_offload.h>
47 1.1 pooka
48 1.1 pooka #include <netinet/in.h>
49 1.1 pooka #include <netinet/in_var.h>
50 1.1 pooka
51 1.64 pooka #include <rump-sys/kern.h>
52 1.64 pooka #include <rump-sys/net.h>
53 1.64 pooka
54 1.1 pooka #include <rump/rump.h>
55 1.1 pooka #include <rump/rumpuser.h>
56 1.1 pooka
57 1.59 pooka #include "shmif_user.h"
58 1.1 pooka
59 1.29 pooka static int shmif_clone(struct if_clone *, int);
60 1.29 pooka static int shmif_unclone(struct ifnet *);
61 1.29 pooka
62 1.29 pooka struct if_clone shmif_cloner =
63 1.29 pooka IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone);
64 1.29 pooka
65 1.1 pooka /*
66 1.28 pooka * Do r/w prefault for backend pages when attaching the interface.
67 1.29 pooka * At least logically thinking improves performance (although no
68 1.29 pooka * mlocking is done, so they might go away).
69 1.28 pooka */
70 1.28 pooka #define PREFAULT_RW
71 1.28 pooka
72 1.28 pooka /*
73 1.1 pooka * A virtual ethernet interface which uses shared memory from a
74 1.1 pooka * memory mapped file as the bus.
75 1.1 pooka */
76 1.1 pooka
77 1.1 pooka static int shmif_init(struct ifnet *);
78 1.1 pooka static int shmif_ioctl(struct ifnet *, u_long, void *);
79 1.1 pooka static void shmif_start(struct ifnet *);
80 1.76 rin static void shmif_snd(struct ifnet *, struct mbuf *);
81 1.1 pooka static void shmif_stop(struct ifnet *, int);
82 1.1 pooka
83 1.16 pooka #include "shmifvar.h"
84 1.16 pooka
85 1.1 pooka struct shmif_sc {
86 1.1 pooka struct ethercom sc_ec;
87 1.16 pooka struct shmif_mem *sc_busmem;
88 1.1 pooka int sc_memfd;
89 1.1 pooka int sc_kq;
90 1.32 pooka int sc_unit;
91 1.1 pooka
92 1.29 pooka char *sc_backfile;
93 1.29 pooka size_t sc_backfilelen;
94 1.29 pooka
95 1.26 pooka uint64_t sc_devgen;
96 1.1 pooka uint32_t sc_nextpacket;
97 1.32 pooka
98 1.32 pooka kmutex_t sc_mtx;
99 1.32 pooka kcondvar_t sc_cv;
100 1.32 pooka
101 1.32 pooka struct lwp *sc_rcvl;
102 1.32 pooka bool sc_dying;
103 1.63 ozaki
104 1.63 ozaki uint64_t sc_uuid;
105 1.1 pooka };
106 1.1 pooka
107 1.1 pooka static void shmif_rcv(void *);
108 1.1 pooka
109 1.23 pooka #define LOCK_UNLOCKED 0
110 1.23 pooka #define LOCK_LOCKED 1
111 1.23 pooka #define LOCK_COOLDOWN 1001
112 1.23 pooka
113 1.31 pooka vmem_t *shmif_units;
114 1.31 pooka
115 1.52 pooka static void
116 1.52 pooka dowakeup(struct shmif_sc *sc)
117 1.52 pooka {
118 1.52 pooka struct rumpuser_iovec iov;
119 1.52 pooka uint32_t ver = SHMIF_VERSION;
120 1.53 pooka size_t n;
121 1.52 pooka
122 1.52 pooka iov.iov_base = &ver;
123 1.52 pooka iov.iov_len = sizeof(ver);
124 1.53 pooka rumpuser_iovwrite(sc->sc_memfd, &iov, 1, IFMEM_WAKEUP, &n);
125 1.52 pooka }
126 1.52 pooka
127 1.23 pooka /*
128 1.23 pooka * This locking needs work and will misbehave severely if:
129 1.23 pooka * 1) the backing memory has to be paged in
130 1.23 pooka * 2) some lockholder exits while holding the lock
131 1.23 pooka */
132 1.23 pooka static void
133 1.23 pooka shmif_lockbus(struct shmif_mem *busmem)
134 1.23 pooka {
135 1.23 pooka int i = 0;
136 1.23 pooka
137 1.23 pooka while (__predict_false(atomic_cas_32(&busmem->shm_lock,
138 1.23 pooka LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) {
139 1.23 pooka if (__predict_false(++i > LOCK_COOLDOWN)) {
140 1.50 pooka /* wait 1ms */
141 1.54 pooka rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL,
142 1.54 pooka 0, 1000*1000);
143 1.23 pooka i = 0;
144 1.23 pooka }
145 1.23 pooka continue;
146 1.23 pooka }
147 1.23 pooka membar_enter();
148 1.23 pooka }
149 1.23 pooka
150 1.23 pooka static void
151 1.23 pooka shmif_unlockbus(struct shmif_mem *busmem)
152 1.23 pooka {
153 1.61 justin unsigned int old __diagused;
154 1.23 pooka
155 1.23 pooka membar_exit();
156 1.23 pooka old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED);
157 1.23 pooka KASSERT(old == LOCK_LOCKED);
158 1.23 pooka }
159 1.23 pooka
160 1.29 pooka static int
161 1.29 pooka allocif(int unit, struct shmif_sc **scp)
162 1.1 pooka {
163 1.29 pooka uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 };
164 1.1 pooka struct shmif_sc *sc;
165 1.1 pooka struct ifnet *ifp;
166 1.1 pooka uint32_t randnum;
167 1.32 pooka int error;
168 1.1 pooka
169 1.44 tls randnum = cprng_fast32();
170 1.15 pooka memcpy(&enaddr[2], &randnum, sizeof(randnum));
171 1.1 pooka
172 1.1 pooka sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);
173 1.29 pooka sc->sc_memfd = -1;
174 1.32 pooka sc->sc_unit = unit;
175 1.63 ozaki sc->sc_uuid = cprng_fast64();
176 1.29 pooka
177 1.1 pooka ifp = &sc->sc_ec.ec_if;
178 1.1 pooka
179 1.60 christos snprintf(ifp->if_xname, sizeof(ifp->if_xname), "shmif%d", unit);
180 1.29 pooka ifp->if_softc = sc;
181 1.63 ozaki ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
182 1.29 pooka ifp->if_init = shmif_init;
183 1.29 pooka ifp->if_ioctl = shmif_ioctl;
184 1.29 pooka ifp->if_start = shmif_start;
185 1.29 pooka ifp->if_stop = shmif_stop;
186 1.29 pooka ifp->if_mtu = ETHERMTU;
187 1.37 pooka ifp->if_dlt = DLT_EN10MB;
188 1.76 rin ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6 |
189 1.76 rin IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
190 1.76 rin IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
191 1.76 rin IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
192 1.76 rin IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx |
193 1.76 rin IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx;
194 1.29 pooka
195 1.32 pooka mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE);
196 1.32 pooka cv_init(&sc->sc_cv, "shmifcv");
197 1.32 pooka
198 1.73 msaitoh error = if_initialize(ifp);
199 1.73 msaitoh if (error != 0) {
200 1.74 msaitoh aprint_error("shmif%d: if_initialize failed(%d)\n", unit,
201 1.74 msaitoh error);
202 1.73 msaitoh cv_destroy(&sc->sc_cv);
203 1.73 msaitoh mutex_destroy(&sc->sc_mtx);
204 1.73 msaitoh kmem_free(sc, sizeof(*sc));
205 1.73 msaitoh
206 1.73 msaitoh return error;
207 1.73 msaitoh }
208 1.76 rin #if 1
209 1.76 rin char buf[256];
210 1.76 rin
211 1.76 rin if (rumpuser_getparam("RUMP_SHMIF_CAPENABLE", buf, sizeof(buf)) == 0) {
212 1.76 rin uint64_t capen = strtoul(buf, NULL, 0);
213 1.76 rin
214 1.76 rin ifp->if_capenable = capen & ifp->if_capabilities;
215 1.76 rin }
216 1.76 rin #endif
217 1.76 rin
218 1.29 pooka ether_ifattach(ifp, enaddr);
219 1.65 ozaki if_register(ifp);
220 1.29 pooka
221 1.29 pooka aprint_verbose("shmif%d: Ethernet address %s\n",
222 1.32 pooka unit, ether_sprintf(enaddr));
223 1.29 pooka
224 1.29 pooka if (scp)
225 1.29 pooka *scp = sc;
226 1.29 pooka
227 1.32 pooka error = 0;
228 1.32 pooka if (rump_threads) {
229 1.32 pooka error = kthread_create(PRI_NONE,
230 1.40 rmind KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL,
231 1.32 pooka shmif_rcv, ifp, &sc->sc_rcvl, "shmif");
232 1.32 pooka } else {
233 1.32 pooka printf("WARNING: threads not enabled, shmif NOT working\n");
234 1.32 pooka }
235 1.32 pooka
236 1.32 pooka if (error) {
237 1.32 pooka shmif_unclone(ifp);
238 1.32 pooka }
239 1.32 pooka
240 1.32 pooka return error;
241 1.29 pooka }
242 1.29 pooka
243 1.29 pooka static int
244 1.29 pooka initbackend(struct shmif_sc *sc, int memfd)
245 1.29 pooka {
246 1.29 pooka volatile uint8_t v;
247 1.29 pooka volatile uint8_t *p;
248 1.53 pooka void *mem;
249 1.29 pooka int error;
250 1.29 pooka
251 1.53 pooka error = rumpcomp_shmif_mmap(memfd, BUSMEM_SIZE, &mem);
252 1.1 pooka if (error)
253 1.29 pooka return error;
254 1.53 pooka sc->sc_busmem = mem;
255 1.17 pooka
256 1.29 pooka if (sc->sc_busmem->shm_magic
257 1.29 pooka && sc->sc_busmem->shm_magic != SHMIF_MAGIC) {
258 1.29 pooka printf("bus is not magical");
259 1.29 pooka rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
260 1.69 msaitoh return ENOEXEC;
261 1.29 pooka }
262 1.28 pooka
263 1.36 pooka /*
264 1.36 pooka * Prefault in pages to minimize runtime penalty with buslock.
265 1.36 pooka * Use 512 instead of PAGE_SIZE to make sure we catch cases where
266 1.36 pooka * rump kernel PAGE_SIZE > host page size.
267 1.36 pooka */
268 1.28 pooka for (p = (uint8_t *)sc->sc_busmem;
269 1.28 pooka p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
270 1.36 pooka p += 512)
271 1.28 pooka v = *p;
272 1.28 pooka
273 1.19 pooka shmif_lockbus(sc->sc_busmem);
274 1.19 pooka /* we're first? initialize bus */
275 1.19 pooka if (sc->sc_busmem->shm_magic == 0) {
276 1.19 pooka sc->sc_busmem->shm_magic = SHMIF_MAGIC;
277 1.19 pooka sc->sc_busmem->shm_first = BUSMEM_DATASIZE;
278 1.19 pooka }
279 1.19 pooka
280 1.16 pooka sc->sc_nextpacket = sc->sc_busmem->shm_last;
281 1.26 pooka sc->sc_devgen = sc->sc_busmem->shm_gen;
282 1.28 pooka
283 1.28 pooka #ifdef PREFAULT_RW
284 1.28 pooka for (p = (uint8_t *)sc->sc_busmem;
285 1.28 pooka p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
286 1.28 pooka p += PAGE_SIZE) {
287 1.28 pooka v = *p;
288 1.28 pooka *p = v;
289 1.28 pooka }
290 1.28 pooka #endif
291 1.19 pooka shmif_unlockbus(sc->sc_busmem);
292 1.1 pooka
293 1.53 pooka sc->sc_kq = -1;
294 1.53 pooka error = rumpcomp_shmif_watchsetup(&sc->sc_kq, memfd);
295 1.53 pooka if (error) {
296 1.32 pooka rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
297 1.29 pooka return error;
298 1.32 pooka }
299 1.1 pooka
300 1.29 pooka sc->sc_memfd = memfd;
301 1.32 pooka
302 1.32 pooka return error;
303 1.29 pooka }
304 1.29 pooka
305 1.29 pooka static void
306 1.29 pooka finibackend(struct shmif_sc *sc)
307 1.29 pooka {
308 1.29 pooka
309 1.32 pooka if (sc->sc_backfile == NULL)
310 1.32 pooka return;
311 1.32 pooka
312 1.32 pooka if (sc->sc_backfile) {
313 1.32 pooka kmem_free(sc->sc_backfile, sc->sc_backfilelen);
314 1.32 pooka sc->sc_backfile = NULL;
315 1.32 pooka sc->sc_backfilelen = 0;
316 1.32 pooka }
317 1.29 pooka
318 1.29 pooka rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
319 1.53 pooka rumpuser_close(sc->sc_memfd);
320 1.53 pooka rumpuser_close(sc->sc_kq);
321 1.32 pooka
322 1.32 pooka sc->sc_memfd = -1;
323 1.29 pooka }
324 1.29 pooka
325 1.29 pooka int
326 1.29 pooka rump_shmif_create(const char *path, int *ifnum)
327 1.29 pooka {
328 1.29 pooka struct shmif_sc *sc;
329 1.43 dyoung vmem_addr_t t;
330 1.33 pooka int unit, error;
331 1.33 pooka int memfd = -1; /* XXXgcc */
332 1.29 pooka
333 1.33 pooka if (path) {
334 1.53 pooka error = rumpuser_open(path,
335 1.53 pooka RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
336 1.53 pooka if (error)
337 1.33 pooka return error;
338 1.33 pooka }
339 1.1 pooka
340 1.43 dyoung error = vmem_xalloc(shmif_units, 1, 0, 0, 0,
341 1.43 dyoung VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_INSTANTFIT | VM_SLEEP, &t);
342 1.43 dyoung
343 1.43 dyoung if (error != 0) {
344 1.43 dyoung if (path)
345 1.53 pooka rumpuser_close(memfd);
346 1.43 dyoung return error;
347 1.43 dyoung }
348 1.43 dyoung
349 1.43 dyoung unit = t - 1;
350 1.31 pooka
351 1.32 pooka if ((error = allocif(unit, &sc)) != 0) {
352 1.33 pooka if (path)
353 1.53 pooka rumpuser_close(memfd);
354 1.29 pooka return error;
355 1.29 pooka }
356 1.33 pooka
357 1.33 pooka if (!path)
358 1.33 pooka goto out;
359 1.33 pooka
360 1.29 pooka error = initbackend(sc, memfd);
361 1.29 pooka if (error) {
362 1.32 pooka shmif_unclone(&sc->sc_ec.ec_if);
363 1.29 pooka return error;
364 1.29 pooka }
365 1.1 pooka
366 1.29 pooka sc->sc_backfilelen = strlen(path)+1;
367 1.29 pooka sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP);
368 1.29 pooka strcpy(sc->sc_backfile, path);
369 1.12 pooka
370 1.33 pooka out:
371 1.2 pooka if (ifnum)
372 1.32 pooka *ifnum = unit;
373 1.29 pooka
374 1.1 pooka return 0;
375 1.29 pooka }
376 1.1 pooka
377 1.29 pooka static int
378 1.29 pooka shmif_clone(struct if_clone *ifc, int unit)
379 1.29 pooka {
380 1.61 justin int rc __diagused;
381 1.43 dyoung vmem_addr_t unit2;
382 1.29 pooka
383 1.31 pooka /*
384 1.31 pooka * Ok, we know the unit number, but we must still reserve it.
385 1.31 pooka * Otherwise the wildcard-side of things might get the same one.
386 1.31 pooka * This is slightly offset-happy due to vmem. First, we offset
387 1.31 pooka * the range of unit numbers by +1 since vmem cannot deal with
388 1.41 dyoung * ranges starting from 0. Talk about uuuh.
389 1.31 pooka */
390 1.43 dyoung rc = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+1,
391 1.43 dyoung VM_SLEEP | VM_INSTANTFIT, &unit2);
392 1.43 dyoung KASSERT(rc == 0 && unit2-1 == unit);
393 1.29 pooka
394 1.29 pooka return allocif(unit, NULL);
395 1.29 pooka }
396 1.29 pooka
397 1.29 pooka static int
398 1.29 pooka shmif_unclone(struct ifnet *ifp)
399 1.29 pooka {
400 1.32 pooka struct shmif_sc *sc = ifp->if_softc;
401 1.32 pooka
402 1.32 pooka shmif_stop(ifp, 1);
403 1.32 pooka if_down(ifp);
404 1.32 pooka
405 1.32 pooka mutex_enter(&sc->sc_mtx);
406 1.32 pooka sc->sc_dying = true;
407 1.32 pooka cv_broadcast(&sc->sc_cv);
408 1.32 pooka mutex_exit(&sc->sc_mtx);
409 1.29 pooka
410 1.32 pooka if (sc->sc_rcvl)
411 1.32 pooka kthread_join(sc->sc_rcvl);
412 1.32 pooka sc->sc_rcvl = NULL;
413 1.32 pooka
414 1.72 ozaki /*
415 1.72 ozaki * Need to be called after the kthread left, otherwise closing kqueue
416 1.72 ozaki * (sc_kq) hangs sometimes perhaps because of a race condition between
417 1.72 ozaki * close and kevent in the kthread on the kqueue.
418 1.72 ozaki */
419 1.72 ozaki finibackend(sc);
420 1.72 ozaki
421 1.32 pooka vmem_xfree(shmif_units, sc->sc_unit+1, 1);
422 1.32 pooka
423 1.32 pooka ether_ifdetach(ifp);
424 1.32 pooka if_detach(ifp);
425 1.32 pooka
426 1.32 pooka cv_destroy(&sc->sc_cv);
427 1.32 pooka mutex_destroy(&sc->sc_mtx);
428 1.32 pooka
429 1.32 pooka kmem_free(sc, sizeof(*sc));
430 1.32 pooka
431 1.32 pooka return 0;
432 1.1 pooka }
433 1.1 pooka
434 1.1 pooka static int
435 1.1 pooka shmif_init(struct ifnet *ifp)
436 1.1 pooka {
437 1.29 pooka struct shmif_sc *sc = ifp->if_softc;
438 1.4 pooka int error = 0;
439 1.4 pooka
440 1.29 pooka if (sc->sc_memfd == -1)
441 1.29 pooka return ENXIO;
442 1.32 pooka KASSERT(sc->sc_busmem);
443 1.29 pooka
444 1.32 pooka ifp->if_flags |= IFF_RUNNING;
445 1.32 pooka
446 1.32 pooka mutex_enter(&sc->sc_mtx);
447 1.32 pooka sc->sc_nextpacket = sc->sc_busmem->shm_last;
448 1.32 pooka sc->sc_devgen = sc->sc_busmem->shm_gen;
449 1.32 pooka
450 1.32 pooka cv_broadcast(&sc->sc_cv);
451 1.32 pooka mutex_exit(&sc->sc_mtx);
452 1.1 pooka
453 1.4 pooka return error;
454 1.1 pooka }
455 1.1 pooka
456 1.1 pooka static int
457 1.1 pooka shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
458 1.1 pooka {
459 1.29 pooka struct shmif_sc *sc = ifp->if_softc;
460 1.29 pooka struct ifdrv *ifd;
461 1.29 pooka char *path;
462 1.32 pooka int s, rv, memfd;
463 1.1 pooka
464 1.1 pooka s = splnet();
465 1.29 pooka switch (cmd) {
466 1.29 pooka case SIOCGLINKSTR:
467 1.29 pooka ifd = data;
468 1.29 pooka
469 1.29 pooka if (sc->sc_backfilelen == 0) {
470 1.29 pooka rv = ENOENT;
471 1.29 pooka break;
472 1.29 pooka }
473 1.29 pooka
474 1.29 pooka ifd->ifd_len = sc->sc_backfilelen;
475 1.29 pooka if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) {
476 1.29 pooka rv = 0;
477 1.29 pooka break;
478 1.29 pooka }
479 1.29 pooka
480 1.29 pooka if (ifd->ifd_cmd != 0) {
481 1.29 pooka rv = EINVAL;
482 1.29 pooka break;
483 1.29 pooka }
484 1.29 pooka
485 1.29 pooka rv = copyoutstr(sc->sc_backfile, ifd->ifd_data,
486 1.29 pooka MIN(sc->sc_backfilelen, ifd->ifd_len), NULL);
487 1.29 pooka break;
488 1.29 pooka case SIOCSLINKSTR:
489 1.29 pooka if (ifp->if_flags & IFF_UP) {
490 1.29 pooka rv = EBUSY;
491 1.29 pooka break;
492 1.29 pooka }
493 1.29 pooka
494 1.29 pooka ifd = data;
495 1.29 pooka if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
496 1.29 pooka finibackend(sc);
497 1.29 pooka rv = 0;
498 1.29 pooka break;
499 1.29 pooka } else if (ifd->ifd_cmd != 0) {
500 1.29 pooka rv = EINVAL;
501 1.29 pooka break;
502 1.29 pooka } else if (sc->sc_backfile) {
503 1.29 pooka rv = EBUSY;
504 1.29 pooka break;
505 1.29 pooka }
506 1.29 pooka
507 1.29 pooka if (ifd->ifd_len > MAXPATHLEN) {
508 1.29 pooka rv = E2BIG;
509 1.29 pooka break;
510 1.29 pooka } else if (ifd->ifd_len < 1) {
511 1.29 pooka rv = EINVAL;
512 1.29 pooka break;
513 1.29 pooka }
514 1.29 pooka
515 1.29 pooka path = kmem_alloc(ifd->ifd_len, KM_SLEEP);
516 1.29 pooka rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL);
517 1.29 pooka if (rv) {
518 1.29 pooka kmem_free(path, ifd->ifd_len);
519 1.29 pooka break;
520 1.29 pooka }
521 1.53 pooka rv = rumpuser_open(path,
522 1.53 pooka RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
523 1.53 pooka if (rv) {
524 1.29 pooka kmem_free(path, ifd->ifd_len);
525 1.29 pooka break;
526 1.29 pooka }
527 1.29 pooka rv = initbackend(sc, memfd);
528 1.29 pooka if (rv) {
529 1.29 pooka kmem_free(path, ifd->ifd_len);
530 1.53 pooka rumpuser_close(memfd);
531 1.29 pooka break;
532 1.29 pooka }
533 1.29 pooka sc->sc_backfile = path;
534 1.29 pooka sc->sc_backfilelen = ifd->ifd_len;
535 1.29 pooka
536 1.29 pooka break;
537 1.29 pooka default:
538 1.29 pooka rv = ether_ioctl(ifp, cmd, data);
539 1.29 pooka if (rv == ENETRESET)
540 1.29 pooka rv = 0;
541 1.29 pooka break;
542 1.29 pooka }
543 1.1 pooka splx(s);
544 1.1 pooka
545 1.1 pooka return rv;
546 1.1 pooka }
547 1.1 pooka
548 1.1 pooka static void
549 1.1 pooka shmif_start(struct ifnet *ifp)
550 1.1 pooka {
551 1.1 pooka struct shmif_sc *sc = ifp->if_softc;
552 1.76 rin struct mbuf *m, *n;
553 1.1 pooka bool wrote = false;
554 1.1 pooka
555 1.26 pooka ifp->if_flags |= IFF_OACTIVE;
556 1.26 pooka
557 1.1 pooka for (;;) {
558 1.76 rin IF_DEQUEUE(&ifp->if_snd, m);
559 1.76 rin if (m == NULL)
560 1.76 rin break;
561 1.20 pooka
562 1.76 rin m = ether_sw_offload_tx(ifp, m);
563 1.76 rin if (m == NULL) {
564 1.76 rin ifp->if_oerrors++;
565 1.1 pooka break;
566 1.1 pooka }
567 1.1 pooka
568 1.76 rin do {
569 1.76 rin n = m->m_nextpkt;
570 1.76 rin shmif_snd(ifp, m);
571 1.76 rin m = n;
572 1.76 rin } while (m != NULL);
573 1.1 pooka
574 1.1 pooka wrote = true;
575 1.1 pooka }
576 1.26 pooka
577 1.26 pooka ifp->if_flags &= ~IFF_OACTIVE;
578 1.26 pooka
579 1.32 pooka /* wakeup? */
580 1.52 pooka if (wrote) {
581 1.52 pooka dowakeup(sc);
582 1.52 pooka }
583 1.1 pooka }
584 1.1 pooka
585 1.76 rin /* send everything in-context since it's just a matter of mem-to-mem copy */
586 1.76 rin static void
587 1.76 rin shmif_snd(struct ifnet *ifp, struct mbuf *m0)
588 1.76 rin {
589 1.76 rin struct shmif_sc *sc = ifp->if_softc;
590 1.76 rin struct shmif_mem *busmem = sc->sc_busmem;
591 1.76 rin struct shmif_pkthdr sp;
592 1.76 rin struct timeval tv;
593 1.76 rin struct mbuf *m;
594 1.76 rin uint32_t dataoff;
595 1.76 rin uint32_t pktsize, pktwrote;
596 1.76 rin bool wrap;
597 1.76 rin
598 1.76 rin pktsize = 0;
599 1.76 rin for (m = m0; m != NULL; m = m->m_next) {
600 1.76 rin pktsize += m->m_len;
601 1.76 rin }
602 1.76 rin KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN);
603 1.76 rin
604 1.76 rin getmicrouptime(&tv);
605 1.76 rin sp.sp_len = pktsize;
606 1.76 rin sp.sp_sec = tv.tv_sec;
607 1.76 rin sp.sp_usec = tv.tv_usec;
608 1.76 rin sp.sp_sender = sc->sc_uuid;
609 1.76 rin
610 1.76 rin bpf_mtap(ifp, m0, BPF_D_OUT);
611 1.76 rin
612 1.76 rin shmif_lockbus(busmem);
613 1.76 rin KASSERT(busmem->shm_magic == SHMIF_MAGIC);
614 1.76 rin busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last);
615 1.76 rin
616 1.76 rin wrap = false;
617 1.76 rin dataoff =
618 1.76 rin shmif_buswrite(busmem, busmem->shm_last, &sp, sizeof(sp), &wrap);
619 1.76 rin pktwrote = 0;
620 1.76 rin for (m = m0; m != NULL; m = m->m_next) {
621 1.76 rin pktwrote += m->m_len;
622 1.76 rin dataoff = shmif_buswrite(busmem, dataoff, mtod(m, void *),
623 1.76 rin m->m_len, &wrap);
624 1.76 rin }
625 1.76 rin KASSERT(pktwrote == pktsize);
626 1.76 rin if (wrap) {
627 1.76 rin busmem->shm_gen++;
628 1.76 rin DPRINTF(("bus generation now %" PRIu64 "\n", busmem->shm_gen));
629 1.76 rin }
630 1.76 rin shmif_unlockbus(busmem);
631 1.76 rin
632 1.76 rin m_freem(m0);
633 1.76 rin ifp->if_opackets++;
634 1.76 rin
635 1.76 rin DPRINTF(("shmif_start: send %d bytes at off %d\n", pktsize,
636 1.76 rin busmem->shm_last));
637 1.76 rin }
638 1.76 rin
639 1.1 pooka static void
640 1.1 pooka shmif_stop(struct ifnet *ifp, int disable)
641 1.1 pooka {
642 1.32 pooka struct shmif_sc *sc = ifp->if_softc;
643 1.1 pooka
644 1.32 pooka ifp->if_flags &= ~IFF_RUNNING;
645 1.32 pooka membar_producer();
646 1.32 pooka
647 1.32 pooka /*
648 1.32 pooka * wakeup thread. this will of course wake up all bus
649 1.32 pooka * listeners, but that's life.
650 1.32 pooka */
651 1.52 pooka if (sc->sc_memfd != -1) {
652 1.52 pooka dowakeup(sc);
653 1.52 pooka }
654 1.1 pooka }
655 1.1 pooka
656 1.27 pooka
657 1.27 pooka /*
658 1.27 pooka * Check if we have been sleeping too long. Basically,
659 1.27 pooka * our in-sc nextpkt must by first <= nextpkt <= last"+1".
660 1.27 pooka * We use the fact that first is guaranteed to never overlap
661 1.27 pooka * with the last frame in the ring.
662 1.27 pooka */
663 1.27 pooka static __inline bool
664 1.27 pooka stillvalid_p(struct shmif_sc *sc)
665 1.27 pooka {
666 1.27 pooka struct shmif_mem *busmem = sc->sc_busmem;
667 1.27 pooka unsigned gendiff = busmem->shm_gen - sc->sc_devgen;
668 1.27 pooka uint32_t lastoff, devoff;
669 1.27 pooka
670 1.27 pooka KASSERT(busmem->shm_first != busmem->shm_last);
671 1.27 pooka
672 1.27 pooka /* normalize onto a 2x busmem chunk */
673 1.27 pooka devoff = sc->sc_nextpacket;
674 1.27 pooka lastoff = shmif_nextpktoff(busmem, busmem->shm_last);
675 1.27 pooka
676 1.27 pooka /* trivial case */
677 1.27 pooka if (gendiff > 1)
678 1.27 pooka return false;
679 1.27 pooka KASSERT(gendiff <= 1);
680 1.27 pooka
681 1.27 pooka /* Normalize onto 2x busmem chunk */
682 1.27 pooka if (busmem->shm_first >= lastoff) {
683 1.27 pooka lastoff += BUSMEM_DATASIZE;
684 1.27 pooka if (gendiff == 0)
685 1.27 pooka devoff += BUSMEM_DATASIZE;
686 1.27 pooka } else {
687 1.27 pooka if (gendiff)
688 1.27 pooka return false;
689 1.27 pooka }
690 1.27 pooka
691 1.27 pooka return devoff >= busmem->shm_first && devoff <= lastoff;
692 1.27 pooka }
693 1.27 pooka
694 1.1 pooka static void
695 1.1 pooka shmif_rcv(void *arg)
696 1.1 pooka {
697 1.1 pooka struct ifnet *ifp = arg;
698 1.1 pooka struct shmif_sc *sc = ifp->if_softc;
699 1.32 pooka struct shmif_mem *busmem;
700 1.1 pooka struct mbuf *m = NULL;
701 1.1 pooka struct ether_header *eth;
702 1.27 pooka uint32_t nextpkt;
703 1.37 pooka bool wrap, passup;
704 1.1 pooka int error;
705 1.57 pooka const int align
706 1.57 pooka = ALIGN(sizeof(struct ether_header)) - sizeof(struct ether_header);
707 1.1 pooka
708 1.32 pooka reup:
709 1.32 pooka mutex_enter(&sc->sc_mtx);
710 1.32 pooka while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying)
711 1.32 pooka cv_wait(&sc->sc_cv, &sc->sc_mtx);
712 1.32 pooka mutex_exit(&sc->sc_mtx);
713 1.32 pooka
714 1.32 pooka busmem = sc->sc_busmem;
715 1.32 pooka
716 1.32 pooka while (ifp->if_flags & IFF_RUNNING) {
717 1.20 pooka struct shmif_pkthdr sp;
718 1.20 pooka
719 1.1 pooka if (m == NULL) {
720 1.1 pooka m = m_gethdr(M_WAIT, MT_DATA);
721 1.1 pooka MCLGET(m, M_WAIT);
722 1.57 pooka m->m_data += align;
723 1.1 pooka }
724 1.1 pooka
725 1.47 pooka DPRINTF(("waiting %d/%" PRIu64 "\n",
726 1.47 pooka sc->sc_nextpacket, sc->sc_devgen));
727 1.26 pooka KASSERT(m->m_flags & M_EXT);
728 1.1 pooka
729 1.26 pooka shmif_lockbus(busmem);
730 1.26 pooka KASSERT(busmem->shm_magic == SHMIF_MAGIC);
731 1.27 pooka KASSERT(busmem->shm_gen >= sc->sc_devgen);
732 1.1 pooka
733 1.1 pooka /* need more data? */
734 1.27 pooka if (sc->sc_devgen == busmem->shm_gen &&
735 1.26 pooka shmif_nextpktoff(busmem, busmem->shm_last)
736 1.26 pooka == sc->sc_nextpacket) {
737 1.26 pooka shmif_unlockbus(busmem);
738 1.71 ozaki error = rumpcomp_shmif_watchwait(sc->sc_kq);
739 1.1 pooka if (__predict_false(error))
740 1.1 pooka printf("shmif_rcv: wait failed %d\n", error);
741 1.32 pooka membar_consumer();
742 1.1 pooka continue;
743 1.1 pooka }
744 1.1 pooka
745 1.27 pooka if (stillvalid_p(sc)) {
746 1.27 pooka nextpkt = sc->sc_nextpacket;
747 1.27 pooka } else {
748 1.27 pooka KASSERT(busmem->shm_gen > 0);
749 1.26 pooka nextpkt = busmem->shm_first;
750 1.26 pooka if (busmem->shm_first > busmem->shm_last)
751 1.27 pooka sc->sc_devgen = busmem->shm_gen - 1;
752 1.26 pooka else
753 1.27 pooka sc->sc_devgen = busmem->shm_gen;
754 1.47 pooka DPRINTF(("dev %p overrun, new data: %d/%" PRIu64 "\n",
755 1.27 pooka sc, nextpkt, sc->sc_devgen));
756 1.26 pooka }
757 1.26 pooka
758 1.26 pooka /*
759 1.26 pooka * If our read pointer is ahead the bus last write, our
760 1.26 pooka * generation must be one behind.
761 1.26 pooka */
762 1.26 pooka KASSERT(!(nextpkt > busmem->shm_last
763 1.27 pooka && sc->sc_devgen == busmem->shm_gen));
764 1.26 pooka
765 1.24 pooka wrap = false;
766 1.26 pooka nextpkt = shmif_busread(busmem, &sp,
767 1.26 pooka nextpkt, sizeof(sp), &wrap);
768 1.26 pooka KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN);
769 1.26 pooka nextpkt = shmif_busread(busmem, mtod(m, void *),
770 1.26 pooka nextpkt, sp.sp_len, &wrap);
771 1.1 pooka
772 1.1 pooka DPRINTF(("shmif_rcv: read packet of length %d at %d\n",
773 1.20 pooka sp.sp_len, nextpkt));
774 1.1 pooka
775 1.26 pooka sc->sc_nextpacket = nextpkt;
776 1.19 pooka shmif_unlockbus(sc->sc_busmem);
777 1.1 pooka
778 1.27 pooka if (wrap) {
779 1.26 pooka sc->sc_devgen++;
780 1.47 pooka DPRINTF(("dev %p generation now %" PRIu64 "\n",
781 1.27 pooka sc, sc->sc_devgen));
782 1.27 pooka }
783 1.26 pooka
784 1.56 pooka /*
785 1.56 pooka * Ignore packets too short to possibly be valid.
786 1.56 pooka * This is hit at least for the first frame on a new bus.
787 1.56 pooka */
788 1.55 pooka if (__predict_false(sp.sp_len < ETHER_HDR_LEN)) {
789 1.55 pooka DPRINTF(("shmif read packet len %d < ETHER_HDR_LEN\n",
790 1.55 pooka sp.sp_len));
791 1.55 pooka continue;
792 1.55 pooka }
793 1.55 pooka
794 1.20 pooka m->m_len = m->m_pkthdr.len = sp.sp_len;
795 1.67 ozaki m_set_rcvif(m, ifp);
796 1.1 pooka
797 1.37 pooka /*
798 1.37 pooka * Test if we want to pass the packet upwards
799 1.37 pooka */
800 1.1 pooka eth = mtod(m, struct ether_header *);
801 1.63 ozaki if (sp.sp_sender == sc->sc_uuid) {
802 1.63 ozaki passup = false;
803 1.63 ozaki } else if (memcmp(eth->ether_dhost, CLLADDR(ifp->if_sadl),
804 1.37 pooka ETHER_ADDR_LEN) == 0) {
805 1.37 pooka passup = true;
806 1.46 pooka } else if (ETHER_IS_MULTICAST(eth->ether_dhost)) {
807 1.37 pooka passup = true;
808 1.37 pooka } else if (ifp->if_flags & IFF_PROMISC) {
809 1.37 pooka m->m_flags |= M_PROMISC;
810 1.37 pooka passup = true;
811 1.38 pooka } else {
812 1.38 pooka passup = false;
813 1.37 pooka }
814 1.37 pooka
815 1.37 pooka if (passup) {
816 1.68 ozaki int bound;
817 1.76 rin
818 1.76 rin m = ether_sw_offload_rx(ifp, m);
819 1.76 rin
820 1.22 pooka KERNEL_LOCK(1, NULL);
821 1.66 ozaki /* Prevent LWP migrations between CPUs for psref(9) */
822 1.68 ozaki bound = curlwp_bind();
823 1.65 ozaki if_input(ifp, m);
824 1.68 ozaki curlwp_bindx(bound);
825 1.22 pooka KERNEL_UNLOCK_ONE(NULL);
826 1.76 rin
827 1.1 pooka m = NULL;
828 1.1 pooka }
829 1.37 pooka /* else: reuse mbuf for a future packet */
830 1.1 pooka }
831 1.32 pooka m_freem(m);
832 1.32 pooka m = NULL;
833 1.32 pooka
834 1.32 pooka if (!sc->sc_dying)
835 1.32 pooka goto reup;
836 1.1 pooka
837 1.32 pooka kthread_exit(0);
838 1.1 pooka }
839