if_shmem.c revision 1.60 1 1.60 christos /* $NetBSD: if_shmem.c,v 1.60 2014/03/20 20:42:08 christos Exp $ */
2 1.1 pooka
3 1.1 pooka /*
4 1.39 pooka * Copyright (c) 2009, 2010 Antti Kantee. All Rights Reserved.
5 1.1 pooka *
6 1.1 pooka * Development of this software was supported by The Nokia Foundation.
7 1.1 pooka *
8 1.1 pooka * Redistribution and use in source and binary forms, with or without
9 1.1 pooka * modification, are permitted provided that the following conditions
10 1.1 pooka * are met:
11 1.1 pooka * 1. Redistributions of source code must retain the above copyright
12 1.1 pooka * notice, this list of conditions and the following disclaimer.
13 1.1 pooka * 2. Redistributions in binary form must reproduce the above copyright
14 1.1 pooka * notice, this list of conditions and the following disclaimer in the
15 1.1 pooka * documentation and/or other materials provided with the distribution.
16 1.1 pooka *
17 1.1 pooka * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
18 1.1 pooka * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 1.1 pooka * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20 1.1 pooka * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 1.1 pooka * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 1.1 pooka * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23 1.1 pooka * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 1.1 pooka * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 1.1 pooka * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 1.1 pooka * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 1.1 pooka * SUCH DAMAGE.
28 1.1 pooka */
29 1.1 pooka
30 1.1 pooka #include <sys/cdefs.h>
31 1.60 christos __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.60 2014/03/20 20:42:08 christos Exp $");
32 1.1 pooka
33 1.1 pooka #include <sys/param.h>
34 1.13 pooka #include <sys/atomic.h>
35 1.1 pooka #include <sys/fcntl.h>
36 1.1 pooka #include <sys/kmem.h>
37 1.1 pooka #include <sys/kthread.h>
38 1.1 pooka #include <sys/lock.h>
39 1.31 pooka #include <sys/vmem.h>
40 1.44 tls #include <sys/cprng.h>
41 1.1 pooka
42 1.35 pooka #include <net/bpf.h>
43 1.1 pooka #include <net/if.h>
44 1.37 pooka #include <net/if_dl.h>
45 1.1 pooka #include <net/if_ether.h>
46 1.1 pooka
47 1.1 pooka #include <netinet/in.h>
48 1.1 pooka #include <netinet/in_var.h>
49 1.1 pooka
50 1.1 pooka #include <rump/rump.h>
51 1.1 pooka #include <rump/rumpuser.h>
52 1.1 pooka
53 1.1 pooka #include "rump_private.h"
54 1.9 pooka #include "rump_net_private.h"
55 1.59 pooka #include "shmif_user.h"
56 1.1 pooka
57 1.29 pooka static int shmif_clone(struct if_clone *, int);
58 1.29 pooka static int shmif_unclone(struct ifnet *);
59 1.29 pooka
60 1.29 pooka struct if_clone shmif_cloner =
61 1.29 pooka IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone);
62 1.29 pooka
63 1.1 pooka /*
64 1.28 pooka * Do r/w prefault for backend pages when attaching the interface.
65 1.29 pooka * At least logically thinking improves performance (although no
66 1.29 pooka * mlocking is done, so they might go away).
67 1.28 pooka */
68 1.28 pooka #define PREFAULT_RW
69 1.28 pooka
70 1.28 pooka /*
71 1.1 pooka * A virtual ethernet interface which uses shared memory from a
72 1.1 pooka * memory mapped file as the bus.
73 1.1 pooka */
74 1.1 pooka
75 1.1 pooka static int shmif_init(struct ifnet *);
76 1.1 pooka static int shmif_ioctl(struct ifnet *, u_long, void *);
77 1.1 pooka static void shmif_start(struct ifnet *);
78 1.1 pooka static void shmif_stop(struct ifnet *, int);
79 1.1 pooka
80 1.16 pooka #include "shmifvar.h"
81 1.16 pooka
82 1.1 pooka struct shmif_sc {
83 1.1 pooka struct ethercom sc_ec;
84 1.16 pooka struct shmif_mem *sc_busmem;
85 1.1 pooka int sc_memfd;
86 1.1 pooka int sc_kq;
87 1.32 pooka int sc_unit;
88 1.1 pooka
89 1.29 pooka char *sc_backfile;
90 1.29 pooka size_t sc_backfilelen;
91 1.29 pooka
92 1.26 pooka uint64_t sc_devgen;
93 1.1 pooka uint32_t sc_nextpacket;
94 1.32 pooka
95 1.32 pooka kmutex_t sc_mtx;
96 1.32 pooka kcondvar_t sc_cv;
97 1.32 pooka
98 1.32 pooka struct lwp *sc_rcvl;
99 1.32 pooka bool sc_dying;
100 1.1 pooka };
101 1.1 pooka
102 1.1 pooka static void shmif_rcv(void *);
103 1.1 pooka
104 1.23 pooka #define LOCK_UNLOCKED 0
105 1.23 pooka #define LOCK_LOCKED 1
106 1.23 pooka #define LOCK_COOLDOWN 1001
107 1.23 pooka
108 1.31 pooka vmem_t *shmif_units;
109 1.31 pooka
110 1.52 pooka static void
111 1.52 pooka dowakeup(struct shmif_sc *sc)
112 1.52 pooka {
113 1.52 pooka struct rumpuser_iovec iov;
114 1.52 pooka uint32_t ver = SHMIF_VERSION;
115 1.53 pooka size_t n;
116 1.52 pooka
117 1.52 pooka iov.iov_base = &ver;
118 1.52 pooka iov.iov_len = sizeof(ver);
119 1.53 pooka rumpuser_iovwrite(sc->sc_memfd, &iov, 1, IFMEM_WAKEUP, &n);
120 1.52 pooka }
121 1.52 pooka
122 1.23 pooka /*
123 1.23 pooka * This locking needs work and will misbehave severely if:
124 1.23 pooka * 1) the backing memory has to be paged in
125 1.23 pooka * 2) some lockholder exits while holding the lock
126 1.23 pooka */
127 1.23 pooka static void
128 1.23 pooka shmif_lockbus(struct shmif_mem *busmem)
129 1.23 pooka {
130 1.23 pooka int i = 0;
131 1.23 pooka
132 1.23 pooka while (__predict_false(atomic_cas_32(&busmem->shm_lock,
133 1.23 pooka LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) {
134 1.23 pooka if (__predict_false(++i > LOCK_COOLDOWN)) {
135 1.50 pooka /* wait 1ms */
136 1.54 pooka rumpuser_clock_sleep(RUMPUSER_CLOCK_RELWALL,
137 1.54 pooka 0, 1000*1000);
138 1.23 pooka i = 0;
139 1.23 pooka }
140 1.23 pooka continue;
141 1.23 pooka }
142 1.23 pooka membar_enter();
143 1.23 pooka }
144 1.23 pooka
145 1.23 pooka static void
146 1.23 pooka shmif_unlockbus(struct shmif_mem *busmem)
147 1.23 pooka {
148 1.23 pooka unsigned int old;
149 1.23 pooka
150 1.23 pooka membar_exit();
151 1.23 pooka old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED);
152 1.23 pooka KASSERT(old == LOCK_LOCKED);
153 1.23 pooka }
154 1.23 pooka
155 1.29 pooka static int
156 1.29 pooka allocif(int unit, struct shmif_sc **scp)
157 1.1 pooka {
158 1.29 pooka uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 };
159 1.1 pooka struct shmif_sc *sc;
160 1.1 pooka struct ifnet *ifp;
161 1.1 pooka uint32_t randnum;
162 1.32 pooka int error;
163 1.1 pooka
164 1.44 tls randnum = cprng_fast32();
165 1.15 pooka memcpy(&enaddr[2], &randnum, sizeof(randnum));
166 1.1 pooka
167 1.1 pooka sc = kmem_zalloc(sizeof(*sc), KM_SLEEP);
168 1.29 pooka sc->sc_memfd = -1;
169 1.32 pooka sc->sc_unit = unit;
170 1.29 pooka
171 1.1 pooka ifp = &sc->sc_ec.ec_if;
172 1.1 pooka
173 1.60 christos snprintf(ifp->if_xname, sizeof(ifp->if_xname), "shmif%d", unit);
174 1.29 pooka ifp->if_softc = sc;
175 1.29 pooka ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
176 1.29 pooka ifp->if_init = shmif_init;
177 1.29 pooka ifp->if_ioctl = shmif_ioctl;
178 1.29 pooka ifp->if_start = shmif_start;
179 1.29 pooka ifp->if_stop = shmif_stop;
180 1.29 pooka ifp->if_mtu = ETHERMTU;
181 1.37 pooka ifp->if_dlt = DLT_EN10MB;
182 1.29 pooka
183 1.32 pooka mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE);
184 1.32 pooka cv_init(&sc->sc_cv, "shmifcv");
185 1.32 pooka
186 1.29 pooka if_attach(ifp);
187 1.29 pooka ether_ifattach(ifp, enaddr);
188 1.29 pooka
189 1.29 pooka aprint_verbose("shmif%d: Ethernet address %s\n",
190 1.32 pooka unit, ether_sprintf(enaddr));
191 1.29 pooka
192 1.29 pooka if (scp)
193 1.29 pooka *scp = sc;
194 1.29 pooka
195 1.32 pooka error = 0;
196 1.32 pooka if (rump_threads) {
197 1.32 pooka error = kthread_create(PRI_NONE,
198 1.40 rmind KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL,
199 1.32 pooka shmif_rcv, ifp, &sc->sc_rcvl, "shmif");
200 1.32 pooka } else {
201 1.32 pooka printf("WARNING: threads not enabled, shmif NOT working\n");
202 1.32 pooka }
203 1.32 pooka
204 1.32 pooka if (error) {
205 1.32 pooka shmif_unclone(ifp);
206 1.32 pooka }
207 1.32 pooka
208 1.32 pooka return error;
209 1.29 pooka }
210 1.29 pooka
211 1.29 pooka static int
212 1.29 pooka initbackend(struct shmif_sc *sc, int memfd)
213 1.29 pooka {
214 1.29 pooka volatile uint8_t v;
215 1.29 pooka volatile uint8_t *p;
216 1.53 pooka void *mem;
217 1.29 pooka int error;
218 1.29 pooka
219 1.53 pooka error = rumpcomp_shmif_mmap(memfd, BUSMEM_SIZE, &mem);
220 1.1 pooka if (error)
221 1.29 pooka return error;
222 1.53 pooka sc->sc_busmem = mem;
223 1.17 pooka
224 1.29 pooka if (sc->sc_busmem->shm_magic
225 1.29 pooka && sc->sc_busmem->shm_magic != SHMIF_MAGIC) {
226 1.29 pooka printf("bus is not magical");
227 1.29 pooka rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
228 1.29 pooka return ENOEXEC;
229 1.29 pooka }
230 1.28 pooka
231 1.36 pooka /*
232 1.36 pooka * Prefault in pages to minimize runtime penalty with buslock.
233 1.36 pooka * Use 512 instead of PAGE_SIZE to make sure we catch cases where
234 1.36 pooka * rump kernel PAGE_SIZE > host page size.
235 1.36 pooka */
236 1.28 pooka for (p = (uint8_t *)sc->sc_busmem;
237 1.28 pooka p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
238 1.36 pooka p += 512)
239 1.28 pooka v = *p;
240 1.28 pooka
241 1.19 pooka shmif_lockbus(sc->sc_busmem);
242 1.19 pooka /* we're first? initialize bus */
243 1.19 pooka if (sc->sc_busmem->shm_magic == 0) {
244 1.19 pooka sc->sc_busmem->shm_magic = SHMIF_MAGIC;
245 1.19 pooka sc->sc_busmem->shm_first = BUSMEM_DATASIZE;
246 1.19 pooka }
247 1.19 pooka
248 1.16 pooka sc->sc_nextpacket = sc->sc_busmem->shm_last;
249 1.26 pooka sc->sc_devgen = sc->sc_busmem->shm_gen;
250 1.28 pooka
251 1.28 pooka #ifdef PREFAULT_RW
252 1.28 pooka for (p = (uint8_t *)sc->sc_busmem;
253 1.28 pooka p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE;
254 1.28 pooka p += PAGE_SIZE) {
255 1.28 pooka v = *p;
256 1.28 pooka *p = v;
257 1.28 pooka }
258 1.28 pooka #endif
259 1.19 pooka shmif_unlockbus(sc->sc_busmem);
260 1.1 pooka
261 1.53 pooka sc->sc_kq = -1;
262 1.53 pooka error = rumpcomp_shmif_watchsetup(&sc->sc_kq, memfd);
263 1.53 pooka if (error) {
264 1.32 pooka rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
265 1.29 pooka return error;
266 1.32 pooka }
267 1.1 pooka
268 1.29 pooka sc->sc_memfd = memfd;
269 1.32 pooka
270 1.32 pooka return error;
271 1.29 pooka }
272 1.29 pooka
273 1.29 pooka static void
274 1.29 pooka finibackend(struct shmif_sc *sc)
275 1.29 pooka {
276 1.29 pooka
277 1.32 pooka if (sc->sc_backfile == NULL)
278 1.32 pooka return;
279 1.32 pooka
280 1.32 pooka if (sc->sc_backfile) {
281 1.32 pooka kmem_free(sc->sc_backfile, sc->sc_backfilelen);
282 1.32 pooka sc->sc_backfile = NULL;
283 1.32 pooka sc->sc_backfilelen = 0;
284 1.32 pooka }
285 1.29 pooka
286 1.29 pooka rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE);
287 1.53 pooka rumpuser_close(sc->sc_memfd);
288 1.53 pooka rumpuser_close(sc->sc_kq);
289 1.32 pooka
290 1.32 pooka sc->sc_memfd = -1;
291 1.29 pooka }
292 1.29 pooka
293 1.29 pooka int
294 1.29 pooka rump_shmif_create(const char *path, int *ifnum)
295 1.29 pooka {
296 1.29 pooka struct shmif_sc *sc;
297 1.43 dyoung vmem_addr_t t;
298 1.33 pooka int unit, error;
299 1.33 pooka int memfd = -1; /* XXXgcc */
300 1.29 pooka
301 1.33 pooka if (path) {
302 1.53 pooka error = rumpuser_open(path,
303 1.53 pooka RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
304 1.53 pooka if (error)
305 1.33 pooka return error;
306 1.33 pooka }
307 1.1 pooka
308 1.43 dyoung error = vmem_xalloc(shmif_units, 1, 0, 0, 0,
309 1.43 dyoung VMEM_ADDR_MIN, VMEM_ADDR_MAX, VM_INSTANTFIT | VM_SLEEP, &t);
310 1.43 dyoung
311 1.43 dyoung if (error != 0) {
312 1.43 dyoung if (path)
313 1.53 pooka rumpuser_close(memfd);
314 1.43 dyoung return error;
315 1.43 dyoung }
316 1.43 dyoung
317 1.43 dyoung unit = t - 1;
318 1.31 pooka
319 1.32 pooka if ((error = allocif(unit, &sc)) != 0) {
320 1.33 pooka if (path)
321 1.53 pooka rumpuser_close(memfd);
322 1.29 pooka return error;
323 1.29 pooka }
324 1.33 pooka
325 1.33 pooka if (!path)
326 1.33 pooka goto out;
327 1.33 pooka
328 1.29 pooka error = initbackend(sc, memfd);
329 1.29 pooka if (error) {
330 1.32 pooka shmif_unclone(&sc->sc_ec.ec_if);
331 1.29 pooka return error;
332 1.29 pooka }
333 1.1 pooka
334 1.29 pooka sc->sc_backfilelen = strlen(path)+1;
335 1.29 pooka sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP);
336 1.29 pooka strcpy(sc->sc_backfile, path);
337 1.12 pooka
338 1.33 pooka out:
339 1.2 pooka if (ifnum)
340 1.32 pooka *ifnum = unit;
341 1.29 pooka
342 1.1 pooka return 0;
343 1.29 pooka }
344 1.1 pooka
345 1.29 pooka static int
346 1.29 pooka shmif_clone(struct if_clone *ifc, int unit)
347 1.29 pooka {
348 1.43 dyoung int rc;
349 1.43 dyoung vmem_addr_t unit2;
350 1.29 pooka
351 1.31 pooka /*
352 1.31 pooka * Ok, we know the unit number, but we must still reserve it.
353 1.31 pooka * Otherwise the wildcard-side of things might get the same one.
354 1.31 pooka * This is slightly offset-happy due to vmem. First, we offset
355 1.31 pooka * the range of unit numbers by +1 since vmem cannot deal with
356 1.41 dyoung * ranges starting from 0. Talk about uuuh.
357 1.31 pooka */
358 1.43 dyoung rc = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+1,
359 1.43 dyoung VM_SLEEP | VM_INSTANTFIT, &unit2);
360 1.43 dyoung KASSERT(rc == 0 && unit2-1 == unit);
361 1.29 pooka
362 1.29 pooka return allocif(unit, NULL);
363 1.29 pooka }
364 1.29 pooka
365 1.29 pooka static int
366 1.29 pooka shmif_unclone(struct ifnet *ifp)
367 1.29 pooka {
368 1.32 pooka struct shmif_sc *sc = ifp->if_softc;
369 1.32 pooka
370 1.32 pooka shmif_stop(ifp, 1);
371 1.32 pooka if_down(ifp);
372 1.32 pooka finibackend(sc);
373 1.32 pooka
374 1.32 pooka mutex_enter(&sc->sc_mtx);
375 1.32 pooka sc->sc_dying = true;
376 1.32 pooka cv_broadcast(&sc->sc_cv);
377 1.32 pooka mutex_exit(&sc->sc_mtx);
378 1.29 pooka
379 1.32 pooka if (sc->sc_rcvl)
380 1.32 pooka kthread_join(sc->sc_rcvl);
381 1.32 pooka sc->sc_rcvl = NULL;
382 1.32 pooka
383 1.32 pooka vmem_xfree(shmif_units, sc->sc_unit+1, 1);
384 1.32 pooka
385 1.32 pooka ether_ifdetach(ifp);
386 1.32 pooka if_detach(ifp);
387 1.32 pooka
388 1.32 pooka cv_destroy(&sc->sc_cv);
389 1.32 pooka mutex_destroy(&sc->sc_mtx);
390 1.32 pooka
391 1.32 pooka kmem_free(sc, sizeof(*sc));
392 1.32 pooka
393 1.32 pooka return 0;
394 1.1 pooka }
395 1.1 pooka
396 1.1 pooka static int
397 1.1 pooka shmif_init(struct ifnet *ifp)
398 1.1 pooka {
399 1.29 pooka struct shmif_sc *sc = ifp->if_softc;
400 1.4 pooka int error = 0;
401 1.4 pooka
402 1.29 pooka if (sc->sc_memfd == -1)
403 1.29 pooka return ENXIO;
404 1.32 pooka KASSERT(sc->sc_busmem);
405 1.29 pooka
406 1.32 pooka ifp->if_flags |= IFF_RUNNING;
407 1.32 pooka
408 1.32 pooka mutex_enter(&sc->sc_mtx);
409 1.32 pooka sc->sc_nextpacket = sc->sc_busmem->shm_last;
410 1.32 pooka sc->sc_devgen = sc->sc_busmem->shm_gen;
411 1.32 pooka
412 1.32 pooka cv_broadcast(&sc->sc_cv);
413 1.32 pooka mutex_exit(&sc->sc_mtx);
414 1.1 pooka
415 1.4 pooka return error;
416 1.1 pooka }
417 1.1 pooka
418 1.1 pooka static int
419 1.1 pooka shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
420 1.1 pooka {
421 1.29 pooka struct shmif_sc *sc = ifp->if_softc;
422 1.29 pooka struct ifdrv *ifd;
423 1.29 pooka char *path;
424 1.32 pooka int s, rv, memfd;
425 1.1 pooka
426 1.1 pooka s = splnet();
427 1.29 pooka switch (cmd) {
428 1.29 pooka case SIOCGLINKSTR:
429 1.29 pooka ifd = data;
430 1.29 pooka
431 1.29 pooka if (sc->sc_backfilelen == 0) {
432 1.29 pooka rv = ENOENT;
433 1.29 pooka break;
434 1.29 pooka }
435 1.29 pooka
436 1.29 pooka ifd->ifd_len = sc->sc_backfilelen;
437 1.29 pooka if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) {
438 1.29 pooka rv = 0;
439 1.29 pooka break;
440 1.29 pooka }
441 1.29 pooka
442 1.29 pooka if (ifd->ifd_cmd != 0) {
443 1.29 pooka rv = EINVAL;
444 1.29 pooka break;
445 1.29 pooka }
446 1.29 pooka
447 1.29 pooka rv = copyoutstr(sc->sc_backfile, ifd->ifd_data,
448 1.29 pooka MIN(sc->sc_backfilelen, ifd->ifd_len), NULL);
449 1.29 pooka break;
450 1.29 pooka case SIOCSLINKSTR:
451 1.29 pooka if (ifp->if_flags & IFF_UP) {
452 1.29 pooka rv = EBUSY;
453 1.29 pooka break;
454 1.29 pooka }
455 1.29 pooka
456 1.29 pooka ifd = data;
457 1.29 pooka if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
458 1.29 pooka finibackend(sc);
459 1.29 pooka rv = 0;
460 1.29 pooka break;
461 1.29 pooka } else if (ifd->ifd_cmd != 0) {
462 1.29 pooka rv = EINVAL;
463 1.29 pooka break;
464 1.29 pooka } else if (sc->sc_backfile) {
465 1.29 pooka rv = EBUSY;
466 1.29 pooka break;
467 1.29 pooka }
468 1.29 pooka
469 1.29 pooka if (ifd->ifd_len > MAXPATHLEN) {
470 1.29 pooka rv = E2BIG;
471 1.29 pooka break;
472 1.29 pooka } else if (ifd->ifd_len < 1) {
473 1.29 pooka rv = EINVAL;
474 1.29 pooka break;
475 1.29 pooka }
476 1.29 pooka
477 1.29 pooka path = kmem_alloc(ifd->ifd_len, KM_SLEEP);
478 1.29 pooka rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL);
479 1.29 pooka if (rv) {
480 1.29 pooka kmem_free(path, ifd->ifd_len);
481 1.29 pooka break;
482 1.29 pooka }
483 1.53 pooka rv = rumpuser_open(path,
484 1.53 pooka RUMPUSER_OPEN_RDWR | RUMPUSER_OPEN_CREATE, &memfd);
485 1.53 pooka if (rv) {
486 1.29 pooka kmem_free(path, ifd->ifd_len);
487 1.29 pooka break;
488 1.29 pooka }
489 1.29 pooka rv = initbackend(sc, memfd);
490 1.29 pooka if (rv) {
491 1.29 pooka kmem_free(path, ifd->ifd_len);
492 1.53 pooka rumpuser_close(memfd);
493 1.29 pooka break;
494 1.29 pooka }
495 1.29 pooka sc->sc_backfile = path;
496 1.29 pooka sc->sc_backfilelen = ifd->ifd_len;
497 1.29 pooka
498 1.29 pooka break;
499 1.29 pooka default:
500 1.29 pooka rv = ether_ioctl(ifp, cmd, data);
501 1.29 pooka if (rv == ENETRESET)
502 1.29 pooka rv = 0;
503 1.29 pooka break;
504 1.29 pooka }
505 1.1 pooka splx(s);
506 1.1 pooka
507 1.1 pooka return rv;
508 1.1 pooka }
509 1.1 pooka
510 1.32 pooka /* send everything in-context since it's just a matter of mem-to-mem copy */
511 1.1 pooka static void
512 1.1 pooka shmif_start(struct ifnet *ifp)
513 1.1 pooka {
514 1.1 pooka struct shmif_sc *sc = ifp->if_softc;
515 1.26 pooka struct shmif_mem *busmem = sc->sc_busmem;
516 1.1 pooka struct mbuf *m, *m0;
517 1.26 pooka uint32_t dataoff;
518 1.26 pooka uint32_t pktsize, pktwrote;
519 1.1 pooka bool wrote = false;
520 1.24 pooka bool wrap;
521 1.1 pooka
522 1.26 pooka ifp->if_flags |= IFF_OACTIVE;
523 1.26 pooka
524 1.1 pooka for (;;) {
525 1.20 pooka struct shmif_pkthdr sp;
526 1.20 pooka struct timeval tv;
527 1.20 pooka
528 1.1 pooka IF_DEQUEUE(&ifp->if_snd, m0);
529 1.1 pooka if (m0 == NULL) {
530 1.1 pooka break;
531 1.1 pooka }
532 1.1 pooka
533 1.25 pooka pktsize = 0;
534 1.19 pooka for (m = m0; m != NULL; m = m->m_next) {
535 1.19 pooka pktsize += m->m_len;
536 1.19 pooka }
537 1.26 pooka KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN);
538 1.19 pooka
539 1.20 pooka getmicrouptime(&tv);
540 1.20 pooka sp.sp_len = pktsize;
541 1.20 pooka sp.sp_sec = tv.tv_sec;
542 1.20 pooka sp.sp_usec = tv.tv_usec;
543 1.20 pooka
544 1.35 pooka bpf_mtap(ifp, m0);
545 1.35 pooka
546 1.26 pooka shmif_lockbus(busmem);
547 1.26 pooka KASSERT(busmem->shm_magic == SHMIF_MAGIC);
548 1.26 pooka busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last);
549 1.21 pooka
550 1.24 pooka wrap = false;
551 1.26 pooka dataoff = shmif_buswrite(busmem,
552 1.26 pooka busmem->shm_last, &sp, sizeof(sp), &wrap);
553 1.26 pooka pktwrote = 0;
554 1.1 pooka for (m = m0; m != NULL; m = m->m_next) {
555 1.26 pooka pktwrote += m->m_len;
556 1.26 pooka dataoff = shmif_buswrite(busmem, dataoff,
557 1.19 pooka mtod(m, void *), m->m_len, &wrap);
558 1.1 pooka }
559 1.26 pooka KASSERT(pktwrote == pktsize);
560 1.27 pooka if (wrap) {
561 1.26 pooka busmem->shm_gen++;
562 1.47 pooka DPRINTF(("bus generation now %" PRIu64 "\n",
563 1.47 pooka busmem->shm_gen));
564 1.27 pooka }
565 1.26 pooka shmif_unlockbus(busmem);
566 1.1 pooka
567 1.1 pooka m_freem(m0);
568 1.1 pooka wrote = true;
569 1.1 pooka
570 1.1 pooka DPRINTF(("shmif_start: send %d bytes at off %d\n",
571 1.27 pooka pktsize, busmem->shm_last));
572 1.1 pooka }
573 1.26 pooka
574 1.26 pooka ifp->if_flags &= ~IFF_OACTIVE;
575 1.26 pooka
576 1.32 pooka /* wakeup? */
577 1.52 pooka if (wrote) {
578 1.52 pooka dowakeup(sc);
579 1.52 pooka }
580 1.1 pooka }
581 1.1 pooka
582 1.1 pooka static void
583 1.1 pooka shmif_stop(struct ifnet *ifp, int disable)
584 1.1 pooka {
585 1.32 pooka struct shmif_sc *sc = ifp->if_softc;
586 1.1 pooka
587 1.32 pooka ifp->if_flags &= ~IFF_RUNNING;
588 1.32 pooka membar_producer();
589 1.32 pooka
590 1.32 pooka /*
591 1.32 pooka * wakeup thread. this will of course wake up all bus
592 1.32 pooka * listeners, but that's life.
593 1.32 pooka */
594 1.52 pooka if (sc->sc_memfd != -1) {
595 1.52 pooka dowakeup(sc);
596 1.52 pooka }
597 1.1 pooka }
598 1.1 pooka
599 1.27 pooka
600 1.27 pooka /*
601 1.27 pooka * Check if we have been sleeping too long. Basically,
602 1.27 pooka * our in-sc nextpkt must by first <= nextpkt <= last"+1".
603 1.27 pooka * We use the fact that first is guaranteed to never overlap
604 1.27 pooka * with the last frame in the ring.
605 1.27 pooka */
606 1.27 pooka static __inline bool
607 1.27 pooka stillvalid_p(struct shmif_sc *sc)
608 1.27 pooka {
609 1.27 pooka struct shmif_mem *busmem = sc->sc_busmem;
610 1.27 pooka unsigned gendiff = busmem->shm_gen - sc->sc_devgen;
611 1.27 pooka uint32_t lastoff, devoff;
612 1.27 pooka
613 1.27 pooka KASSERT(busmem->shm_first != busmem->shm_last);
614 1.27 pooka
615 1.27 pooka /* normalize onto a 2x busmem chunk */
616 1.27 pooka devoff = sc->sc_nextpacket;
617 1.27 pooka lastoff = shmif_nextpktoff(busmem, busmem->shm_last);
618 1.27 pooka
619 1.27 pooka /* trivial case */
620 1.27 pooka if (gendiff > 1)
621 1.27 pooka return false;
622 1.27 pooka KASSERT(gendiff <= 1);
623 1.27 pooka
624 1.27 pooka /* Normalize onto 2x busmem chunk */
625 1.27 pooka if (busmem->shm_first >= lastoff) {
626 1.27 pooka lastoff += BUSMEM_DATASIZE;
627 1.27 pooka if (gendiff == 0)
628 1.27 pooka devoff += BUSMEM_DATASIZE;
629 1.27 pooka } else {
630 1.27 pooka if (gendiff)
631 1.27 pooka return false;
632 1.27 pooka }
633 1.27 pooka
634 1.27 pooka return devoff >= busmem->shm_first && devoff <= lastoff;
635 1.27 pooka }
636 1.27 pooka
637 1.1 pooka static void
638 1.1 pooka shmif_rcv(void *arg)
639 1.1 pooka {
640 1.1 pooka struct ifnet *ifp = arg;
641 1.1 pooka struct shmif_sc *sc = ifp->if_softc;
642 1.32 pooka struct shmif_mem *busmem;
643 1.1 pooka struct mbuf *m = NULL;
644 1.1 pooka struct ether_header *eth;
645 1.27 pooka uint32_t nextpkt;
646 1.37 pooka bool wrap, passup;
647 1.1 pooka int error;
648 1.57 pooka const int align
649 1.57 pooka = ALIGN(sizeof(struct ether_header)) - sizeof(struct ether_header);
650 1.1 pooka
651 1.32 pooka reup:
652 1.32 pooka mutex_enter(&sc->sc_mtx);
653 1.32 pooka while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying)
654 1.32 pooka cv_wait(&sc->sc_cv, &sc->sc_mtx);
655 1.32 pooka mutex_exit(&sc->sc_mtx);
656 1.32 pooka
657 1.32 pooka busmem = sc->sc_busmem;
658 1.32 pooka
659 1.32 pooka while (ifp->if_flags & IFF_RUNNING) {
660 1.20 pooka struct shmif_pkthdr sp;
661 1.20 pooka
662 1.1 pooka if (m == NULL) {
663 1.1 pooka m = m_gethdr(M_WAIT, MT_DATA);
664 1.1 pooka MCLGET(m, M_WAIT);
665 1.57 pooka m->m_data += align;
666 1.1 pooka }
667 1.1 pooka
668 1.47 pooka DPRINTF(("waiting %d/%" PRIu64 "\n",
669 1.47 pooka sc->sc_nextpacket, sc->sc_devgen));
670 1.26 pooka KASSERT(m->m_flags & M_EXT);
671 1.1 pooka
672 1.26 pooka shmif_lockbus(busmem);
673 1.26 pooka KASSERT(busmem->shm_magic == SHMIF_MAGIC);
674 1.27 pooka KASSERT(busmem->shm_gen >= sc->sc_devgen);
675 1.1 pooka
676 1.1 pooka /* need more data? */
677 1.27 pooka if (sc->sc_devgen == busmem->shm_gen &&
678 1.26 pooka shmif_nextpktoff(busmem, busmem->shm_last)
679 1.26 pooka == sc->sc_nextpacket) {
680 1.26 pooka shmif_unlockbus(busmem);
681 1.1 pooka error = 0;
682 1.53 pooka rumpcomp_shmif_watchwait(sc->sc_kq);
683 1.1 pooka if (__predict_false(error))
684 1.1 pooka printf("shmif_rcv: wait failed %d\n", error);
685 1.32 pooka membar_consumer();
686 1.1 pooka continue;
687 1.1 pooka }
688 1.1 pooka
689 1.27 pooka if (stillvalid_p(sc)) {
690 1.27 pooka nextpkt = sc->sc_nextpacket;
691 1.27 pooka } else {
692 1.27 pooka KASSERT(busmem->shm_gen > 0);
693 1.26 pooka nextpkt = busmem->shm_first;
694 1.26 pooka if (busmem->shm_first > busmem->shm_last)
695 1.27 pooka sc->sc_devgen = busmem->shm_gen - 1;
696 1.26 pooka else
697 1.27 pooka sc->sc_devgen = busmem->shm_gen;
698 1.47 pooka DPRINTF(("dev %p overrun, new data: %d/%" PRIu64 "\n",
699 1.27 pooka sc, nextpkt, sc->sc_devgen));
700 1.26 pooka }
701 1.26 pooka
702 1.26 pooka /*
703 1.26 pooka * If our read pointer is ahead the bus last write, our
704 1.26 pooka * generation must be one behind.
705 1.26 pooka */
706 1.26 pooka KASSERT(!(nextpkt > busmem->shm_last
707 1.27 pooka && sc->sc_devgen == busmem->shm_gen));
708 1.26 pooka
709 1.24 pooka wrap = false;
710 1.26 pooka nextpkt = shmif_busread(busmem, &sp,
711 1.26 pooka nextpkt, sizeof(sp), &wrap);
712 1.26 pooka KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN);
713 1.26 pooka nextpkt = shmif_busread(busmem, mtod(m, void *),
714 1.26 pooka nextpkt, sp.sp_len, &wrap);
715 1.1 pooka
716 1.1 pooka DPRINTF(("shmif_rcv: read packet of length %d at %d\n",
717 1.20 pooka sp.sp_len, nextpkt));
718 1.1 pooka
719 1.26 pooka sc->sc_nextpacket = nextpkt;
720 1.19 pooka shmif_unlockbus(sc->sc_busmem);
721 1.1 pooka
722 1.27 pooka if (wrap) {
723 1.26 pooka sc->sc_devgen++;
724 1.47 pooka DPRINTF(("dev %p generation now %" PRIu64 "\n",
725 1.27 pooka sc, sc->sc_devgen));
726 1.27 pooka }
727 1.26 pooka
728 1.56 pooka /*
729 1.56 pooka * Ignore packets too short to possibly be valid.
730 1.56 pooka * This is hit at least for the first frame on a new bus.
731 1.56 pooka */
732 1.55 pooka if (__predict_false(sp.sp_len < ETHER_HDR_LEN)) {
733 1.55 pooka DPRINTF(("shmif read packet len %d < ETHER_HDR_LEN\n",
734 1.55 pooka sp.sp_len));
735 1.55 pooka continue;
736 1.55 pooka }
737 1.55 pooka
738 1.20 pooka m->m_len = m->m_pkthdr.len = sp.sp_len;
739 1.1 pooka m->m_pkthdr.rcvif = ifp;
740 1.1 pooka
741 1.37 pooka /*
742 1.37 pooka * Test if we want to pass the packet upwards
743 1.37 pooka */
744 1.1 pooka eth = mtod(m, struct ether_header *);
745 1.37 pooka if (memcmp(eth->ether_dhost, CLLADDR(ifp->if_sadl),
746 1.37 pooka ETHER_ADDR_LEN) == 0) {
747 1.37 pooka passup = true;
748 1.46 pooka } else if (ETHER_IS_MULTICAST(eth->ether_dhost)) {
749 1.37 pooka passup = true;
750 1.37 pooka } else if (ifp->if_flags & IFF_PROMISC) {
751 1.37 pooka m->m_flags |= M_PROMISC;
752 1.37 pooka passup = true;
753 1.38 pooka } else {
754 1.38 pooka passup = false;
755 1.37 pooka }
756 1.37 pooka
757 1.37 pooka if (passup) {
758 1.22 pooka KERNEL_LOCK(1, NULL);
759 1.35 pooka bpf_mtap(ifp, m);
760 1.1 pooka ifp->if_input(ifp, m);
761 1.22 pooka KERNEL_UNLOCK_ONE(NULL);
762 1.1 pooka m = NULL;
763 1.1 pooka }
764 1.37 pooka /* else: reuse mbuf for a future packet */
765 1.1 pooka }
766 1.32 pooka m_freem(m);
767 1.32 pooka m = NULL;
768 1.32 pooka
769 1.32 pooka if (!sc->sc_dying)
770 1.32 pooka goto reup;
771 1.1 pooka
772 1.32 pooka kthread_exit(0);
773 1.1 pooka }
774