if_vioif.c revision 1.86 1 /* $NetBSD: if_vioif.c,v 1.86 2023/03/23 01:33:20 yamaguchi Exp $ */
2
3 /*
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * Copyright (c) 2010 Minoura Makoto.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.86 2023/03/23 01:33:20 yamaguchi Exp $");
31
32 #ifdef _KERNEL_OPT
33 #include "opt_net_mpsafe.h"
34 #endif
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/atomic.h>
40 #include <sys/bus.h>
41 #include <sys/condvar.h>
42 #include <sys/device.h>
43 #include <sys/evcnt.h>
44 #include <sys/intr.h>
45 #include <sys/kmem.h>
46 #include <sys/mbuf.h>
47 #include <sys/mutex.h>
48 #include <sys/sockio.h>
49 #include <sys/syslog.h>
50 #include <sys/cpu.h>
51 #include <sys/module.h>
52 #include <sys/pcq.h>
53 #include <sys/workqueue.h>
54 #include <sys/xcall.h>
55
56 #include <dev/pci/virtioreg.h>
57 #include <dev/pci/virtiovar.h>
58
59 #include <net/if.h>
60 #include <net/if_dl.h>
61 #include <net/if_media.h>
62 #include <net/if_ether.h>
63
64 #include <net/bpf.h>
65
66 #include "ioconf.h"
67
68 #ifdef NET_MPSAFE
69 #define VIOIF_MPSAFE 1
70 #define VIOIF_MULTIQ 1
71 #endif
72
73 /*
74 * if_vioifreg.h:
75 */
76 /* Configuration registers */
77 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
78 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
79 #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
80 #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */
81
82 /* Feature bits */
83 #define VIRTIO_NET_F_CSUM __BIT(0)
84 #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
85 #define VIRTIO_NET_F_MAC __BIT(5)
86 #define VIRTIO_NET_F_GSO __BIT(6)
87 #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
88 #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
89 #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
90 #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
91 #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
92 #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
93 #define VIRTIO_NET_F_HOST_ECN __BIT(13)
94 #define VIRTIO_NET_F_HOST_UFO __BIT(14)
95 #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
96 #define VIRTIO_NET_F_STATUS __BIT(16)
97 #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
98 #define VIRTIO_NET_F_CTRL_RX __BIT(18)
99 #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
100 #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
101 #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
102 #define VIRTIO_NET_F_MQ __BIT(22)
103 #define VIRTIO_NET_F_CTRL_MAC_ADDR __BIT(23)
104
105 #define VIRTIO_NET_FLAG_BITS \
106 VIRTIO_COMMON_FLAG_BITS \
107 "b\x17" "CTRL_MAC\0" \
108 "b\x16" "MQ\0" \
109 "b\x15" "GUEST_ANNOUNCE\0" \
110 "b\x14" "CTRL_RX_EXTRA\0" \
111 "b\x13" "CTRL_VLAN\0" \
112 "b\x12" "CTRL_RX\0" \
113 "b\x11" "CTRL_VQ\0" \
114 "b\x10" "STATUS\0" \
115 "b\x0f" "MRG_RXBUF\0" \
116 "b\x0e" "HOST_UFO\0" \
117 "b\x0d" "HOST_ECN\0" \
118 "b\x0c" "HOST_TSO6\0" \
119 "b\x0b" "HOST_TSO4\0" \
120 "b\x0a" "GUEST_UFO\0" \
121 "b\x09" "GUEST_ECN\0" \
122 "b\x08" "GUEST_TSO6\0" \
123 "b\x07" "GUEST_TSO4\0" \
124 "b\x06" "GSO\0" \
125 "b\x05" "MAC\0" \
126 "b\x01" "GUEST_CSUM\0" \
127 "b\x00" "CSUM\0"
128
129 /* Status */
130 #define VIRTIO_NET_S_LINK_UP 1
131
132 /* Packet header structure */
133 struct virtio_net_hdr {
134 uint8_t flags;
135 uint8_t gso_type;
136 uint16_t hdr_len;
137 uint16_t gso_size;
138 uint16_t csum_start;
139 uint16_t csum_offset;
140
141 uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
142 } __packed;
143
144 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
145 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
146 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
147 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
148 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
149 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
150
151 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
152
153 /* Control virtqueue */
154 struct virtio_net_ctrl_cmd {
155 uint8_t class;
156 uint8_t command;
157 } __packed;
158 #define VIRTIO_NET_CTRL_RX 0
159 # define VIRTIO_NET_CTRL_RX_PROMISC 0
160 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
161
162 #define VIRTIO_NET_CTRL_MAC 1
163 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
164 # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
165
166 #define VIRTIO_NET_CTRL_VLAN 2
167 # define VIRTIO_NET_CTRL_VLAN_ADD 0
168 # define VIRTIO_NET_CTRL_VLAN_DEL 1
169
170 #define VIRTIO_NET_CTRL_MQ 4
171 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
172 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
173 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
174
175 struct virtio_net_ctrl_status {
176 uint8_t ack;
177 } __packed;
178 #define VIRTIO_NET_OK 0
179 #define VIRTIO_NET_ERR 1
180
181 struct virtio_net_ctrl_rx {
182 uint8_t onoff;
183 } __packed;
184
185 struct virtio_net_ctrl_mac_tbl {
186 uint32_t nentries;
187 uint8_t macs[][ETHER_ADDR_LEN];
188 } __packed;
189
190 struct virtio_net_ctrl_mac_addr {
191 uint8_t mac[ETHER_ADDR_LEN];
192 } __packed;
193
194 struct virtio_net_ctrl_vlan {
195 uint16_t id;
196 } __packed;
197
198 struct virtio_net_ctrl_mq {
199 uint16_t virtqueue_pairs;
200 } __packed;
201
202 /*
203 * if_vioifvar.h:
204 */
205
206 /*
207 * Locking notes:
208 * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
209 * a field in vioif_rxqueue is protected by rxq_lock (a spin mutex).
210 * - more than one lock cannot be held at onece
211 * + ctrlq_inuse is protected by ctrlq_wait_lock.
212 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
213 * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
214 * + fields in vioif_softc except queues are protected by
215 * sc->sc_lock(an adaptive mutex)
216 * - the lock is held before acquisition of other locks
217 */
218
219 struct vioif_ctrl_cmdspec {
220 bus_dmamap_t dmamap;
221 void *buf;
222 bus_size_t bufsize;
223 };
224
225 struct vioif_work {
226 struct work cookie;
227 void (*func)(void *);
228 void *arg;
229 unsigned int added;
230 };
231
232 struct vioif_txqueue {
233 kmutex_t *txq_lock; /* lock for tx operations */
234
235 struct virtqueue *txq_vq;
236 bool txq_stopping;
237 bool txq_link_active;
238 pcq_t *txq_intrq;
239
240 struct virtio_net_hdr *txq_hdrs;
241 bus_dmamap_t *txq_hdr_dmamaps;
242
243 struct mbuf **txq_mbufs;
244 bus_dmamap_t *txq_dmamaps;
245
246 void *txq_deferred_transmit;
247 void *txq_handle_si;
248 struct vioif_work txq_work;
249 bool txq_workqueue;
250 bool txq_running_handle;
251
252 char txq_evgroup[16];
253 struct evcnt txq_defrag_failed;
254 struct evcnt txq_mbuf_load_failed;
255 struct evcnt txq_enqueue_reserve_failed;
256 };
257
258 struct vioif_rxqueue {
259 kmutex_t *rxq_lock; /* lock for rx operations */
260
261 struct virtqueue *rxq_vq;
262 bool rxq_stopping;
263
264 struct virtio_net_hdr *rxq_hdrs;
265 bus_dmamap_t *rxq_hdr_dmamaps;
266
267 struct mbuf **rxq_mbufs;
268 bus_dmamap_t *rxq_dmamaps;
269
270 void *rxq_handle_si;
271 struct vioif_work rxq_work;
272 bool rxq_workqueue;
273 bool rxq_running_handle;
274
275 char rxq_evgroup[16];
276 struct evcnt rxq_mbuf_add_failed;
277 };
278
279 struct vioif_ctrlqueue {
280 struct virtqueue *ctrlq_vq;
281 enum {
282 FREE, INUSE, DONE
283 } ctrlq_inuse;
284 kcondvar_t ctrlq_wait;
285 kmutex_t ctrlq_wait_lock;
286 struct lwp *ctrlq_owner;
287
288 struct virtio_net_ctrl_cmd *ctrlq_cmd;
289 struct virtio_net_ctrl_status *ctrlq_status;
290 struct virtio_net_ctrl_rx *ctrlq_rx;
291 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
292 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
293 struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr;
294 struct virtio_net_ctrl_mq *ctrlq_mq;
295
296 bus_dmamap_t ctrlq_cmd_dmamap;
297 bus_dmamap_t ctrlq_status_dmamap;
298 bus_dmamap_t ctrlq_rx_dmamap;
299 bus_dmamap_t ctrlq_tbl_uc_dmamap;
300 bus_dmamap_t ctrlq_tbl_mc_dmamap;
301 bus_dmamap_t ctrlq_mac_addr_dmamap;
302 bus_dmamap_t ctrlq_mq_dmamap;
303
304 struct evcnt ctrlq_cmd_load_failed;
305 struct evcnt ctrlq_cmd_failed;
306 };
307
308 struct vioif_softc {
309 device_t sc_dev;
310 kmutex_t sc_lock;
311 struct sysctllog *sc_sysctllog;
312
313 struct virtio_softc *sc_virtio;
314 struct virtqueue *sc_vqs;
315 u_int sc_hdr_size;
316
317 int sc_max_nvq_pairs;
318 int sc_req_nvq_pairs;
319 int sc_act_nvq_pairs;
320
321 uint8_t sc_mac[ETHER_ADDR_LEN];
322 struct ethercom sc_ethercom;
323 int sc_link_state;
324
325 struct vioif_txqueue *sc_txq;
326 struct vioif_rxqueue *sc_rxq;
327
328 bool sc_has_ctrl;
329 struct vioif_ctrlqueue sc_ctrlq;
330
331 bus_dma_segment_t sc_hdr_segs[1];
332 void *sc_dmamem;
333 void *sc_kmem;
334
335 void *sc_ctl_softint;
336
337 struct workqueue *sc_txrx_workqueue;
338 bool sc_txrx_workqueue_sysctl;
339 u_int sc_tx_intr_process_limit;
340 u_int sc_tx_process_limit;
341 u_int sc_rx_intr_process_limit;
342 u_int sc_rx_process_limit;
343 };
344 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
345 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
346
347 #define VIOIF_TX_INTR_PROCESS_LIMIT 256
348 #define VIOIF_TX_PROCESS_LIMIT 256
349 #define VIOIF_RX_INTR_PROCESS_LIMIT 0U
350 #define VIOIF_RX_PROCESS_LIMIT 256
351
352 #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET
353 #define VIOIF_IS_LINK_ACTIVE(_sc) ((_sc)->sc_link_state == LINK_STATE_UP ? \
354 true : false)
355
356 /* cfattach interface functions */
357 static int vioif_match(device_t, cfdata_t, void *);
358 static void vioif_attach(device_t, device_t, void *);
359 static int vioif_finalize_teardown(device_t);
360
361 /* ifnet interface functions */
362 static int vioif_init(struct ifnet *);
363 static void vioif_stop(struct ifnet *, int);
364 static void vioif_start(struct ifnet *);
365 static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
366 static int vioif_transmit(struct ifnet *, struct mbuf *);
367 static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
368 static int vioif_ioctl(struct ifnet *, u_long, void *);
369 static void vioif_watchdog(struct ifnet *);
370 static int vioif_ifflags_cb(struct ethercom *);
371
372 /* rx */
373 static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
374 static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
375 static void vioif_populate_rx_mbufs_locked(struct vioif_softc *,
376 struct vioif_rxqueue *);
377 static void vioif_rx_queue_clear(struct vioif_rxqueue *);
378 static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
379 struct vioif_rxqueue *, u_int);
380 static int vioif_rx_intr(void *);
381 static void vioif_rx_handle(void *);
382 static void vioif_rx_sched_handle(struct vioif_softc *,
383 struct vioif_rxqueue *);
384 static void vioif_rx_drain(struct vioif_rxqueue *);
385
386 /* tx */
387 static int vioif_tx_intr(void *);
388 static void vioif_tx_handle(void *);
389 static void vioif_tx_sched_handle(struct vioif_softc *,
390 struct vioif_txqueue *);
391 static void vioif_tx_queue_clear(struct vioif_txqueue *);
392 static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
393 struct vioif_txqueue *, u_int);
394 static void vioif_tx_drain(struct vioif_txqueue *);
395 static void vioif_deferred_transmit(void *);
396
397 /* workqueue */
398 static struct workqueue*
399 vioif_workq_create(const char *, pri_t, int, int);
400 static void vioif_workq_destroy(struct workqueue *);
401 static void vioif_workq_work(struct work *, void *);
402 static void vioif_work_set(struct vioif_work *, void(*)(void *), void *);
403 static void vioif_work_add(struct workqueue *, struct vioif_work *);
404 static void vioif_work_wait(struct workqueue *, struct vioif_work *);
405
406 /* other control */
407 static int vioif_get_link_status(struct vioif_softc *);
408 static void vioif_update_link_status(struct vioif_softc *);
409 static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
410 static int vioif_set_promisc(struct vioif_softc *, bool);
411 static int vioif_set_allmulti(struct vioif_softc *, bool);
412 static int vioif_set_rx_filter(struct vioif_softc *);
413 static int vioif_rx_filter(struct vioif_softc *);
414 static int vioif_set_mac_addr(struct vioif_softc *);
415 static int vioif_ctrl_intr(void *);
416 static int vioif_config_change(struct virtio_softc *);
417 static void vioif_ctl_softint(void *);
418 static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
419 static void vioif_enable_interrupt_vqpairs(struct vioif_softc *);
420 static void vioif_disable_interrupt_vqpairs(struct vioif_softc *);
421 static int vioif_setup_sysctl(struct vioif_softc *);
422 static void vioif_setup_stats(struct vioif_softc *);
423 static int vioif_ifflags(struct vioif_softc *);
424 static void vioif_intr_barrier(void);
425
426 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
427 vioif_match, vioif_attach, NULL, NULL);
428
429 static int
430 vioif_match(device_t parent, cfdata_t match, void *aux)
431 {
432 struct virtio_attach_args *va = aux;
433
434 if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
435 return 1;
436
437 return 0;
438 }
439
440 static int
441 vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
442 bus_size_t size, int nsegs, const char *usage)
443 {
444 int r;
445
446 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
447 nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
448
449 if (r != 0) {
450 aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
451 "error code %d\n", usage, r);
452 }
453
454 return r;
455 }
456
457 static void
458 vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
459 {
460
461 if (*map) {
462 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
463 *map = NULL;
464 }
465 }
466
467 static int
468 vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
469 void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
470 {
471 int r;
472
473 r = vioif_dmamap_create(sc, map, size, nsegs, usage);
474 if (r != 0)
475 return 1;
476
477 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
478 size, NULL, rw | BUS_DMA_NOWAIT);
479 if (r != 0) {
480 vioif_dmamap_destroy(sc, map);
481 aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
482 "error code %d\n", usage, r);
483 }
484
485 return r;
486 }
487
488 static void *
489 vioif_assign_mem(intptr_t *p, size_t size)
490 {
491 intptr_t rv;
492
493 rv = *p;
494 *p += size;
495
496 return (void *)rv;
497 }
498
499 static void
500 vioif_alloc_queues(struct vioif_softc *sc)
501 {
502 int nvq_pairs = sc->sc_max_nvq_pairs;
503 int nvqs = nvq_pairs * 2;
504 int i;
505
506 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
507
508 sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
509 KM_SLEEP);
510 sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
511 KM_SLEEP);
512
513 if (sc->sc_has_ctrl)
514 nvqs++;
515
516 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
517 nvqs = 0;
518 for (i = 0; i < nvq_pairs; i++) {
519 sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
520 sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
521 }
522
523 if (sc->sc_has_ctrl)
524 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
525 }
526
527 static void
528 vioif_free_queues(struct vioif_softc *sc)
529 {
530 int nvq_pairs = sc->sc_max_nvq_pairs;
531 int nvqs = nvq_pairs * 2;
532
533 if (sc->sc_ctrlq.ctrlq_vq)
534 nvqs++;
535
536 if (sc->sc_txq) {
537 kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
538 sc->sc_txq = NULL;
539 }
540
541 if (sc->sc_rxq) {
542 kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
543 sc->sc_rxq = NULL;
544 }
545
546 if (sc->sc_vqs) {
547 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
548 sc->sc_vqs = NULL;
549 }
550 }
551
552 /* allocate memory */
553 /*
554 * dma memory is used for:
555 * rxq_hdrs[slot]: metadata array for received frames (READ)
556 * txq_hdrs[slot]: metadata array for frames to be sent (WRITE)
557 * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
558 * ctrlq_status: return value for a command via ctrl vq (READ)
559 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
560 * (WRITE)
561 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
562 * class command (WRITE)
563 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
564 * class command (WRITE)
565 * ctrlq_* structures are allocated only one each; they are protected by
566 * ctrlq_inuse variable and ctrlq_wait condvar.
567 */
568 /*
569 * dynamically allocated memory is used for:
570 * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot]
571 * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot]
572 * rxq_dmamaps[slot]: bus_dmamap_t array for received payload
573 * txq_dmamaps[slot]: bus_dmamap_t array for sent payload
574 * rxq_mbufs[slot]: mbuf pointer array for received frames
575 * txq_mbufs[slot]: mbuf pointer array for sent frames
576 */
577 static int
578 vioif_alloc_mems(struct vioif_softc *sc)
579 {
580 struct virtio_softc *vsc = sc->sc_virtio;
581 struct vioif_txqueue *txq;
582 struct vioif_rxqueue *rxq;
583 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
584 int allocsize, allocsize2, r, rsegs, i, qid;
585 void *vaddr;
586 intptr_t p;
587
588 allocsize = 0;
589 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
590 rxq = &sc->sc_rxq[qid];
591 txq = &sc->sc_txq[qid];
592
593 allocsize += sizeof(struct virtio_net_hdr) *
594 (rxq->rxq_vq->vq_num + txq->txq_vq->vq_num);
595 }
596 if (sc->sc_has_ctrl) {
597 allocsize += sizeof(struct virtio_net_ctrl_cmd);
598 allocsize += sizeof(struct virtio_net_ctrl_status);
599 allocsize += sizeof(struct virtio_net_ctrl_rx);
600 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
601 + ETHER_ADDR_LEN;
602 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
603 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
604 allocsize += sizeof(struct virtio_net_ctrl_mac_addr);
605 allocsize += sizeof(struct virtio_net_ctrl_mq);
606 }
607 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
608 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
609 if (r != 0) {
610 aprint_error_dev(sc->sc_dev,
611 "DMA memory allocation failed, size %d, "
612 "error code %d\n", allocsize, r);
613 goto err_none;
614 }
615 r = bus_dmamem_map(virtio_dmat(vsc),
616 &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT);
617 if (r != 0) {
618 aprint_error_dev(sc->sc_dev,
619 "DMA memory map failed, error code %d\n", r);
620 goto err_dmamem_alloc;
621 }
622
623 memset(vaddr, 0, allocsize);
624 sc->sc_dmamem = vaddr;
625 p = (intptr_t) vaddr;
626
627 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
628 rxq = &sc->sc_rxq[qid];
629 txq = &sc->sc_txq[qid];
630
631 rxq->rxq_hdrs = vioif_assign_mem(&p,
632 sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num);
633 txq->txq_hdrs = vioif_assign_mem(&p,
634 sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num);
635 }
636 if (sc->sc_has_ctrl) {
637 ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
638 sizeof(*ctrlq->ctrlq_cmd));
639 ctrlq->ctrlq_status = vioif_assign_mem(&p,
640 sizeof(*ctrlq->ctrlq_status));
641 ctrlq->ctrlq_rx = vioif_assign_mem(&p,
642 sizeof(*ctrlq->ctrlq_rx));
643 ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
644 sizeof(*ctrlq->ctrlq_mac_tbl_uc)
645 + ETHER_ADDR_LEN);
646 ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
647 sizeof(*ctrlq->ctrlq_mac_tbl_mc)
648 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
649 ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
650 sizeof(*ctrlq->ctrlq_mac_addr));
651 ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
652 }
653
654 allocsize2 = 0;
655 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
656 int rxqsize, txqsize;
657
658 rxq = &sc->sc_rxq[qid];
659 txq = &sc->sc_txq[qid];
660 rxqsize = rxq->rxq_vq->vq_num;
661 txqsize = txq->txq_vq->vq_num;
662
663 allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
664 allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
665 allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
666
667 allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
668 allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
669 allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
670 }
671 vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
672 sc->sc_kmem = vaddr;
673 p = (intptr_t) vaddr;
674
675 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
676 int rxqsize, txqsize;
677 rxq = &sc->sc_rxq[qid];
678 txq = &sc->sc_txq[qid];
679 rxqsize = rxq->rxq_vq->vq_num;
680 txqsize = txq->txq_vq->vq_num;
681
682 rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p,
683 sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
684 txq->txq_hdr_dmamaps = vioif_assign_mem(&p,
685 sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
686 rxq->rxq_dmamaps = vioif_assign_mem(&p,
687 sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
688 txq->txq_dmamaps = vioif_assign_mem(&p,
689 sizeof(txq->txq_dmamaps[0]) * txqsize);
690 rxq->rxq_mbufs = vioif_assign_mem(&p,
691 sizeof(rxq->rxq_mbufs[0]) * rxqsize);
692 txq->txq_mbufs = vioif_assign_mem(&p,
693 sizeof(txq->txq_mbufs[0]) * txqsize);
694 }
695
696 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
697 rxq = &sc->sc_rxq[qid];
698 txq = &sc->sc_txq[qid];
699
700 for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
701 r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i],
702 &rxq->rxq_hdrs[i], sc->sc_hdr_size, 1,
703 BUS_DMA_READ, "rx header");
704 if (r != 0)
705 goto err_reqs;
706
707 r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i],
708 MCLBYTES, 1, "rx payload");
709 if (r != 0)
710 goto err_reqs;
711 }
712
713 for (i = 0; i < txq->txq_vq->vq_num; i++) {
714 r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i],
715 &txq->txq_hdrs[i], sc->sc_hdr_size, 1,
716 BUS_DMA_READ, "tx header");
717 if (r != 0)
718 goto err_reqs;
719
720 r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN,
721 VIRTIO_NET_TX_MAXNSEGS, "tx payload");
722 if (r != 0)
723 goto err_reqs;
724 }
725 }
726
727 if (sc->sc_has_ctrl) {
728 /* control vq class & command */
729 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
730 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
731 BUS_DMA_WRITE, "control command");
732 if (r != 0)
733 goto err_reqs;
734
735 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
736 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
737 BUS_DMA_READ, "control status");
738 if (r != 0)
739 goto err_reqs;
740
741 /* control vq rx mode command parameter */
742 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
743 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
744 BUS_DMA_WRITE, "rx mode control command");
745 if (r != 0)
746 goto err_reqs;
747
748 /* multiqueue set command */
749 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
750 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
751 BUS_DMA_WRITE, "multiqueue set command");
752 if (r != 0)
753 goto err_reqs;
754
755 /* control vq MAC filter table for unicast */
756 /* do not load now since its length is variable */
757 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
758 sizeof(*ctrlq->ctrlq_mac_tbl_uc)
759 + ETHER_ADDR_LEN, 1,
760 "unicast MAC address filter command");
761 if (r != 0)
762 goto err_reqs;
763
764 /* control vq MAC filter table for multicast */
765 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
766 sizeof(*ctrlq->ctrlq_mac_tbl_mc)
767 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
768 "multicast MAC address filter command");
769 if (r != 0)
770 goto err_reqs;
771
772 /* control vq MAC address set command */
773 r = vioif_dmamap_create_load(sc,
774 &ctrlq->ctrlq_mac_addr_dmamap,
775 ctrlq->ctrlq_mac_addr,
776 sizeof(*ctrlq->ctrlq_mac_addr), 1,
777 BUS_DMA_WRITE, "mac addr set command");
778 if (r != 0)
779 goto err_reqs;
780 }
781
782 return 0;
783
784 err_reqs:
785 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
786 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
787 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
788 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
789 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
790 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
791 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
792 rxq = &sc->sc_rxq[qid];
793 txq = &sc->sc_txq[qid];
794
795 for (i = 0; i < txq->txq_vq->vq_num; i++) {
796 vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]);
797 vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]);
798 }
799 for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
800 vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]);
801 vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]);
802 }
803 }
804 if (sc->sc_kmem) {
805 kmem_free(sc->sc_kmem, allocsize2);
806 sc->sc_kmem = NULL;
807 }
808 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
809 err_dmamem_alloc:
810 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
811 err_none:
812 return -1;
813 }
814
815 static void
816 vioif_attach(device_t parent, device_t self, void *aux)
817 {
818 struct vioif_softc *sc = device_private(self);
819 struct virtio_softc *vsc = device_private(parent);
820 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
821 struct vioif_txqueue *txq;
822 struct vioif_rxqueue *rxq;
823 uint64_t features, req_features;
824 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
825 u_int softint_flags;
826 int r, i, nvqs = 0, req_flags;
827 char xnamebuf[MAXCOMLEN];
828
829 if (virtio_child(vsc) != NULL) {
830 aprint_normal(": child already attached for %s; "
831 "something wrong...\n", device_xname(parent));
832 return;
833 }
834
835 sc->sc_dev = self;
836 sc->sc_virtio = vsc;
837 sc->sc_link_state = LINK_STATE_UNKNOWN;
838
839 sc->sc_max_nvq_pairs = 1;
840 sc->sc_req_nvq_pairs = 1;
841 sc->sc_act_nvq_pairs = 1;
842 sc->sc_txrx_workqueue_sysctl = true;
843 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
844 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
845 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
846 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
847
848 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
849
850 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
851 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
852 IPL_NET, WQ_PERCPU | WQ_MPSAFE);
853 if (sc->sc_txrx_workqueue == NULL)
854 goto err;
855
856 req_flags = 0;
857
858 #ifdef VIOIF_MPSAFE
859 req_flags |= VIRTIO_F_INTR_MPSAFE;
860 #endif
861 req_flags |= VIRTIO_F_INTR_MSIX;
862
863 req_features =
864 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
865 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
866 req_features |= VIRTIO_F_RING_EVENT_IDX;
867 req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
868 #ifdef VIOIF_MULTIQ
869 req_features |= VIRTIO_NET_F_MQ;
870 #endif
871 virtio_child_attach_start(vsc, self, IPL_NET, NULL,
872 vioif_config_change, virtio_vq_intrhand, req_flags,
873 req_features, VIRTIO_NET_FLAG_BITS);
874
875 features = virtio_features(vsc);
876 if (features == 0)
877 goto err;
878
879 if (features & VIRTIO_NET_F_MAC) {
880 for (i = 0; i < __arraycount(sc->sc_mac); i++) {
881 sc->sc_mac[i] = virtio_read_device_config_1(vsc,
882 VIRTIO_NET_CONFIG_MAC + i);
883 }
884 } else {
885 /* code stolen from sys/net/if_tap.c */
886 struct timeval tv;
887 uint32_t ui;
888 getmicrouptime(&tv);
889 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
890 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
891 for (i = 0; i < __arraycount(sc->sc_mac); i++) {
892 virtio_write_device_config_1(vsc,
893 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
894 }
895 }
896
897 /* 'Ethernet' with capital follows other ethernet driver attachment */
898 aprint_normal_dev(self, "Ethernet address %s\n",
899 ether_sprintf(sc->sc_mac));
900
901 if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
902 sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
903 } else {
904 sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
905 }
906
907 if ((features & VIRTIO_NET_F_CTRL_VQ) &&
908 (features & VIRTIO_NET_F_CTRL_RX)) {
909 sc->sc_has_ctrl = true;
910
911 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
912 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
913 ctrlq->ctrlq_inuse = FREE;
914 } else {
915 sc->sc_has_ctrl = false;
916 }
917
918 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
919 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
920 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
921
922 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
923 goto err;
924
925 /* Limit the number of queue pairs to use */
926 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
927 }
928
929 vioif_alloc_queues(sc);
930 virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
931
932 #ifdef VIOIF_MPSAFE
933 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
934 #else
935 softint_flags = SOFTINT_NET;
936 #endif
937
938 /*
939 * Allocating virtqueues
940 */
941 for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
942 rxq = &sc->sc_rxq[i];
943 txq = &sc->sc_txq[i];
944 char qname[32];
945
946 rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
947
948 rxq->rxq_handle_si = softint_establish(softint_flags,
949 vioif_rx_handle, rxq);
950 if (rxq->rxq_handle_si == NULL) {
951 aprint_error_dev(self, "cannot establish rx softint\n");
952 goto err;
953 }
954
955 snprintf(qname, sizeof(qname), "rx%d", i);
956 r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
957 MCLBYTES + sc->sc_hdr_size, 2, qname);
958 if (r != 0)
959 goto err;
960 nvqs++;
961 rxq->rxq_vq->vq_intrhand = vioif_rx_intr;
962 rxq->rxq_vq->vq_intrhand_arg = (void *)rxq;
963 rxq->rxq_stopping = false;
964 rxq->rxq_running_handle = false;
965 vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq);
966
967 txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
968
969 txq->txq_deferred_transmit = softint_establish(softint_flags,
970 vioif_deferred_transmit, txq);
971 if (txq->txq_deferred_transmit == NULL) {
972 aprint_error_dev(self, "cannot establish tx softint\n");
973 goto err;
974 }
975 txq->txq_handle_si = softint_establish(softint_flags,
976 vioif_tx_handle, txq);
977 if (txq->txq_handle_si == NULL) {
978 aprint_error_dev(self, "cannot establish tx softint\n");
979 goto err;
980 }
981
982 snprintf(qname, sizeof(qname), "tx%d", i);
983 r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
984 sc->sc_hdr_size + (ETHER_MAX_LEN - ETHER_HDR_LEN),
985 VIRTIO_NET_TX_MAXNSEGS + 1, qname);
986 if (r != 0)
987 goto err;
988 nvqs++;
989 txq->txq_vq->vq_intrhand = vioif_tx_intr;
990 txq->txq_vq->vq_intrhand_arg = (void *)txq;
991 txq->txq_link_active = VIOIF_IS_LINK_ACTIVE(sc);
992 txq->txq_stopping = false;
993 txq->txq_running_handle = false;
994 txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP);
995 vioif_work_set(&txq->txq_work, vioif_tx_handle, txq);
996 }
997
998 if (sc->sc_has_ctrl) {
999 /*
1000 * Allocating a virtqueue for control channel
1001 */
1002 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
1003 NBPG, 1, "control");
1004 if (r != 0) {
1005 aprint_error_dev(self, "failed to allocate "
1006 "a virtqueue for control channel, error code %d\n",
1007 r);
1008
1009 sc->sc_has_ctrl = false;
1010 cv_destroy(&ctrlq->ctrlq_wait);
1011 mutex_destroy(&ctrlq->ctrlq_wait_lock);
1012 } else {
1013 nvqs++;
1014 ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
1015 ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
1016 }
1017 }
1018
1019 sc->sc_ctl_softint = softint_establish(softint_flags,
1020 vioif_ctl_softint, sc);
1021 if (sc->sc_ctl_softint == NULL) {
1022 aprint_error_dev(self, "cannot establish ctl softint\n");
1023 goto err;
1024 }
1025
1026 if (vioif_alloc_mems(sc) < 0)
1027 goto err;
1028
1029 if (virtio_child_attach_finish(vsc) != 0)
1030 goto err;
1031
1032 if (vioif_setup_sysctl(sc) != 0) {
1033 aprint_error_dev(self, "unable to create sysctl node\n");
1034 /* continue */
1035 }
1036
1037 vioif_setup_stats(sc);
1038
1039 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
1040 ifp->if_softc = sc;
1041 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1042 #ifdef VIOIF_MPSAFE
1043 ifp->if_extflags = IFEF_MPSAFE;
1044 #endif
1045 ifp->if_start = vioif_start;
1046 if (sc->sc_req_nvq_pairs > 1)
1047 ifp->if_transmit = vioif_transmit;
1048 ifp->if_ioctl = vioif_ioctl;
1049 ifp->if_init = vioif_init;
1050 ifp->if_stop = vioif_stop;
1051 ifp->if_capabilities = 0;
1052 ifp->if_watchdog = vioif_watchdog;
1053 txq = &sc->sc_txq[0];
1054 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
1055 IFQ_SET_READY(&ifp->if_snd);
1056
1057 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
1058
1059 if_attach(ifp);
1060 if_deferred_start_init(ifp, NULL);
1061 ether_ifattach(ifp, sc->sc_mac);
1062 ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
1063
1064 return;
1065
1066 err:
1067 for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
1068 rxq = &sc->sc_rxq[i];
1069 txq = &sc->sc_txq[i];
1070
1071 if (rxq->rxq_lock) {
1072 mutex_obj_free(rxq->rxq_lock);
1073 rxq->rxq_lock = NULL;
1074 }
1075
1076 if (rxq->rxq_handle_si) {
1077 softint_disestablish(rxq->rxq_handle_si);
1078 rxq->rxq_handle_si = NULL;
1079 }
1080
1081 if (txq->txq_lock) {
1082 mutex_obj_free(txq->txq_lock);
1083 txq->txq_lock = NULL;
1084 }
1085
1086 if (txq->txq_handle_si) {
1087 softint_disestablish(txq->txq_handle_si);
1088 txq->txq_handle_si = NULL;
1089 }
1090
1091 if (txq->txq_deferred_transmit) {
1092 softint_disestablish(txq->txq_deferred_transmit);
1093 txq->txq_deferred_transmit = NULL;
1094 }
1095
1096 if (txq->txq_intrq) {
1097 pcq_destroy(txq->txq_intrq);
1098 txq->txq_intrq = NULL;
1099 }
1100 }
1101
1102 if (sc->sc_has_ctrl) {
1103 cv_destroy(&ctrlq->ctrlq_wait);
1104 mutex_destroy(&ctrlq->ctrlq_wait_lock);
1105 }
1106
1107 while (nvqs > 0)
1108 virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
1109
1110 vioif_free_queues(sc);
1111 mutex_destroy(&sc->sc_lock);
1112 virtio_child_attach_failed(vsc);
1113 config_finalize_register(self, vioif_finalize_teardown);
1114
1115 return;
1116 }
1117
1118 static int
1119 vioif_finalize_teardown(device_t self)
1120 {
1121 struct vioif_softc *sc = device_private(self);
1122
1123 if (sc->sc_txrx_workqueue != NULL) {
1124 vioif_workq_destroy(sc->sc_txrx_workqueue);
1125 sc->sc_txrx_workqueue = NULL;
1126 }
1127
1128 return 0;
1129 }
1130
1131 static void
1132 vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
1133 {
1134 struct virtio_softc *vsc = sc->sc_virtio;
1135 struct vioif_txqueue *txq;
1136 struct vioif_rxqueue *rxq;
1137 int i;
1138
1139 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1140 txq = &sc->sc_txq[i];
1141 rxq = &sc->sc_rxq[i];
1142
1143 virtio_start_vq_intr(vsc, txq->txq_vq);
1144 virtio_start_vq_intr(vsc, rxq->rxq_vq);
1145 }
1146 }
1147
1148 static void
1149 vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
1150 {
1151 struct virtio_softc *vsc = sc->sc_virtio;
1152 struct vioif_txqueue *txq;
1153 struct vioif_rxqueue *rxq;
1154 int i;
1155
1156 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1157 rxq = &sc->sc_rxq[i];
1158 txq = &sc->sc_txq[i];
1159
1160 virtio_stop_vq_intr(vsc, rxq->rxq_vq);
1161 virtio_stop_vq_intr(vsc, txq->txq_vq);
1162 }
1163 }
1164
1165 /*
1166 * Interface functions for ifnet
1167 */
1168 static int
1169 vioif_init(struct ifnet *ifp)
1170 {
1171 struct vioif_softc *sc = ifp->if_softc;
1172 struct virtio_softc *vsc = sc->sc_virtio;
1173 struct vioif_rxqueue *rxq;
1174 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1175 int r, i;
1176
1177 vioif_stop(ifp, 0);
1178
1179 r = virtio_reinit_start(vsc);
1180 if (r != 0) {
1181 log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
1182 return EIO;
1183 }
1184
1185 virtio_negotiate_features(vsc, virtio_features(vsc));
1186
1187 for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
1188 rxq = &sc->sc_rxq[i];
1189
1190 mutex_enter(rxq->rxq_lock);
1191 vioif_populate_rx_mbufs_locked(sc, rxq);
1192 mutex_exit(rxq->rxq_lock);
1193
1194 }
1195
1196 virtio_reinit_end(vsc);
1197
1198 if (sc->sc_has_ctrl)
1199 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
1200
1201 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
1202 if (r == 0)
1203 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
1204 else
1205 sc->sc_act_nvq_pairs = 1;
1206
1207 vioif_enable_interrupt_vqpairs(sc);
1208
1209 vioif_update_link_status(sc);
1210 ifp->if_flags |= IFF_RUNNING;
1211 ifp->if_flags &= ~IFF_OACTIVE;
1212 r = vioif_rx_filter(sc);
1213
1214 return r;
1215 }
1216
1217 static void
1218 vioif_stop(struct ifnet *ifp, int disable)
1219 {
1220 struct vioif_softc *sc = ifp->if_softc;
1221 struct virtio_softc *vsc = sc->sc_virtio;
1222 struct vioif_txqueue *txq;
1223 struct vioif_rxqueue *rxq;
1224 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1225 int i;
1226
1227
1228 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1229 txq = &sc->sc_txq[i];
1230 rxq = &sc->sc_rxq[i];
1231
1232 mutex_enter(rxq->rxq_lock);
1233 rxq->rxq_stopping = true;
1234 mutex_exit(rxq->rxq_lock);
1235
1236 mutex_enter(txq->txq_lock);
1237 txq->txq_stopping = true;
1238 mutex_exit(txq->txq_lock);
1239 }
1240
1241 /* disable interrupts */
1242 vioif_disable_interrupt_vqpairs(sc);
1243 if (sc->sc_has_ctrl)
1244 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
1245
1246 /*
1247 * only way to stop interrupt, I/O and DMA is resetting...
1248 *
1249 * NOTE: Devices based on VirtIO draft specification can not
1250 * stop interrupt completely even if virtio_stop_vq_intr() is called.
1251 */
1252 virtio_reset(vsc);
1253
1254 vioif_intr_barrier();
1255
1256 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1257 txq = &sc->sc_txq[i];
1258 rxq = &sc->sc_rxq[i];
1259
1260 vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work);
1261 vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work);
1262 }
1263
1264 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1265 vioif_rx_queue_clear(&sc->sc_rxq[i]);
1266 vioif_tx_queue_clear(&sc->sc_txq[i]);
1267 }
1268
1269 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1270
1271 /* all packet processing is stopped */
1272 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1273 txq = &sc->sc_txq[i];
1274 rxq = &sc->sc_rxq[i];
1275
1276 mutex_enter(rxq->rxq_lock);
1277 rxq->rxq_stopping = false;
1278 KASSERT(!rxq->rxq_running_handle);
1279 mutex_exit(rxq->rxq_lock);
1280
1281 mutex_enter(txq->txq_lock);
1282 txq->txq_stopping = false;
1283 KASSERT(!txq->txq_running_handle);
1284 mutex_exit(txq->txq_lock);
1285 }
1286
1287 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1288 txq = &sc->sc_txq[i];
1289 rxq = &sc->sc_rxq[i];
1290
1291 if (disable)
1292 vioif_rx_drain(rxq);
1293
1294 vioif_tx_drain(txq);
1295 }
1296 }
1297
1298 static void
1299 vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq,
1300 bool is_transmit)
1301 {
1302 struct vioif_softc *sc = ifp->if_softc;
1303 struct virtio_softc *vsc = sc->sc_virtio;
1304 struct virtqueue *vq = txq->txq_vq;
1305 struct virtio_net_hdr *hdr;
1306 struct mbuf *m;
1307 int queued = 0;
1308
1309 KASSERT(mutex_owned(txq->txq_lock));
1310
1311 if ((ifp->if_flags & IFF_RUNNING) == 0)
1312 return;
1313
1314 if (!txq->txq_link_active || txq->txq_stopping)
1315 return;
1316
1317 if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
1318 return;
1319
1320 for (;;) {
1321 int slot, r;
1322
1323 if (is_transmit)
1324 m = pcq_get(txq->txq_intrq);
1325 else
1326 IFQ_DEQUEUE(&ifp->if_snd, m);
1327
1328 if (m == NULL)
1329 break;
1330
1331 r = virtio_enqueue_prep(vsc, vq, &slot);
1332 if (r == EAGAIN) {
1333 ifp->if_flags |= IFF_OACTIVE;
1334 m_freem(m);
1335 break;
1336 }
1337 if (r != 0)
1338 panic("enqueue_prep for a tx buffer");
1339
1340 r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1341 txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1342 if (r != 0) {
1343 /* maybe just too fragmented */
1344 struct mbuf *newm;
1345
1346 newm = m_defrag(m, M_NOWAIT);
1347 if (newm == NULL) {
1348 txq->txq_defrag_failed.ev_count++;
1349 goto skip;
1350 }
1351
1352 m = newm;
1353 r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1354 txq->txq_dmamaps[slot], m,
1355 BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1356 if (r != 0) {
1357 txq->txq_mbuf_load_failed.ev_count++;
1358 skip:
1359 m_freem(m);
1360 virtio_enqueue_abort(vsc, vq, slot);
1361 continue;
1362 }
1363 }
1364
1365 /* This should actually never fail */
1366 r = virtio_enqueue_reserve(vsc, vq, slot,
1367 txq->txq_dmamaps[slot]->dm_nsegs + 1);
1368 if (r != 0) {
1369 txq->txq_enqueue_reserve_failed.ev_count++;
1370 bus_dmamap_unload(virtio_dmat(vsc),
1371 txq->txq_dmamaps[slot]);
1372 /* slot already freed by virtio_enqueue_reserve */
1373 m_freem(m);
1374 continue;
1375 }
1376
1377 txq->txq_mbufs[slot] = m;
1378
1379 hdr = &txq->txq_hdrs[slot];
1380 memset(hdr, 0, sc->sc_hdr_size);
1381 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1382 0, txq->txq_dmamaps[slot]->dm_mapsize,
1383 BUS_DMASYNC_PREWRITE);
1384 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1385 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
1386 BUS_DMASYNC_PREWRITE);
1387 virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
1388 virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
1389 virtio_enqueue_commit(vsc, vq, slot, false);
1390
1391 queued++;
1392 bpf_mtap(ifp, m, BPF_D_OUT);
1393 }
1394
1395 if (queued > 0) {
1396 virtio_enqueue_commit(vsc, vq, -1, true);
1397 ifp->if_timer = 5;
1398 }
1399 }
1400
1401 static void
1402 vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1403 {
1404
1405 /*
1406 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
1407 */
1408 vioif_send_common_locked(ifp, txq, false);
1409
1410 }
1411
1412 static void
1413 vioif_start(struct ifnet *ifp)
1414 {
1415 struct vioif_softc *sc = ifp->if_softc;
1416 struct vioif_txqueue *txq = &sc->sc_txq[0];
1417
1418 #ifdef VIOIF_MPSAFE
1419 KASSERT(if_is_mpsafe(ifp));
1420 #endif
1421
1422 mutex_enter(txq->txq_lock);
1423 vioif_start_locked(ifp, txq);
1424 mutex_exit(txq->txq_lock);
1425 }
1426
1427 static inline int
1428 vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
1429 {
1430 struct vioif_softc *sc = ifp->if_softc;
1431 u_int cpuid = cpu_index(curcpu());
1432
1433 return cpuid % sc->sc_act_nvq_pairs;
1434 }
1435
1436 static void
1437 vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1438 {
1439
1440 vioif_send_common_locked(ifp, txq, true);
1441 }
1442
1443 static int
1444 vioif_transmit(struct ifnet *ifp, struct mbuf *m)
1445 {
1446 struct vioif_softc *sc = ifp->if_softc;
1447 struct vioif_txqueue *txq;
1448 int qid;
1449
1450 qid = vioif_select_txqueue(ifp, m);
1451 txq = &sc->sc_txq[qid];
1452
1453 if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
1454 m_freem(m);
1455 return ENOBUFS;
1456 }
1457
1458 net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
1459 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
1460 if (m->m_flags & M_MCAST)
1461 if_statinc_ref(nsr, if_omcasts);
1462 IF_STAT_PUTREF(ifp);
1463
1464 if (mutex_tryenter(txq->txq_lock)) {
1465 vioif_transmit_locked(ifp, txq);
1466 mutex_exit(txq->txq_lock);
1467 }
1468
1469 return 0;
1470 }
1471
1472 static void
1473 vioif_deferred_transmit(void *arg)
1474 {
1475 struct vioif_txqueue *txq = arg;
1476 struct virtio_softc *vsc = txq->txq_vq->vq_owner;
1477 struct vioif_softc *sc = device_private(virtio_child(vsc));
1478 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1479
1480 mutex_enter(txq->txq_lock);
1481 vioif_send_common_locked(ifp, txq, true);
1482 mutex_exit(txq->txq_lock);
1483 }
1484
1485 static int
1486 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
1487 {
1488 int s, r;
1489
1490 s = splnet();
1491
1492 r = ether_ioctl(ifp, cmd, data);
1493 if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
1494 if (ifp->if_flags & IFF_RUNNING) {
1495 r = vioif_rx_filter(ifp->if_softc);
1496 } else {
1497 r = 0;
1498 }
1499 }
1500
1501 splx(s);
1502
1503 return r;
1504 }
1505
1506 void
1507 vioif_watchdog(struct ifnet *ifp)
1508 {
1509 struct vioif_softc *sc = ifp->if_softc;
1510 int i;
1511
1512 if (ifp->if_flags & IFF_RUNNING) {
1513 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1514 vioif_tx_queue_clear(&sc->sc_txq[i]);
1515 }
1516 }
1517 }
1518
1519 /*
1520 * Receive implementation
1521 */
1522 /* allocate and initialize a mbuf for receive */
1523 static int
1524 vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1525 {
1526 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1527 struct mbuf *m;
1528 int r;
1529
1530 MGETHDR(m, M_DONTWAIT, MT_DATA);
1531 if (m == NULL)
1532 return ENOBUFS;
1533 MCLGET(m, M_DONTWAIT);
1534 if ((m->m_flags & M_EXT) == 0) {
1535 m_freem(m);
1536 return ENOBUFS;
1537 }
1538 rxq->rxq_mbufs[i] = m;
1539 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1540 r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1541 rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT);
1542 if (r) {
1543 m_freem(m);
1544 rxq->rxq_mbufs[i] = NULL;
1545 return r;
1546 }
1547
1548 return 0;
1549 }
1550
1551 /* free a mbuf for receive */
1552 static void
1553 vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1554 {
1555 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1556
1557 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
1558 m_freem(rxq->rxq_mbufs[i]);
1559 rxq->rxq_mbufs[i] = NULL;
1560 }
1561
1562 /* add mbufs for all the empty receive slots */
1563 static void
1564 vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
1565 {
1566 struct virtqueue *vq = rxq->rxq_vq;
1567 struct virtio_softc *vsc = vq->vq_owner;
1568 int i, r, ndone = 0;
1569
1570 KASSERT(mutex_owned(rxq->rxq_lock));
1571
1572 if (rxq->rxq_stopping)
1573 return;
1574
1575 for (i = 0; i < vq->vq_num; i++) {
1576 int slot;
1577 r = virtio_enqueue_prep(vsc, vq, &slot);
1578 if (r == EAGAIN)
1579 break;
1580 if (r != 0)
1581 panic("enqueue_prep for rx buffers");
1582 if (rxq->rxq_mbufs[slot] == NULL) {
1583 r = vioif_add_rx_mbuf(rxq, slot);
1584 if (r != 0) {
1585 rxq->rxq_mbuf_add_failed.ev_count++;
1586 break;
1587 }
1588 }
1589 r = virtio_enqueue_reserve(vsc, vq, slot,
1590 rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
1591 if (r != 0) {
1592 vioif_free_rx_mbuf(rxq, slot);
1593 break;
1594 }
1595 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1596 0, sc->sc_hdr_size, BUS_DMASYNC_PREREAD);
1597 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1598 0, MCLBYTES, BUS_DMASYNC_PREREAD);
1599 virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot],
1600 false);
1601 virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
1602 virtio_enqueue_commit(vsc, vq, slot, false);
1603 ndone++;
1604 }
1605 if (ndone > 0)
1606 virtio_enqueue_commit(vsc, vq, -1, true);
1607 }
1608
1609 static void
1610 vioif_rx_queue_clear(struct vioif_rxqueue *rxq)
1611 {
1612 struct virtqueue *vq = rxq->rxq_vq;
1613 struct virtio_softc *vsc = vq->vq_owner;
1614 struct vioif_softc *sc = device_private(virtio_child(vsc));
1615 u_int limit = UINT_MAX;
1616 bool more;
1617
1618 KASSERT(rxq->rxq_stopping);
1619
1620 mutex_enter(rxq->rxq_lock);
1621 for (;;) {
1622 more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1623 if (more == false)
1624 break;
1625 }
1626 mutex_exit(rxq->rxq_lock);
1627 }
1628
1629 /* dequeue received packets */
1630 static bool
1631 vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1632 struct vioif_rxqueue *rxq, u_int limit)
1633 {
1634 struct virtqueue *vq = rxq->rxq_vq;
1635 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1636 struct mbuf *m;
1637 int slot, len;
1638 bool more = false, dequeued = false;
1639
1640 KASSERT(mutex_owned(rxq->rxq_lock));
1641
1642 if (virtio_vq_is_enqueued(vsc, vq) == false)
1643 return false;
1644
1645 for (;;) {
1646 if (limit-- == 0) {
1647 more = true;
1648 break;
1649 }
1650
1651 if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1652 break;
1653
1654 dequeued = true;
1655
1656 len -= sc->sc_hdr_size;
1657 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1658 0, sc->sc_hdr_size, BUS_DMASYNC_POSTREAD);
1659 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1660 0, MCLBYTES, BUS_DMASYNC_POSTREAD);
1661 m = rxq->rxq_mbufs[slot];
1662 KASSERT(m != NULL);
1663 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
1664 rxq->rxq_mbufs[slot] = NULL;
1665 virtio_dequeue_commit(vsc, vq, slot);
1666 m_set_rcvif(m, ifp);
1667 m->m_len = m->m_pkthdr.len = len;
1668
1669 if_percpuq_enqueue(ifp->if_percpuq, m);
1670 }
1671
1672 if (dequeued)
1673 vioif_populate_rx_mbufs_locked(sc, rxq);
1674
1675 return more;
1676 }
1677
1678 /* rx interrupt; call _dequeue above and schedule a softint */
1679
1680 static void
1681 vioif_rx_handle_locked(void *xrxq, u_int limit)
1682 {
1683 struct vioif_rxqueue *rxq = xrxq;
1684 struct virtqueue *vq = rxq->rxq_vq;
1685 struct virtio_softc *vsc = vq->vq_owner;
1686 struct vioif_softc *sc = device_private(virtio_child(vsc));
1687 bool more;
1688
1689 KASSERT(mutex_owned(rxq->rxq_lock));
1690 KASSERT(!rxq->rxq_stopping);
1691
1692 more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1693 if (more) {
1694 vioif_rx_sched_handle(sc, rxq);
1695 return;
1696 }
1697 more = virtio_start_vq_intr(vsc, rxq->rxq_vq);
1698 if (more) {
1699 vioif_rx_sched_handle(sc, rxq);
1700 return;
1701 }
1702
1703 rxq->rxq_running_handle = false;
1704 }
1705
1706 static int
1707 vioif_rx_intr(void *arg)
1708 {
1709 struct vioif_rxqueue *rxq = arg;
1710 struct virtqueue *vq = rxq->rxq_vq;
1711 struct virtio_softc *vsc = vq->vq_owner;
1712 struct vioif_softc *sc = device_private(virtio_child(vsc));
1713 u_int limit;
1714
1715
1716 mutex_enter(rxq->rxq_lock);
1717
1718 /* rx handler is already running in softint/workqueue */
1719 if (rxq->rxq_running_handle)
1720 goto done;
1721
1722 if (rxq->rxq_stopping)
1723 goto done;
1724
1725 rxq->rxq_running_handle = true;
1726
1727 limit = sc->sc_rx_intr_process_limit;
1728 virtio_stop_vq_intr(vsc, vq);
1729 vioif_rx_handle_locked(rxq, limit);
1730
1731 done:
1732 mutex_exit(rxq->rxq_lock);
1733 return 1;
1734 }
1735
1736 static void
1737 vioif_rx_handle(void *xrxq)
1738 {
1739 struct vioif_rxqueue *rxq = xrxq;
1740 struct virtqueue *vq = rxq->rxq_vq;
1741 struct virtio_softc *vsc = vq->vq_owner;
1742 struct vioif_softc *sc = device_private(virtio_child(vsc));
1743 u_int limit;
1744
1745 mutex_enter(rxq->rxq_lock);
1746
1747 KASSERT(rxq->rxq_running_handle);
1748
1749 if (rxq->rxq_stopping) {
1750 rxq->rxq_running_handle = false;
1751 goto done;
1752 }
1753
1754 limit = sc->sc_rx_process_limit;
1755 vioif_rx_handle_locked(rxq, limit);
1756
1757 done:
1758 mutex_exit(rxq->rxq_lock);
1759 }
1760
1761 static void
1762 vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
1763 {
1764
1765 KASSERT(mutex_owned(rxq->rxq_lock));
1766
1767 if (rxq->rxq_stopping)
1768 return;
1769
1770 if (rxq->rxq_workqueue)
1771 vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work);
1772 else
1773 softint_schedule(rxq->rxq_handle_si);
1774 }
1775
1776 /* free all the mbufs; called from if_stop(disable) */
1777 static void
1778 vioif_rx_drain(struct vioif_rxqueue *rxq)
1779 {
1780 struct virtqueue *vq = rxq->rxq_vq;
1781 int i;
1782
1783 for (i = 0; i < vq->vq_num; i++) {
1784 if (rxq->rxq_mbufs[i] == NULL)
1785 continue;
1786 vioif_free_rx_mbuf(rxq, i);
1787 }
1788 }
1789
1790 /*
1791 * Transmition implementation
1792 */
1793 /* actual transmission is done in if_start */
1794 /* tx interrupt; dequeue and free mbufs */
1795 /*
1796 * tx interrupt is actually disabled; this should be called upon
1797 * tx vq full and watchdog
1798 */
1799
1800 static void
1801 vioif_tx_handle_locked(struct vioif_txqueue *txq, u_int limit)
1802 {
1803 struct virtqueue *vq = txq->txq_vq;
1804 struct virtio_softc *vsc = vq->vq_owner;
1805 struct vioif_softc *sc = device_private(virtio_child(vsc));
1806 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1807 bool more;
1808
1809 KASSERT(mutex_owned(txq->txq_lock));
1810 KASSERT(!txq->txq_stopping);
1811
1812 more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1813 if (more) {
1814 vioif_tx_sched_handle(sc, txq);
1815 return;
1816 }
1817
1818 if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
1819 more = virtio_postpone_intr_smart(vsc, vq);
1820 else
1821 more = virtio_start_vq_intr(vsc, vq);
1822 if (more) {
1823 vioif_tx_sched_handle(sc, txq);
1824 return;
1825 }
1826
1827 txq->txq_running_handle = false;
1828
1829 /* for ALTQ */
1830 if (txq == &sc->sc_txq[0]) {
1831 if_schedule_deferred_start(ifp);
1832 ifp->if_flags &= ~IFF_OACTIVE;
1833 }
1834 softint_schedule(txq->txq_deferred_transmit);
1835 }
1836
1837
1838 static int
1839 vioif_tx_intr(void *arg)
1840 {
1841 struct vioif_txqueue *txq = arg;
1842 struct virtqueue *vq = txq->txq_vq;
1843 struct virtio_softc *vsc = vq->vq_owner;
1844 struct vioif_softc *sc = device_private(virtio_child(vsc));
1845 u_int limit;
1846
1847 limit = sc->sc_tx_intr_process_limit;
1848
1849 mutex_enter(txq->txq_lock);
1850
1851 /* tx handler is already running in softint/workqueue */
1852 if (txq->txq_running_handle)
1853 goto done;
1854
1855 if (txq->txq_stopping)
1856 goto done;
1857
1858 txq->txq_running_handle = true;
1859
1860 virtio_stop_vq_intr(vsc, vq);
1861 txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl;
1862 vioif_tx_handle_locked(txq, limit);
1863
1864 done:
1865 mutex_exit(txq->txq_lock);
1866 return 1;
1867 }
1868
1869 static void
1870 vioif_tx_handle(void *xtxq)
1871 {
1872 struct vioif_txqueue *txq = xtxq;
1873 struct virtqueue *vq = txq->txq_vq;
1874 struct virtio_softc *vsc = vq->vq_owner;
1875 struct vioif_softc *sc = device_private(virtio_child(vsc));
1876 u_int limit;
1877
1878 mutex_enter(txq->txq_lock);
1879
1880 KASSERT(txq->txq_running_handle);
1881
1882 if (txq->txq_stopping) {
1883 txq->txq_running_handle = false;
1884 goto done;
1885 }
1886
1887 limit = sc->sc_tx_process_limit;
1888 vioif_tx_handle_locked(txq, limit);
1889
1890 done:
1891 mutex_exit(txq->txq_lock);
1892 }
1893
1894 static void
1895 vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq)
1896 {
1897
1898 KASSERT(mutex_owned(txq->txq_lock));
1899
1900 if (txq->txq_stopping)
1901 return;
1902
1903 if (txq->txq_workqueue)
1904 vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work);
1905 else
1906 softint_schedule(txq->txq_handle_si);
1907 }
1908
1909 static void
1910 vioif_tx_queue_clear(struct vioif_txqueue *txq)
1911 {
1912 struct virtqueue *vq = txq->txq_vq;
1913 struct virtio_softc *vsc = vq->vq_owner;
1914 struct vioif_softc *sc = device_private(virtio_child(vsc));
1915 u_int limit = UINT_MAX;
1916 bool more;
1917
1918 mutex_enter(txq->txq_lock);
1919 for (;;) {
1920 more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1921 if (more == false)
1922 break;
1923 }
1924 mutex_exit(txq->txq_lock);
1925 }
1926
1927 static bool
1928 vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1929 struct vioif_txqueue *txq, u_int limit)
1930 {
1931 struct virtqueue *vq = txq->txq_vq;
1932 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1933 struct mbuf *m;
1934 int slot, len;
1935 bool more = false;
1936
1937 KASSERT(mutex_owned(txq->txq_lock));
1938
1939 if (virtio_vq_is_enqueued(vsc, vq) == false)
1940 return false;
1941
1942 for (;;) {
1943 if (limit-- == 0) {
1944 more = true;
1945 break;
1946 }
1947
1948 if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1949 break;
1950
1951 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1952 0, sc->sc_hdr_size, BUS_DMASYNC_POSTWRITE);
1953 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1954 0, txq->txq_dmamaps[slot]->dm_mapsize,
1955 BUS_DMASYNC_POSTWRITE);
1956 m = txq->txq_mbufs[slot];
1957 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
1958 txq->txq_mbufs[slot] = NULL;
1959 virtio_dequeue_commit(vsc, vq, slot);
1960 if_statinc(ifp, if_opackets);
1961 m_freem(m);
1962 }
1963
1964 return more;
1965 }
1966
1967 /* free all the mbufs already put on vq; called from if_stop(disable) */
1968 static void
1969 vioif_tx_drain(struct vioif_txqueue *txq)
1970 {
1971 struct virtqueue *vq = txq->txq_vq;
1972 struct virtio_softc *vsc = vq->vq_owner;
1973 int i;
1974
1975 for (i = 0; i < vq->vq_num; i++) {
1976 if (txq->txq_mbufs[i] == NULL)
1977 continue;
1978 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
1979 m_freem(txq->txq_mbufs[i]);
1980 txq->txq_mbufs[i] = NULL;
1981 }
1982 }
1983
1984 /*
1985 * Control vq
1986 */
1987 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
1988 static void
1989 vioif_ctrl_acquire(struct vioif_softc *sc)
1990 {
1991 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1992
1993 mutex_enter(&ctrlq->ctrlq_wait_lock);
1994 while (ctrlq->ctrlq_inuse != FREE)
1995 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1996 ctrlq->ctrlq_inuse = INUSE;
1997 ctrlq->ctrlq_owner = curlwp;
1998 mutex_exit(&ctrlq->ctrlq_wait_lock);
1999 }
2000
2001 static void
2002 vioif_ctrl_release(struct vioif_softc *sc)
2003 {
2004 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2005
2006 KASSERT(ctrlq->ctrlq_inuse != FREE);
2007 KASSERT(ctrlq->ctrlq_owner == curlwp);
2008
2009 mutex_enter(&ctrlq->ctrlq_wait_lock);
2010 ctrlq->ctrlq_inuse = FREE;
2011 ctrlq->ctrlq_owner = NULL;
2012 cv_signal(&ctrlq->ctrlq_wait);
2013 mutex_exit(&ctrlq->ctrlq_wait_lock);
2014 }
2015
2016 static int
2017 vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
2018 struct vioif_ctrl_cmdspec *specs, int nspecs)
2019 {
2020 struct virtio_softc *vsc = sc->sc_virtio;
2021 int i, r, loaded;
2022
2023 loaded = 0;
2024 for (i = 0; i < nspecs; i++) {
2025 r = bus_dmamap_load(virtio_dmat(vsc),
2026 specs[i].dmamap, specs[i].buf, specs[i].bufsize,
2027 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
2028 if (r) {
2029 sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
2030 goto err;
2031 }
2032 loaded++;
2033
2034 }
2035
2036 return r;
2037
2038 err:
2039 for (i = 0; i < loaded; i++) {
2040 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2041 }
2042
2043 return r;
2044 }
2045
2046 static void
2047 vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
2048 struct vioif_ctrl_cmdspec *specs, int nspecs)
2049 {
2050 struct virtio_softc *vsc = sc->sc_virtio;
2051 int i;
2052
2053 for (i = 0; i < nspecs; i++) {
2054 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2055 }
2056 }
2057
2058 static int
2059 vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
2060 struct vioif_ctrl_cmdspec *specs, int nspecs)
2061 {
2062 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2063 struct virtqueue *vq = ctrlq->ctrlq_vq;
2064 struct virtio_softc *vsc = sc->sc_virtio;
2065 int i, r, slot;
2066
2067 ctrlq->ctrlq_cmd->class = class;
2068 ctrlq->ctrlq_cmd->command = cmd;
2069
2070 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
2071 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
2072 for (i = 0; i < nspecs; i++) {
2073 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
2074 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
2075 }
2076 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
2077 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
2078
2079 /* we need to explicitly (re)start vq intr when using RING EVENT IDX */
2080 if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
2081 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
2082
2083 r = virtio_enqueue_prep(vsc, vq, &slot);
2084 if (r != 0)
2085 panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2086 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
2087 if (r != 0)
2088 panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2089 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
2090 for (i = 0; i < nspecs; i++) {
2091 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
2092 }
2093 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
2094 virtio_enqueue_commit(vsc, vq, slot, true);
2095
2096 /* wait for done */
2097 mutex_enter(&ctrlq->ctrlq_wait_lock);
2098 while (ctrlq->ctrlq_inuse != DONE)
2099 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2100 mutex_exit(&ctrlq->ctrlq_wait_lock);
2101 /* already dequeueued */
2102
2103 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
2104 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
2105 for (i = 0; i < nspecs; i++) {
2106 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
2107 specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
2108 }
2109 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
2110 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
2111
2112 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
2113 r = 0;
2114 else {
2115 device_printf(sc->sc_dev, "failed setting rx mode\n");
2116 sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
2117 r = EIO;
2118 }
2119
2120 return r;
2121 }
2122
2123 static int
2124 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
2125 {
2126 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
2127 struct vioif_ctrl_cmdspec specs[1];
2128 int r;
2129
2130 if (!sc->sc_has_ctrl)
2131 return ENOTSUP;
2132
2133 vioif_ctrl_acquire(sc);
2134
2135 rx->onoff = onoff;
2136 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
2137 specs[0].buf = rx;
2138 specs[0].bufsize = sizeof(*rx);
2139
2140 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
2141 specs, __arraycount(specs));
2142
2143 vioif_ctrl_release(sc);
2144 return r;
2145 }
2146
2147 static int
2148 vioif_set_promisc(struct vioif_softc *sc, bool onoff)
2149 {
2150 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
2151 }
2152
2153 static int
2154 vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
2155 {
2156 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
2157 }
2158
2159 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
2160 static int
2161 vioif_set_rx_filter(struct vioif_softc *sc)
2162 {
2163 /* filter already set in ctrlq->ctrlq_mac_tbl */
2164 struct virtio_softc *vsc = sc->sc_virtio;
2165 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
2166 struct vioif_ctrl_cmdspec specs[2];
2167 int nspecs = __arraycount(specs);
2168 int r;
2169
2170 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
2171 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
2172
2173 if (!sc->sc_has_ctrl)
2174 return ENOTSUP;
2175
2176 vioif_ctrl_acquire(sc);
2177
2178 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
2179 specs[0].buf = mac_tbl_uc;
2180 specs[0].bufsize = sizeof(*mac_tbl_uc)
2181 + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
2182
2183 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
2184 specs[1].buf = mac_tbl_mc;
2185 specs[1].bufsize = sizeof(*mac_tbl_mc)
2186 + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
2187
2188 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
2189 if (r != 0)
2190 goto out;
2191
2192 r = vioif_ctrl_send_command(sc,
2193 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
2194 specs, nspecs);
2195
2196 vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
2197
2198 out:
2199 vioif_ctrl_release(sc);
2200
2201 return r;
2202 }
2203
2204 static int
2205 vioif_set_mac_addr(struct vioif_softc *sc)
2206 {
2207 struct virtio_net_ctrl_mac_addr *ma =
2208 sc->sc_ctrlq.ctrlq_mac_addr;
2209 struct vioif_ctrl_cmdspec specs[1];
2210 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2211 int nspecs = __arraycount(specs);
2212 uint64_t features;
2213 int r;
2214 size_t i;
2215
2216 if (!sc->sc_has_ctrl)
2217 return ENOTSUP;
2218
2219 if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
2220 ETHER_ADDR_LEN) == 0) {
2221 return 0;
2222 }
2223
2224 memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2225
2226 features = virtio_features(sc->sc_virtio);
2227 if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
2228 vioif_ctrl_acquire(sc);
2229
2230 memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
2231 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
2232 specs[0].buf = ma;
2233 specs[0].bufsize = sizeof(*ma);
2234
2235 r = vioif_ctrl_send_command(sc,
2236 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
2237 specs, nspecs);
2238
2239 vioif_ctrl_release(sc);
2240 } else {
2241 for (i = 0; i < __arraycount(sc->sc_mac); i++) {
2242 virtio_write_device_config_1(sc->sc_virtio,
2243 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
2244 }
2245 r = 0;
2246 }
2247
2248 return r;
2249 }
2250
2251 static int
2252 vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
2253 {
2254 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
2255 struct vioif_ctrl_cmdspec specs[1];
2256 int r;
2257
2258 if (!sc->sc_has_ctrl)
2259 return ENOTSUP;
2260
2261 if (nvq_pairs <= 1)
2262 return EINVAL;
2263
2264 vioif_ctrl_acquire(sc);
2265
2266 mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
2267 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
2268 specs[0].buf = mq;
2269 specs[0].bufsize = sizeof(*mq);
2270
2271 r = vioif_ctrl_send_command(sc,
2272 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
2273 specs, __arraycount(specs));
2274
2275 vioif_ctrl_release(sc);
2276
2277 return r;
2278 }
2279
2280 /* ctrl vq interrupt; wake up the command issuer */
2281 static int
2282 vioif_ctrl_intr(void *arg)
2283 {
2284 struct vioif_ctrlqueue *ctrlq = arg;
2285 struct virtqueue *vq = ctrlq->ctrlq_vq;
2286 struct virtio_softc *vsc = vq->vq_owner;
2287 int r, slot;
2288
2289 if (virtio_vq_is_enqueued(vsc, vq) == false)
2290 return 0;
2291
2292 r = virtio_dequeue(vsc, vq, &slot, NULL);
2293 if (r == ENOENT)
2294 return 0;
2295 virtio_dequeue_commit(vsc, vq, slot);
2296
2297 mutex_enter(&ctrlq->ctrlq_wait_lock);
2298 ctrlq->ctrlq_inuse = DONE;
2299 cv_signal(&ctrlq->ctrlq_wait);
2300 mutex_exit(&ctrlq->ctrlq_wait_lock);
2301
2302 return 1;
2303 }
2304
2305 static int
2306 vioif_ifflags(struct vioif_softc *sc)
2307 {
2308 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2309 bool onoff;
2310 int r;
2311
2312 if (!sc->sc_has_ctrl) {
2313 /* no ctrl vq; always promisc and allmulti */
2314 ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
2315 return 0;
2316 }
2317
2318 onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
2319 r = vioif_set_allmulti(sc, onoff);
2320 if (r != 0) {
2321 log(LOG_WARNING,
2322 "%s: couldn't %sable ALLMULTI\n",
2323 ifp->if_xname, onoff ? "en" : "dis");
2324 if (onoff == false) {
2325 ifp->if_flags |= IFF_ALLMULTI;
2326 }
2327 }
2328
2329 onoff = ifp->if_flags & IFF_PROMISC ? true : false;
2330 r = vioif_set_promisc(sc, onoff);
2331 if (r != 0) {
2332 log(LOG_WARNING,
2333 "%s: couldn't %sable PROMISC\n",
2334 ifp->if_xname, onoff ? "en" : "dis");
2335 if (onoff == false) {
2336 ifp->if_flags |= IFF_PROMISC;
2337 }
2338 }
2339
2340 return 0;
2341 }
2342
2343 static int
2344 vioif_ifflags_cb(struct ethercom *ec)
2345 {
2346 struct ifnet *ifp = &ec->ec_if;
2347 struct vioif_softc *sc = ifp->if_softc;
2348
2349 return vioif_ifflags(sc);
2350 }
2351
2352 /*
2353 * If multicast filter small enough (<=MAXENTRIES) set rx filter
2354 * If large multicast filter exist use ALLMULTI
2355 * If setting rx filter fails fall back to ALLMULTI
2356 */
2357 static int
2358 vioif_rx_filter(struct vioif_softc *sc)
2359 {
2360 struct virtio_softc *vsc = sc->sc_virtio;
2361 struct ethercom *ec = &sc->sc_ethercom;
2362 struct ifnet *ifp = &ec->ec_if;
2363 struct ether_multi *enm;
2364 struct ether_multistep step;
2365 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2366 int nentries;
2367 bool allmulti = 0;
2368 int r;
2369
2370 if (!sc->sc_has_ctrl) {
2371 goto set_ifflags;
2372 }
2373
2374 memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
2375 CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2376
2377 nentries = 0;
2378 allmulti = false;
2379
2380 ETHER_LOCK(ec);
2381 for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
2382 ETHER_NEXT_MULTI(step, enm)) {
2383 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
2384 allmulti = true;
2385 break;
2386 }
2387 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2388 allmulti = true;
2389 break;
2390 }
2391
2392 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
2393 enm->enm_addrlo, ETHER_ADDR_LEN);
2394 nentries++;
2395 }
2396 ETHER_UNLOCK(ec);
2397
2398 r = vioif_set_mac_addr(sc);
2399 if (r != 0) {
2400 log(LOG_WARNING, "%s: couldn't set MAC address\n",
2401 ifp->if_xname);
2402 }
2403
2404 if (!allmulti) {
2405 ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
2406 ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
2407 r = vioif_set_rx_filter(sc);
2408 if (r != 0) {
2409 allmulti = true; /* fallback */
2410 }
2411 }
2412
2413 if (allmulti) {
2414 ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
2415 ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
2416 r = vioif_set_rx_filter(sc);
2417 if (r != 0) {
2418 log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
2419 ifp->if_xname);
2420 /* what to do on failure? */
2421 }
2422
2423 ifp->if_flags |= IFF_ALLMULTI;
2424 }
2425
2426 set_ifflags:
2427 r = vioif_ifflags(sc);
2428
2429 return r;
2430 }
2431
2432 static int
2433 vioif_get_link_status(struct vioif_softc *sc)
2434 {
2435 struct virtio_softc *vsc = sc->sc_virtio;
2436 uint16_t status;
2437
2438 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2439 status = virtio_read_device_config_2(vsc,
2440 VIRTIO_NET_CONFIG_STATUS);
2441 else
2442 status = VIRTIO_NET_S_LINK_UP;
2443
2444 if ((status & VIRTIO_NET_S_LINK_UP) != 0)
2445 return LINK_STATE_UP;
2446
2447 return LINK_STATE_DOWN;
2448 }
2449
2450 /* change link status */
2451 static void
2452 vioif_update_link_status(struct vioif_softc *sc)
2453 {
2454 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2455 struct vioif_txqueue *txq;
2456 bool active;
2457 int link, i;
2458
2459 mutex_enter(&sc->sc_lock);
2460
2461 link = vioif_get_link_status(sc);
2462
2463 if (link == sc->sc_link_state)
2464 goto done;
2465
2466 sc->sc_link_state = link;
2467
2468 active = VIOIF_IS_LINK_ACTIVE(sc);
2469 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2470 txq = &sc->sc_txq[i];
2471
2472 mutex_enter(txq->txq_lock);
2473 txq->txq_link_active = active;
2474 mutex_exit(txq->txq_lock);
2475 }
2476
2477 if_link_state_change(ifp, sc->sc_link_state);
2478
2479 done:
2480 mutex_exit(&sc->sc_lock);
2481 }
2482
2483 static int
2484 vioif_config_change(struct virtio_softc *vsc)
2485 {
2486 struct vioif_softc *sc = device_private(virtio_child(vsc));
2487
2488 softint_schedule(sc->sc_ctl_softint);
2489 return 0;
2490 }
2491
2492 static void
2493 vioif_ctl_softint(void *arg)
2494 {
2495 struct vioif_softc *sc = arg;
2496 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2497
2498 vioif_update_link_status(sc);
2499 vioif_start(ifp);
2500 }
2501
2502 static struct workqueue *
2503 vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
2504 {
2505 struct workqueue *wq;
2506 int error;
2507
2508 error = workqueue_create(&wq, name, vioif_workq_work, NULL,
2509 prio, ipl, flags);
2510
2511 if (error)
2512 return NULL;
2513
2514 return wq;
2515 }
2516
2517 static void
2518 vioif_workq_destroy(struct workqueue *wq)
2519 {
2520
2521 workqueue_destroy(wq);
2522 }
2523
2524 static void
2525 vioif_workq_work(struct work *wk, void *context)
2526 {
2527 struct vioif_work *work;
2528
2529 work = container_of(wk, struct vioif_work, cookie);
2530
2531 atomic_store_relaxed(&work->added, 0);
2532 work->func(work->arg);
2533 }
2534
2535 static void
2536 vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
2537 {
2538
2539 memset(work, 0, sizeof(*work));
2540 work->func = func;
2541 work->arg = arg;
2542 }
2543
2544 static void
2545 vioif_work_add(struct workqueue *wq, struct vioif_work *work)
2546 {
2547
2548 if (atomic_load_relaxed(&work->added) != 0)
2549 return;
2550
2551 atomic_store_relaxed(&work->added, 1);
2552 kpreempt_disable();
2553 workqueue_enqueue(wq, &work->cookie, NULL);
2554 kpreempt_enable();
2555 }
2556
2557 static void
2558 vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
2559 {
2560
2561 workqueue_wait(wq, &work->cookie);
2562 }
2563
2564 static int
2565 vioif_setup_sysctl(struct vioif_softc *sc)
2566 {
2567 const char *devname;
2568 struct sysctllog **log;
2569 const struct sysctlnode *rnode, *rxnode, *txnode;
2570 int error;
2571
2572 log = &sc->sc_sysctllog;
2573 devname = device_xname(sc->sc_dev);
2574
2575 error = sysctl_createv(log, 0, NULL, &rnode,
2576 0, CTLTYPE_NODE, devname,
2577 SYSCTL_DESCR("virtio-net information and settings"),
2578 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
2579 if (error)
2580 goto out;
2581
2582 error = sysctl_createv(log, 0, &rnode, NULL,
2583 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
2584 SYSCTL_DESCR("Use workqueue for packet processing"),
2585 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
2586 if (error)
2587 goto out;
2588
2589 error = sysctl_createv(log, 0, &rnode, &rxnode,
2590 0, CTLTYPE_NODE, "rx",
2591 SYSCTL_DESCR("virtio-net information and settings for Rx"),
2592 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2593 if (error)
2594 goto out;
2595
2596 error = sysctl_createv(log, 0, &rxnode, NULL,
2597 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2598 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
2599 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2600 if (error)
2601 goto out;
2602
2603 error = sysctl_createv(log, 0, &rxnode, NULL,
2604 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2605 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
2606 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
2607 if (error)
2608 goto out;
2609
2610 error = sysctl_createv(log, 0, &rnode, &txnode,
2611 0, CTLTYPE_NODE, "tx",
2612 SYSCTL_DESCR("virtio-net information and settings for Tx"),
2613 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2614 if (error)
2615 goto out;
2616
2617 error = sysctl_createv(log, 0, &txnode, NULL,
2618 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2619 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
2620 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2621 if (error)
2622 goto out;
2623
2624 error = sysctl_createv(log, 0, &txnode, NULL,
2625 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2626 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
2627 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
2628
2629 out:
2630 if (error)
2631 sysctl_teardown(log);
2632
2633 return error;
2634 }
2635
2636 static void
2637 vioif_setup_stats(struct vioif_softc *sc)
2638 {
2639 struct vioif_rxqueue *rxq;
2640 struct vioif_txqueue *txq;
2641 int i;
2642
2643 for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
2644 rxq = &sc->sc_rxq[i];
2645 txq = &sc->sc_txq[i];
2646
2647 snprintf(txq->txq_evgroup, sizeof(txq->txq_evgroup), "%s-TX%d",
2648 device_xname(sc->sc_dev), i);
2649 evcnt_attach_dynamic(&txq->txq_defrag_failed, EVCNT_TYPE_MISC,
2650 NULL, txq->txq_evgroup, "tx m_defrag() failed");
2651 evcnt_attach_dynamic(&txq->txq_mbuf_load_failed, EVCNT_TYPE_MISC,
2652 NULL, txq->txq_evgroup, "tx dmamap load failed");
2653 evcnt_attach_dynamic(&txq->txq_enqueue_reserve_failed, EVCNT_TYPE_MISC,
2654 NULL, txq->txq_evgroup, "virtio_enqueue_reserve failed");
2655
2656 snprintf(rxq->rxq_evgroup, sizeof(rxq->rxq_evgroup), "%s-RX%d",
2657 device_xname(sc->sc_dev), i);
2658 evcnt_attach_dynamic(&rxq->rxq_mbuf_add_failed, EVCNT_TYPE_MISC,
2659 NULL, rxq->rxq_evgroup, "rx mbuf allocation failed");
2660 }
2661
2662 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
2663 NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
2664 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
2665 NULL, device_xname(sc->sc_dev), "control command failed");
2666 }
2667
2668 static void
2669 vioif_intr_barrier(void)
2670 {
2671
2672 /* wait for finish all interrupt handler */
2673 xc_barrier(0);
2674 }
2675
2676 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
2677
2678 #ifdef _MODULE
2679 #include "ioconf.c"
2680 #endif
2681
2682 static int
2683 if_vioif_modcmd(modcmd_t cmd, void *opaque)
2684 {
2685 int error = 0;
2686
2687 #ifdef _MODULE
2688 switch (cmd) {
2689 case MODULE_CMD_INIT:
2690 error = config_init_component(cfdriver_ioconf_if_vioif,
2691 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2692 break;
2693 case MODULE_CMD_FINI:
2694 error = config_fini_component(cfdriver_ioconf_if_vioif,
2695 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2696 break;
2697 default:
2698 error = ENOTTY;
2699 break;
2700 }
2701 #endif
2702
2703 return error;
2704 }
2705