if_vioif.c revision 1.55 1 /* $NetBSD: if_vioif.c,v 1.55 2020/05/25 08:16:23 yamaguchi Exp $ */
2
3 /*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.55 2020/05/25 08:16:23 yamaguchi Exp $");
30
31 #ifdef _KERNEL_OPT
32 #include "opt_net_mpsafe.h"
33 #endif
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/atomic.h>
39 #include <sys/bus.h>
40 #include <sys/condvar.h>
41 #include <sys/device.h>
42 #include <sys/intr.h>
43 #include <sys/kmem.h>
44 #include <sys/mbuf.h>
45 #include <sys/mutex.h>
46 #include <sys/sockio.h>
47 #include <sys/cpu.h>
48 #include <sys/module.h>
49 #include <sys/pcq.h>
50 #include <sys/workqueue.h>
51
52 #include <dev/pci/virtioreg.h>
53 #include <dev/pci/virtiovar.h>
54
55 #include <net/if.h>
56 #include <net/if_media.h>
57 #include <net/if_ether.h>
58
59 #include <net/bpf.h>
60
61 #include "ioconf.h"
62
63 #ifdef NET_MPSAFE
64 #define VIOIF_MPSAFE 1
65 #define VIOIF_MULTIQ 1
66 #endif
67
68 /*
69 * if_vioifreg.h:
70 */
71 /* Configuration registers */
72 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
73 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
74 #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
75
76 /* Feature bits */
77 #define VIRTIO_NET_F_CSUM __BIT(0)
78 #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
79 #define VIRTIO_NET_F_MAC __BIT(5)
80 #define VIRTIO_NET_F_GSO __BIT(6)
81 #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
82 #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
83 #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
84 #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
85 #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
86 #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
87 #define VIRTIO_NET_F_HOST_ECN __BIT(13)
88 #define VIRTIO_NET_F_HOST_UFO __BIT(14)
89 #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
90 #define VIRTIO_NET_F_STATUS __BIT(16)
91 #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
92 #define VIRTIO_NET_F_CTRL_RX __BIT(18)
93 #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
94 #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
95 #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
96 #define VIRTIO_NET_F_MQ __BIT(22)
97
98 #define VIRTIO_NET_FLAG_BITS \
99 VIRTIO_COMMON_FLAG_BITS \
100 "\x17""MQ" \
101 "\x16""GUEST_ANNOUNCE" \
102 "\x15""CTRL_RX_EXTRA" \
103 "\x14""CTRL_VLAN" \
104 "\x13""CTRL_RX" \
105 "\x12""CTRL_VQ" \
106 "\x11""STATUS" \
107 "\x10""MRG_RXBUF" \
108 "\x0f""HOST_UFO" \
109 "\x0e""HOST_ECN" \
110 "\x0d""HOST_TSO6" \
111 "\x0c""HOST_TSO4" \
112 "\x0b""GUEST_UFO" \
113 "\x0a""GUEST_ECN" \
114 "\x09""GUEST_TSO6" \
115 "\x08""GUEST_TSO4" \
116 "\x07""GSO" \
117 "\x06""MAC" \
118 "\x02""GUEST_CSUM" \
119 "\x01""CSUM"
120
121 /* Status */
122 #define VIRTIO_NET_S_LINK_UP 1
123
124 /* Packet header structure */
125 struct virtio_net_hdr {
126 uint8_t flags;
127 uint8_t gso_type;
128 uint16_t hdr_len;
129 uint16_t gso_size;
130 uint16_t csum_start;
131 uint16_t csum_offset;
132 #if 0
133 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */
134 #endif
135 } __packed;
136
137 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
138 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
139 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
140 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
141 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
142 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
143
144 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
145
146 /* Control virtqueue */
147 struct virtio_net_ctrl_cmd {
148 uint8_t class;
149 uint8_t command;
150 } __packed;
151 #define VIRTIO_NET_CTRL_RX 0
152 # define VIRTIO_NET_CTRL_RX_PROMISC 0
153 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
154
155 #define VIRTIO_NET_CTRL_MAC 1
156 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
157
158 #define VIRTIO_NET_CTRL_VLAN 2
159 # define VIRTIO_NET_CTRL_VLAN_ADD 0
160 # define VIRTIO_NET_CTRL_VLAN_DEL 1
161
162 #define VIRTIO_NET_CTRL_MQ 4
163 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
164 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
165 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
166
167 struct virtio_net_ctrl_status {
168 uint8_t ack;
169 } __packed;
170 #define VIRTIO_NET_OK 0
171 #define VIRTIO_NET_ERR 1
172
173 struct virtio_net_ctrl_rx {
174 uint8_t onoff;
175 } __packed;
176
177 struct virtio_net_ctrl_mac_tbl {
178 uint32_t nentries;
179 uint8_t macs[][ETHER_ADDR_LEN];
180 } __packed;
181
182 struct virtio_net_ctrl_vlan {
183 uint16_t id;
184 } __packed;
185
186 struct virtio_net_ctrl_mq {
187 uint16_t virtqueue_pairs;
188 } __packed;
189
190 struct vioif_ctrl_cmdspec {
191 bus_dmamap_t dmamap;
192 void *buf;
193 bus_size_t bufsize;
194 };
195
196 /*
197 * if_vioifvar.h:
198 */
199
200 /*
201 * Locking notes:
202 * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
203 * a filds in vioif_rxqueue is protected by rxq_lock (a spin mutex).
204 * - more than one lock cannot be held at onece
205 * + ctrlq_inuse is protected by ctrlq_wait_lock.
206 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
207 * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
208 */
209
210 struct vioif_work {
211 struct work cookie;
212 void (*func)(void *);
213 void *arg;
214 unsigned int added;
215 };
216
217 struct vioif_txqueue {
218 kmutex_t *txq_lock; /* lock for tx operations */
219
220 struct virtqueue *txq_vq;
221 bool txq_stopping;
222 bool txq_link_active;
223 pcq_t *txq_intrq;
224
225 struct virtio_net_hdr *txq_hdrs;
226 bus_dmamap_t *txq_hdr_dmamaps;
227
228 struct mbuf **txq_mbufs;
229 bus_dmamap_t *txq_dmamaps;
230
231 void *txq_deferred_transmit;
232 void *txq_handle_si;
233 struct vioif_work txq_work;
234 bool txq_workqueue;
235 bool txq_active;
236 };
237
238 struct vioif_rxqueue {
239 kmutex_t *rxq_lock; /* lock for rx operations */
240
241 struct virtqueue *rxq_vq;
242 bool rxq_stopping;
243
244 struct virtio_net_hdr *rxq_hdrs;
245 bus_dmamap_t *rxq_hdr_dmamaps;
246
247 struct mbuf **rxq_mbufs;
248 bus_dmamap_t *rxq_dmamaps;
249
250 void *rxq_softint;
251 void *rxq_handle_si;
252 struct vioif_work rxq_work;
253 bool rxq_workqueue;
254 bool rxq_active;
255 };
256
257 struct vioif_ctrlqueue {
258 struct virtqueue *ctrlq_vq;
259 enum {
260 FREE, INUSE, DONE
261 } ctrlq_inuse;
262 kcondvar_t ctrlq_wait;
263 kmutex_t ctrlq_wait_lock;
264 struct lwp *ctrlq_owner;
265
266 struct virtio_net_ctrl_cmd *ctrlq_cmd;
267 struct virtio_net_ctrl_status *ctrlq_status;
268 struct virtio_net_ctrl_rx *ctrlq_rx;
269 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
270 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
271 struct virtio_net_ctrl_mq *ctrlq_mq;
272
273 bus_dmamap_t ctrlq_cmd_dmamap;
274 bus_dmamap_t ctrlq_status_dmamap;
275 bus_dmamap_t ctrlq_rx_dmamap;
276 bus_dmamap_t ctrlq_tbl_uc_dmamap;
277 bus_dmamap_t ctrlq_tbl_mc_dmamap;
278 bus_dmamap_t ctrlq_mq_dmamap;
279 };
280
281 struct vioif_softc {
282 device_t sc_dev;
283 struct sysctllog *sc_sysctllog;
284
285 struct virtio_softc *sc_virtio;
286 struct virtqueue *sc_vqs;
287
288 int sc_max_nvq_pairs;
289 int sc_req_nvq_pairs;
290 int sc_act_nvq_pairs;
291
292 uint8_t sc_mac[ETHER_ADDR_LEN];
293 struct ethercom sc_ethercom;
294 short sc_deferred_init_done;
295 bool sc_link_active;
296
297 struct vioif_txqueue *sc_txq;
298 struct vioif_rxqueue *sc_rxq;
299
300 bool sc_has_ctrl;
301 struct vioif_ctrlqueue sc_ctrlq;
302
303 bus_dma_segment_t sc_hdr_segs[1];
304 void *sc_dmamem;
305 void *sc_kmem;
306
307 void *sc_ctl_softint;
308
309 struct workqueue *sc_txrx_workqueue;
310 bool sc_txrx_workqueue_sysctl;
311 u_int sc_tx_intr_process_limit;
312 u_int sc_tx_process_limit;
313 u_int sc_rx_intr_process_limit;
314 u_int sc_rx_process_limit;
315 };
316 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
317 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
318
319 #define VIOIF_TX_INTR_PROCESS_LIMIT 256
320 #define VIOIF_TX_PROCESS_LIMIT 256
321 #define VIOIF_RX_INTR_PROCESS_LIMIT 0U
322 #define VIOIF_RX_PROCESS_LIMIT 256
323
324 #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET
325
326 /* cfattach interface functions */
327 static int vioif_match(device_t, cfdata_t, void *);
328 static void vioif_attach(device_t, device_t, void *);
329 static void vioif_deferred_init(device_t);
330 static int vioif_finalize_teardown(device_t);
331
332 /* ifnet interface functions */
333 static int vioif_init(struct ifnet *);
334 static void vioif_stop(struct ifnet *, int);
335 static void vioif_start(struct ifnet *);
336 static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
337 static int vioif_transmit(struct ifnet *, struct mbuf *);
338 static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
339 static int vioif_ioctl(struct ifnet *, u_long, void *);
340 static void vioif_watchdog(struct ifnet *);
341
342 /* rx */
343 static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
344 static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
345 static void vioif_populate_rx_mbufs(struct vioif_rxqueue *);
346 static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *);
347 static void vioif_rx_queue_clear(struct vioif_rxqueue *);
348 static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
349 struct vioif_rxqueue *, u_int);
350 static int vioif_rx_intr(void *);
351 static void vioif_rx_handle(void *);
352 static void vioif_rx_sched_handle(struct vioif_softc *,
353 struct vioif_rxqueue *);
354 static void vioif_rx_softint(void *);
355 static void vioif_rx_drain(struct vioif_rxqueue *);
356
357 /* tx */
358 static int vioif_tx_intr(void *);
359 static void vioif_tx_handle(void *);
360 static void vioif_tx_sched_handle(struct vioif_softc *,
361 struct vioif_txqueue *);
362 static void vioif_tx_queue_clear(struct vioif_txqueue *);
363 static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
364 struct vioif_txqueue *, u_int);
365 static void vioif_tx_drain(struct vioif_txqueue *);
366 static void vioif_deferred_transmit(void *);
367
368 /* workqueue */
369 static struct workqueue*
370 vioif_workq_create(const char *, pri_t, int, int);
371 static void vioif_workq_destroy(struct workqueue *);
372 static void vioif_workq_work(struct work *, void *);
373 static void vioif_work_set(struct vioif_work *, void(*)(void *), void *);
374 static void vioif_work_add(struct workqueue *, struct vioif_work *);
375 static void vioif_work_wait(struct workqueue *, struct vioif_work *);
376
377 /* other control */
378 static bool vioif_is_link_up(struct vioif_softc *);
379 static void vioif_update_link_status(struct vioif_softc *);
380 static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
381 static int vioif_set_promisc(struct vioif_softc *, bool);
382 static int vioif_set_allmulti(struct vioif_softc *, bool);
383 static int vioif_set_rx_filter(struct vioif_softc *);
384 static int vioif_rx_filter(struct vioif_softc *);
385 static int vioif_ctrl_intr(void *);
386 static int vioif_config_change(struct virtio_softc *);
387 static void vioif_ctl_softint(void *);
388 static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
389 static void vioif_enable_interrupt_vqpairs(struct vioif_softc *);
390 static void vioif_disable_interrupt_vqpairs(struct vioif_softc *);
391 static int vioif_setup_sysctl(struct vioif_softc *);
392
393 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
394 vioif_match, vioif_attach, NULL, NULL);
395
396 static int
397 vioif_match(device_t parent, cfdata_t match, void *aux)
398 {
399 struct virtio_attach_args *va = aux;
400
401 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK)
402 return 1;
403
404 return 0;
405 }
406
407 static void
408 vioif_alloc_queues(struct vioif_softc *sc)
409 {
410 int nvq_pairs = sc->sc_max_nvq_pairs;
411 int nvqs = nvq_pairs * 2;
412 int i;
413
414 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
415
416 sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
417 KM_SLEEP);
418 sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
419 KM_SLEEP);
420
421 if (sc->sc_has_ctrl)
422 nvqs++;
423
424 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
425 nvqs = 0;
426 for (i = 0; i < nvq_pairs; i++) {
427 sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
428 sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
429 }
430
431 if (sc->sc_has_ctrl)
432 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
433 }
434
435 static void
436 vioif_free_queues(struct vioif_softc *sc)
437 {
438 int nvq_pairs = sc->sc_max_nvq_pairs;
439 int nvqs = nvq_pairs * 2;
440
441 if (sc->sc_ctrlq.ctrlq_vq)
442 nvqs++;
443
444 if (sc->sc_txq) {
445 kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
446 sc->sc_txq = NULL;
447 }
448
449 if (sc->sc_rxq) {
450 kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
451 sc->sc_rxq = NULL;
452 }
453
454 if (sc->sc_vqs) {
455 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
456 sc->sc_vqs = NULL;
457 }
458 }
459
460 /* allocate memory */
461 /*
462 * dma memory is used for:
463 * rxq_hdrs[slot]: metadata array for received frames (READ)
464 * txq_hdrs[slot]: metadata array for frames to be sent (WRITE)
465 * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
466 * ctrlq_status: return value for a command via ctrl vq (READ)
467 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
468 * (WRITE)
469 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
470 * class command (WRITE)
471 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
472 * class command (WRITE)
473 * ctrlq_* structures are allocated only one each; they are protected by
474 * ctrlq_inuse variable and ctrlq_wait condvar.
475 */
476 /*
477 * dynamically allocated memory is used for:
478 * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot]
479 * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot]
480 * rxq_dmamaps[slot]: bus_dmamap_t array for received payload
481 * txq_dmamaps[slot]: bus_dmamap_t array for sent payload
482 * rxq_mbufs[slot]: mbuf pointer array for received frames
483 * txq_mbufs[slot]: mbuf pointer array for sent frames
484 */
485 static int
486 vioif_alloc_mems(struct vioif_softc *sc)
487 {
488 struct virtio_softc *vsc = sc->sc_virtio;
489 struct vioif_txqueue *txq;
490 struct vioif_rxqueue *rxq;
491 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
492 int allocsize, allocsize2, r, rsegs, i, qid;
493 void *vaddr;
494 intptr_t p;
495
496 allocsize = 0;
497 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
498 rxq = &sc->sc_rxq[qid];
499 txq = &sc->sc_txq[qid];
500
501 allocsize +=
502 sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num;
503 allocsize +=
504 sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num;
505 }
506 if (sc->sc_has_ctrl) {
507 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
508 allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
509 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
510 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
511 + sizeof(struct virtio_net_ctrl_mac_tbl)
512 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
513 allocsize += sizeof(struct virtio_net_ctrl_mq) * 1;
514 }
515 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
516 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
517 if (r != 0) {
518 aprint_error_dev(sc->sc_dev,
519 "DMA memory allocation failed, size %d, "
520 "error code %d\n", allocsize, r);
521 goto err_none;
522 }
523 r = bus_dmamem_map(virtio_dmat(vsc),
524 &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT);
525 if (r != 0) {
526 aprint_error_dev(sc->sc_dev,
527 "DMA memory map failed, error code %d\n", r);
528 goto err_dmamem_alloc;
529 }
530
531 #define P(p, p0, p0size) do { p0 = (void *) p; \
532 p += p0size; } while (0)
533 memset(vaddr, 0, allocsize);
534 sc->sc_dmamem = vaddr;
535 p = (intptr_t) vaddr;
536
537 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
538 rxq = &sc->sc_rxq[qid];
539 txq = &sc->sc_txq[qid];
540
541 P(p, rxq->rxq_hdrs,
542 sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num);
543 P(p, txq->txq_hdrs,
544 sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num);
545 }
546 if (sc->sc_has_ctrl) {
547 P(p, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd));
548 P(p, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status));
549 P(p, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx));
550 P(p, ctrlq->ctrlq_mac_tbl_uc, sizeof(*ctrlq->ctrlq_mac_tbl_uc));
551 P(p, ctrlq->ctrlq_mac_tbl_mc, sizeof(*ctrlq->ctrlq_mac_tbl_mc)
552 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
553 P(p, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq));
554 }
555
556 allocsize2 = 0;
557 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
558 int rxqsize, txqsize;
559
560 rxq = &sc->sc_rxq[qid];
561 txq = &sc->sc_txq[qid];
562 rxqsize = rxq->rxq_vq->vq_num;
563 txqsize = txq->txq_vq->vq_num;
564
565 allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
566 allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
567 allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
568
569 allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
570 allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
571 allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
572 }
573 vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
574 sc->sc_kmem = vaddr;
575 p = (intptr_t) vaddr;
576
577 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
578 int rxqsize, txqsize;
579 rxq = &sc->sc_rxq[qid];
580 txq = &sc->sc_txq[qid];
581 rxqsize = rxq->rxq_vq->vq_num;
582 txqsize = txq->txq_vq->vq_num;
583
584 P(p, rxq->rxq_hdr_dmamaps,
585 sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
586 P(p, txq->txq_hdr_dmamaps,
587 sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
588 P(p, rxq->rxq_dmamaps, sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
589 P(p, txq->txq_dmamaps, sizeof(txq->txq_dmamaps[0]) * txqsize);
590 P(p, rxq->rxq_mbufs, sizeof(rxq->rxq_mbufs[0]) * rxqsize);
591 P(p, txq->txq_mbufs, sizeof(txq->txq_mbufs[0]) * txqsize);
592 }
593 #undef P
594
595 #define C(map, size, nsegs, usage) \
596 do { \
597 r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \
598 BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, \
599 &map); \
600 if (r != 0) { \
601 aprint_error_dev(sc->sc_dev, \
602 usage " dmamap creation failed, " \
603 "error code %d\n", r); \
604 goto err_reqs; \
605 } \
606 } while (0)
607 #define C_L(map, buf, size, nsegs, rw, usage) \
608 C(map, size, nsegs, usage); \
609 do { \
610 r = bus_dmamap_load(virtio_dmat(vsc), map, \
611 buf, size, NULL, \
612 rw | BUS_DMA_NOWAIT); \
613 if (r != 0) { \
614 aprint_error_dev(sc->sc_dev, \
615 usage " dmamap load failed, " \
616 "error code %d\n", r); \
617 goto err_reqs; \
618 } \
619 } while (0)
620
621 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
622 rxq = &sc->sc_rxq[qid];
623 txq = &sc->sc_txq[qid];
624
625 for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
626 C_L(rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i],
627 sizeof(rxq->rxq_hdrs[0]), 1,
628 BUS_DMA_READ, "rx header");
629 C(rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload");
630 }
631
632 for (i = 0; i < txq->txq_vq->vq_num; i++) {
633 C_L(txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i],
634 sizeof(txq->txq_hdrs[0]), 1,
635 BUS_DMA_READ, "tx header");
636 C(txq->txq_dmamaps[i], ETHER_MAX_LEN,
637 VIRTIO_NET_TX_MAXNSEGS, "tx payload");
638 }
639 }
640
641 if (sc->sc_has_ctrl) {
642 /* control vq class & command */
643 C_L(ctrlq->ctrlq_cmd_dmamap,
644 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
645 BUS_DMA_WRITE, "control command");
646 C_L(ctrlq->ctrlq_status_dmamap,
647 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
648 BUS_DMA_READ, "control status");
649
650 /* control vq rx mode command parameter */
651 C_L(ctrlq->ctrlq_rx_dmamap,
652 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
653 BUS_DMA_WRITE, "rx mode control command");
654
655 /* multiqueue set command */
656 C_L(ctrlq->ctrlq_mq_dmamap,
657 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
658 BUS_DMA_WRITE, "multiqueue set command");
659
660 /* control vq MAC filter table for unicast */
661 /* do not load now since its length is variable */
662 C(ctrlq->ctrlq_tbl_uc_dmamap,
663 sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1,
664 "unicast MAC address filter command");
665
666 /* control vq MAC filter table for multicast */
667 C(ctrlq->ctrlq_tbl_mc_dmamap,
668 sizeof(*ctrlq->ctrlq_mac_tbl_mc)
669 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
670 "multicast MAC address filter command");
671 }
672 #undef C_L
673 #undef C
674
675 return 0;
676
677 err_reqs:
678 #define D(map) \
679 do { \
680 if (map) { \
681 bus_dmamap_destroy(virtio_dmat(vsc), map); \
682 map = NULL; \
683 } \
684 } while (0)
685 D(ctrlq->ctrlq_tbl_mc_dmamap);
686 D(ctrlq->ctrlq_tbl_uc_dmamap);
687 D(ctrlq->ctrlq_rx_dmamap);
688 D(ctrlq->ctrlq_status_dmamap);
689 D(ctrlq->ctrlq_cmd_dmamap);
690 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
691 rxq = &sc->sc_rxq[qid];
692 txq = &sc->sc_txq[qid];
693
694 for (i = 0; i < txq->txq_vq->vq_num; i++) {
695 D(txq->txq_dmamaps[i]);
696 D(txq->txq_hdr_dmamaps[i]);
697 }
698 for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
699 D(rxq->rxq_dmamaps[i]);
700 D(rxq->rxq_hdr_dmamaps[i]);
701 }
702 }
703 #undef D
704 if (sc->sc_kmem) {
705 kmem_free(sc->sc_kmem, allocsize2);
706 sc->sc_kmem = NULL;
707 }
708 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
709 err_dmamem_alloc:
710 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
711 err_none:
712 return -1;
713 }
714
715 static void
716 vioif_attach(device_t parent, device_t self, void *aux)
717 {
718 struct vioif_softc *sc = device_private(self);
719 struct virtio_softc *vsc = device_private(parent);
720 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
721 struct vioif_txqueue *txq;
722 struct vioif_rxqueue *rxq;
723 uint32_t features, req_features;
724 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
725 u_int softint_flags;
726 int r, i, nvqs=0, req_flags;
727 char xnamebuf[MAXCOMLEN];
728
729 if (virtio_child(vsc) != NULL) {
730 aprint_normal(": child already attached for %s; "
731 "something wrong...\n", device_xname(parent));
732 return;
733 }
734
735 sc->sc_dev = self;
736 sc->sc_virtio = vsc;
737 sc->sc_link_active = false;
738
739 sc->sc_max_nvq_pairs = 1;
740 sc->sc_req_nvq_pairs = 1;
741 sc->sc_act_nvq_pairs = 1;
742 sc->sc_txrx_workqueue_sysctl = true;
743 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
744 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
745 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
746 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
747
748 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
749 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
750 IPL_NET, WQ_PERCPU | WQ_MPSAFE);
751 if (sc->sc_txrx_workqueue == NULL)
752 goto err;
753
754 req_flags = 0;
755
756 #ifdef VIOIF_MPSAFE
757 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE;
758 #endif
759 req_flags |= VIRTIO_F_PCI_INTR_MSIX;
760
761 req_features =
762 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
763 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
764 #ifdef VIOIF_MULTIQ
765 req_features |= VIRTIO_NET_F_MQ;
766 #endif
767 virtio_child_attach_start(vsc, self, IPL_NET, NULL,
768 vioif_config_change, virtio_vq_intrhand, req_flags,
769 req_features, VIRTIO_NET_FLAG_BITS);
770
771 features = virtio_features(vsc);
772
773 if (features & VIRTIO_NET_F_MAC) {
774 for (i = 0; i < __arraycount(sc->sc_mac); i++) {
775 sc->sc_mac[i] = virtio_read_device_config_1(vsc,
776 VIRTIO_NET_CONFIG_MAC + i);
777 }
778 } else {
779 /* code stolen from sys/net/if_tap.c */
780 struct timeval tv;
781 uint32_t ui;
782 getmicrouptime(&tv);
783 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
784 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
785 for (i = 0; i < __arraycount(sc->sc_mac); i++) {
786 virtio_write_device_config_1(vsc,
787 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
788 }
789 }
790
791 aprint_normal_dev(self, "Ethernet address %s\n",
792 ether_sprintf(sc->sc_mac));
793
794 if ((features & VIRTIO_NET_F_CTRL_VQ) &&
795 (features & VIRTIO_NET_F_CTRL_RX)) {
796 sc->sc_has_ctrl = true;
797
798 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
799 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
800 ctrlq->ctrlq_inuse = FREE;
801 } else {
802 sc->sc_has_ctrl = false;
803 }
804
805 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
806 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
807 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
808
809 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
810 goto err;
811
812 /* Limit the number of queue pairs to use */
813 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
814 }
815
816 vioif_alloc_queues(sc);
817 virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
818
819 #ifdef VIOIF_MPSAFE
820 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
821 #else
822 softint_flags = SOFTINT_NET;
823 #endif
824
825 /*
826 * Allocating a virtqueues
827 */
828 for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
829 rxq = &sc->sc_rxq[i];
830 txq = &sc->sc_txq[i];
831 char qname[32];
832
833 rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
834
835 rxq->rxq_softint = softint_establish(softint_flags,
836 vioif_rx_softint, rxq);
837 if (rxq->rxq_softint == NULL) {
838 aprint_error_dev(self, "cannot establish rx softint\n");
839 goto err;
840 }
841 rxq->rxq_handle_si = softint_establish(softint_flags,
842 vioif_rx_handle, rxq);
843 if (rxq->rxq_handle_si == NULL) {
844 aprint_error_dev(self, "cannot establish rx softint\n");
845 goto err;
846 }
847
848 snprintf(qname, sizeof(qname), "rx%d", i);
849 r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
850 MCLBYTES+sizeof(struct virtio_net_hdr), nvqs, qname);
851 if (r != 0)
852 goto err;
853 nvqs++;
854 rxq->rxq_vq->vq_intrhand = vioif_rx_intr;
855 rxq->rxq_vq->vq_intrhand_arg = (void *)rxq;
856 rxq->rxq_stopping = true;
857 vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq);
858
859 txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
860
861 txq->txq_deferred_transmit = softint_establish(softint_flags,
862 vioif_deferred_transmit, txq);
863 if (txq->txq_deferred_transmit == NULL) {
864 aprint_error_dev(self, "cannot establish tx softint\n");
865 goto err;
866 }
867 txq->txq_handle_si = softint_establish(softint_flags,
868 vioif_tx_handle, txq);
869 if (txq->txq_handle_si == NULL) {
870 aprint_error_dev(self, "cannot establish tx softint\n");
871 goto err;
872 }
873
874 snprintf(qname, sizeof(qname), "tx%d", i);
875 r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
876 sizeof(struct virtio_net_hdr)
877 + (ETHER_MAX_LEN - ETHER_HDR_LEN),
878 VIRTIO_NET_TX_MAXNSEGS + 1, qname);
879 if (r != 0)
880 goto err;
881 nvqs++;
882 txq->txq_vq->vq_intrhand = vioif_tx_intr;
883 txq->txq_vq->vq_intrhand_arg = (void *)txq;
884 txq->txq_link_active = sc->sc_link_active;
885 txq->txq_stopping = false;
886 txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP);
887 vioif_work_set(&txq->txq_work, vioif_tx_handle, txq);
888 }
889
890 if (sc->sc_has_ctrl) {
891 /*
892 * Allocating a virtqueue for control channel
893 */
894 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
895 NBPG, 1, "control");
896 if (r != 0) {
897 aprint_error_dev(self, "failed to allocate "
898 "a virtqueue for control channel, error code %d\n",
899 r);
900
901 sc->sc_has_ctrl = false;
902 cv_destroy(&ctrlq->ctrlq_wait);
903 mutex_destroy(&ctrlq->ctrlq_wait_lock);
904 } else {
905 nvqs++;
906 ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
907 ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
908 }
909 }
910
911 sc->sc_ctl_softint = softint_establish(softint_flags,
912 vioif_ctl_softint, sc);
913 if (sc->sc_ctl_softint == NULL) {
914 aprint_error_dev(self, "cannot establish ctl softint\n");
915 goto err;
916 }
917
918 if (vioif_alloc_mems(sc) < 0)
919 goto err;
920
921 if (virtio_child_attach_finish(vsc) != 0)
922 goto err;
923
924 if (vioif_setup_sysctl(sc) != 0) {
925 aprint_error_dev(self, "unable to create sysctl node\n");
926 /* continue */
927 }
928
929 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
930 ifp->if_softc = sc;
931 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
932 #ifdef VIOIF_MPSAFE
933 ifp->if_extflags = IFEF_MPSAFE;
934 #endif
935 ifp->if_start = vioif_start;
936 if (sc->sc_req_nvq_pairs > 1)
937 ifp->if_transmit = vioif_transmit;
938 ifp->if_ioctl = vioif_ioctl;
939 ifp->if_init = vioif_init;
940 ifp->if_stop = vioif_stop;
941 ifp->if_capabilities = 0;
942 ifp->if_watchdog = vioif_watchdog;
943 txq = &sc->sc_txq[0];
944 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
945 IFQ_SET_READY(&ifp->if_snd);
946
947 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
948
949 if_attach(ifp);
950 if_deferred_start_init(ifp, NULL);
951 ether_ifattach(ifp, sc->sc_mac);
952
953 return;
954
955 err:
956 for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
957 rxq = &sc->sc_rxq[i];
958 txq = &sc->sc_txq[i];
959
960 if (rxq->rxq_lock) {
961 mutex_obj_free(rxq->rxq_lock);
962 rxq->rxq_lock = NULL;
963 }
964
965 if (rxq->rxq_softint) {
966 softint_disestablish(rxq->rxq_softint);
967 rxq->rxq_softint = NULL;
968 }
969
970 if (rxq->rxq_handle_si) {
971 softint_disestablish(rxq->rxq_handle_si);
972 rxq->rxq_handle_si = NULL;
973 }
974
975 if (txq->txq_lock) {
976 mutex_obj_free(txq->txq_lock);
977 txq->txq_lock = NULL;
978 }
979
980 if (txq->txq_handle_si) {
981 softint_disestablish(txq->txq_handle_si);
982 txq->txq_handle_si = NULL;
983 }
984
985 if (txq->txq_deferred_transmit) {
986 softint_disestablish(txq->txq_deferred_transmit);
987 txq->txq_deferred_transmit = NULL;
988 }
989
990 if (txq->txq_intrq) {
991 pcq_destroy(txq->txq_intrq);
992 txq->txq_intrq = NULL;
993 }
994 }
995
996 if (sc->sc_has_ctrl) {
997 cv_destroy(&ctrlq->ctrlq_wait);
998 mutex_destroy(&ctrlq->ctrlq_wait_lock);
999 }
1000
1001 while (nvqs > 0)
1002 virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
1003
1004 vioif_free_queues(sc);
1005 virtio_child_attach_failed(vsc);
1006 config_finalize_register(self, vioif_finalize_teardown);
1007
1008 return;
1009 }
1010
1011 static int
1012 vioif_finalize_teardown(device_t self)
1013 {
1014 struct vioif_softc *sc = device_private(self);
1015
1016 if (sc->sc_txrx_workqueue != NULL) {
1017 vioif_workq_destroy(sc->sc_txrx_workqueue);
1018 sc->sc_txrx_workqueue = NULL;
1019 }
1020
1021 return 0;
1022 }
1023
1024 /* we need interrupts to make promiscuous mode off */
1025 static void
1026 vioif_deferred_init(device_t self)
1027 {
1028 struct vioif_softc *sc = device_private(self);
1029 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1030 int r;
1031
1032 if (ifp->if_flags & IFF_PROMISC)
1033 return;
1034
1035 r = vioif_set_promisc(sc, false);
1036 if (r != 0)
1037 aprint_error_dev(self, "resetting promisc mode failed, "
1038 "error code %d\n", r);
1039 }
1040
1041 static void
1042 vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
1043 {
1044 struct virtio_softc *vsc = sc->sc_virtio;
1045 struct vioif_txqueue *txq;
1046 struct vioif_rxqueue *rxq;
1047 int i;
1048
1049 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1050 txq = &sc->sc_txq[i];
1051 rxq = &sc->sc_rxq[i];
1052
1053 virtio_start_vq_intr(vsc, txq->txq_vq);
1054 virtio_start_vq_intr(vsc, rxq->rxq_vq);
1055 }
1056 }
1057
1058 static void
1059 vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
1060 {
1061 struct virtio_softc *vsc = sc->sc_virtio;
1062 struct vioif_txqueue *txq;
1063 struct vioif_rxqueue *rxq;
1064 int i;
1065
1066 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1067 txq = &sc->sc_txq[i];
1068 rxq = &sc->sc_rxq[i];
1069
1070 virtio_stop_vq_intr(vsc, txq->txq_vq);
1071 virtio_stop_vq_intr(vsc, rxq->rxq_vq);
1072 }
1073 }
1074
1075 /*
1076 * Interface functions for ifnet
1077 */
1078 static int
1079 vioif_init(struct ifnet *ifp)
1080 {
1081 struct vioif_softc *sc = ifp->if_softc;
1082 struct virtio_softc *vsc = sc->sc_virtio;
1083 struct vioif_rxqueue *rxq;
1084 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1085 int r, i;
1086
1087 vioif_stop(ifp, 0);
1088
1089 virtio_reinit_start(vsc);
1090 virtio_negotiate_features(vsc, virtio_features(vsc));
1091
1092 for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
1093 rxq = &sc->sc_rxq[i];
1094
1095 /* Have to set false before vioif_populate_rx_mbufs */
1096 rxq->rxq_stopping = false;
1097 vioif_populate_rx_mbufs(rxq);
1098 }
1099
1100 virtio_reinit_end(vsc);
1101
1102 if (sc->sc_has_ctrl)
1103 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
1104
1105 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
1106 if (r == 0)
1107 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
1108 else
1109 sc->sc_act_nvq_pairs = 1;
1110
1111 for (i = 0; i < sc->sc_act_nvq_pairs; i++)
1112 sc->sc_txq[i].txq_stopping = false;
1113
1114 vioif_enable_interrupt_vqpairs(sc);
1115
1116 if (!sc->sc_deferred_init_done) {
1117 sc->sc_deferred_init_done = 1;
1118 if (sc->sc_has_ctrl)
1119 vioif_deferred_init(sc->sc_dev);
1120 }
1121
1122 vioif_update_link_status(sc);
1123 ifp->if_flags |= IFF_RUNNING;
1124 ifp->if_flags &= ~IFF_OACTIVE;
1125 vioif_rx_filter(sc);
1126
1127 return 0;
1128 }
1129
1130 static void
1131 vioif_stop(struct ifnet *ifp, int disable)
1132 {
1133 struct vioif_softc *sc = ifp->if_softc;
1134 struct virtio_softc *vsc = sc->sc_virtio;
1135 struct vioif_txqueue *txq;
1136 struct vioif_rxqueue *rxq;
1137 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1138 int i;
1139
1140 /* Take the locks to ensure that ongoing TX/RX finish */
1141 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1142 txq = &sc->sc_txq[i];
1143 rxq = &sc->sc_rxq[i];
1144
1145 mutex_enter(txq->txq_lock);
1146 txq->txq_stopping = true;
1147 mutex_exit(txq->txq_lock);
1148
1149 mutex_enter(rxq->rxq_lock);
1150 rxq->rxq_stopping = true;
1151 mutex_exit(rxq->rxq_lock);
1152 }
1153
1154 /* disable interrupts */
1155 vioif_disable_interrupt_vqpairs(sc);
1156
1157 if (sc->sc_has_ctrl)
1158 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
1159
1160 /* only way to stop I/O and DMA is resetting... */
1161 virtio_reset(vsc);
1162
1163 /* rendezvous for finish of handlers */
1164 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1165 txq = &sc->sc_txq[i];
1166 rxq = &sc->sc_rxq[i];
1167
1168 mutex_enter(txq->txq_lock);
1169 mutex_exit(txq->txq_lock);
1170
1171 mutex_enter(rxq->rxq_lock);
1172 mutex_exit(rxq->rxq_lock);
1173
1174 vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work);
1175 vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work);
1176 }
1177
1178 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1179 vioif_rx_queue_clear(&sc->sc_rxq[i]);
1180 vioif_tx_queue_clear(&sc->sc_txq[i]);
1181 }
1182
1183 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1184 sc->sc_link_active = false;
1185
1186 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1187 txq = &sc->sc_txq[i];
1188 rxq = &sc->sc_rxq[i];
1189
1190 txq->txq_link_active = false;
1191
1192 if (disable)
1193 vioif_rx_drain(rxq);
1194
1195 vioif_tx_drain(txq);
1196 }
1197 }
1198
1199 static void
1200 vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq,
1201 bool is_transmit)
1202 {
1203 struct vioif_softc *sc = ifp->if_softc;
1204 struct virtio_softc *vsc = sc->sc_virtio;
1205 struct virtqueue *vq = txq->txq_vq;
1206 struct mbuf *m;
1207 int queued = 0;
1208
1209 KASSERT(mutex_owned(txq->txq_lock));
1210
1211 if ((ifp->if_flags & IFF_RUNNING) == 0)
1212 return;
1213
1214 if (!txq->txq_link_active || txq->txq_stopping)
1215 return;
1216
1217 if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
1218 return;
1219
1220 for (;;) {
1221 int slot, r;
1222
1223 if (is_transmit)
1224 m = pcq_get(txq->txq_intrq);
1225 else
1226 IFQ_DEQUEUE(&ifp->if_snd, m);
1227
1228 if (m == NULL)
1229 break;
1230
1231 r = virtio_enqueue_prep(vsc, vq, &slot);
1232 if (r == EAGAIN) {
1233 ifp->if_flags |= IFF_OACTIVE;
1234 m_freem(m);
1235 break;
1236 }
1237 if (r != 0)
1238 panic("enqueue_prep for a tx buffer");
1239
1240 r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1241 txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1242 if (r != 0) {
1243 /* maybe just too fragmented */
1244 struct mbuf *newm;
1245
1246 newm = m_defrag(m, M_NOWAIT);
1247 if (newm == NULL) {
1248 aprint_error_dev(sc->sc_dev,
1249 "m_defrag() failed\n");
1250 goto skip;
1251 }
1252
1253 m = newm;
1254 r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1255 txq->txq_dmamaps[slot], m,
1256 BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1257 if (r != 0) {
1258 aprint_error_dev(sc->sc_dev,
1259 "tx dmamap load failed, error code %d\n",
1260 r);
1261 skip:
1262 m_freem(m);
1263 virtio_enqueue_abort(vsc, vq, slot);
1264 continue;
1265 }
1266 }
1267
1268 /* This should actually never fail */
1269 r = virtio_enqueue_reserve(vsc, vq, slot,
1270 txq->txq_dmamaps[slot]->dm_nsegs + 1);
1271 if (r != 0) {
1272 aprint_error_dev(sc->sc_dev,
1273 "virtio_enqueue_reserve failed, error code %d\n",
1274 r);
1275 bus_dmamap_unload(virtio_dmat(vsc),
1276 txq->txq_dmamaps[slot]);
1277 /* slot already freed by virtio_enqueue_reserve */
1278 m_freem(m);
1279 continue;
1280 }
1281
1282 txq->txq_mbufs[slot] = m;
1283
1284 memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr));
1285 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1286 0, txq->txq_dmamaps[slot]->dm_mapsize,
1287 BUS_DMASYNC_PREWRITE);
1288 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1289 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
1290 BUS_DMASYNC_PREWRITE);
1291 virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
1292 virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
1293 virtio_enqueue_commit(vsc, vq, slot, false);
1294
1295 queued++;
1296 bpf_mtap(ifp, m, BPF_D_OUT);
1297 }
1298
1299 if (queued > 0) {
1300 virtio_enqueue_commit(vsc, vq, -1, true);
1301 ifp->if_timer = 5;
1302 }
1303 }
1304
1305 static void
1306 vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1307 {
1308
1309 /*
1310 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
1311 */
1312 vioif_send_common_locked(ifp, txq, false);
1313
1314 }
1315
1316 static void
1317 vioif_start(struct ifnet *ifp)
1318 {
1319 struct vioif_softc *sc = ifp->if_softc;
1320 struct vioif_txqueue *txq = &sc->sc_txq[0];
1321
1322 #ifdef VIOIF_MPSAFE
1323 KASSERT(if_is_mpsafe(ifp));
1324 #endif
1325
1326 mutex_enter(txq->txq_lock);
1327 if (!txq->txq_stopping)
1328 vioif_start_locked(ifp, txq);
1329 mutex_exit(txq->txq_lock);
1330 }
1331
1332 static inline int
1333 vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
1334 {
1335 struct vioif_softc *sc = ifp->if_softc;
1336 u_int cpuid = cpu_index(curcpu());
1337
1338 return cpuid % sc->sc_act_nvq_pairs;
1339 }
1340
1341 static void
1342 vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1343 {
1344
1345 vioif_send_common_locked(ifp, txq, true);
1346 }
1347
1348 static int
1349 vioif_transmit(struct ifnet *ifp, struct mbuf *m)
1350 {
1351 struct vioif_softc *sc = ifp->if_softc;
1352 struct vioif_txqueue *txq;
1353 int qid;
1354
1355 qid = vioif_select_txqueue(ifp, m);
1356 txq = &sc->sc_txq[qid];
1357
1358 if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
1359 m_freem(m);
1360 return ENOBUFS;
1361 }
1362
1363 net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
1364 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
1365 if (m->m_flags & M_MCAST)
1366 if_statinc_ref(nsr, if_omcasts);
1367 IF_STAT_PUTREF(ifp);
1368
1369 if (mutex_tryenter(txq->txq_lock)) {
1370 if (!txq->txq_stopping)
1371 vioif_transmit_locked(ifp, txq);
1372 mutex_exit(txq->txq_lock);
1373 }
1374
1375 return 0;
1376 }
1377
1378 static void
1379 vioif_deferred_transmit(void *arg)
1380 {
1381 struct vioif_txqueue *txq = arg;
1382 struct virtio_softc *vsc = txq->txq_vq->vq_owner;
1383 struct vioif_softc *sc = device_private(virtio_child(vsc));
1384 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1385
1386 if (mutex_tryenter(txq->txq_lock)) {
1387 vioif_send_common_locked(ifp, txq, true);
1388 mutex_exit(txq->txq_lock);
1389 }
1390 }
1391
1392 static int
1393 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
1394 {
1395 int s, r;
1396
1397 s = splnet();
1398
1399 r = ether_ioctl(ifp, cmd, data);
1400 if ((r == 0 && cmd == SIOCSIFFLAGS) ||
1401 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) {
1402 if (ifp->if_flags & IFF_RUNNING)
1403 r = vioif_rx_filter(ifp->if_softc);
1404 else
1405 r = 0;
1406 }
1407
1408 splx(s);
1409
1410 return r;
1411 }
1412
1413 void
1414 vioif_watchdog(struct ifnet *ifp)
1415 {
1416 struct vioif_softc *sc = ifp->if_softc;
1417 int i;
1418
1419 if (ifp->if_flags & IFF_RUNNING) {
1420 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1421 vioif_tx_queue_clear(&sc->sc_txq[i]);
1422 }
1423 }
1424 }
1425
1426 /*
1427 * Receive implementation
1428 */
1429 /* allocate and initialize a mbuf for receive */
1430 static int
1431 vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1432 {
1433 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1434 struct mbuf *m;
1435 int r;
1436
1437 MGETHDR(m, M_DONTWAIT, MT_DATA);
1438 if (m == NULL)
1439 return ENOBUFS;
1440 MCLGET(m, M_DONTWAIT);
1441 if ((m->m_flags & M_EXT) == 0) {
1442 m_freem(m);
1443 return ENOBUFS;
1444 }
1445 rxq->rxq_mbufs[i] = m;
1446 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1447 r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1448 rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT);
1449 if (r) {
1450 m_freem(m);
1451 rxq->rxq_mbufs[i] = NULL;
1452 return r;
1453 }
1454
1455 return 0;
1456 }
1457
1458 /* free a mbuf for receive */
1459 static void
1460 vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1461 {
1462 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1463
1464 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
1465 m_freem(rxq->rxq_mbufs[i]);
1466 rxq->rxq_mbufs[i] = NULL;
1467 }
1468
1469 /* add mbufs for all the empty receive slots */
1470 static void
1471 vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq)
1472 {
1473
1474 mutex_enter(rxq->rxq_lock);
1475 vioif_populate_rx_mbufs_locked(rxq);
1476 mutex_exit(rxq->rxq_lock);
1477 }
1478
1479 static void
1480 vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq)
1481 {
1482 struct virtqueue *vq = rxq->rxq_vq;
1483 struct virtio_softc *vsc = vq->vq_owner;
1484 struct vioif_softc *sc = device_private(virtio_child(vsc));
1485 int i, r, ndone = 0;
1486
1487 KASSERT(mutex_owned(rxq->rxq_lock));
1488
1489 if (rxq->rxq_stopping)
1490 return;
1491
1492 for (i = 0; i < vq->vq_num; i++) {
1493 int slot;
1494 r = virtio_enqueue_prep(vsc, vq, &slot);
1495 if (r == EAGAIN)
1496 break;
1497 if (r != 0)
1498 panic("enqueue_prep for rx buffers");
1499 if (rxq->rxq_mbufs[slot] == NULL) {
1500 r = vioif_add_rx_mbuf(rxq, slot);
1501 if (r != 0) {
1502 aprint_error_dev(sc->sc_dev,
1503 "rx mbuf allocation failed, "
1504 "error code %d\n", r);
1505 break;
1506 }
1507 }
1508 r = virtio_enqueue_reserve(vsc, vq, slot,
1509 rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
1510 if (r != 0) {
1511 vioif_free_rx_mbuf(rxq, slot);
1512 break;
1513 }
1514 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1515 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD);
1516 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1517 0, MCLBYTES, BUS_DMASYNC_PREREAD);
1518 virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot],
1519 false);
1520 virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
1521 virtio_enqueue_commit(vsc, vq, slot, false);
1522 ndone++;
1523 }
1524 if (ndone > 0)
1525 virtio_enqueue_commit(vsc, vq, -1, true);
1526 }
1527
1528 static void
1529 vioif_rx_queue_clear(struct vioif_rxqueue *rxq)
1530 {
1531 struct virtqueue *vq = rxq->rxq_vq;
1532 struct virtio_softc *vsc = vq->vq_owner;
1533 struct vioif_softc *sc = device_private(virtio_child(vsc));
1534 u_int limit = UINT_MAX;
1535 bool more;
1536
1537 KASSERT(rxq->rxq_stopping);
1538
1539 mutex_enter(rxq->rxq_lock);
1540 for (;;) {
1541 more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1542 if (more == false)
1543 break;
1544 }
1545 mutex_exit(rxq->rxq_lock);
1546 }
1547
1548 /* dequeue received packets */
1549 static bool
1550 vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1551 struct vioif_rxqueue *rxq, u_int limit)
1552 {
1553 struct virtqueue *vq = rxq->rxq_vq;
1554 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1555 struct mbuf *m;
1556 int slot, len;
1557 bool more = false, dequeued = false;
1558
1559 KASSERT(mutex_owned(rxq->rxq_lock));
1560
1561 if (virtio_vq_is_enqueued(vsc, vq) == false)
1562 return false;
1563
1564 for (;;) {
1565 if (limit-- == 0) {
1566 more = true;
1567 break;
1568 }
1569
1570 if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1571 break;
1572
1573 dequeued = true;
1574
1575 len -= sizeof(struct virtio_net_hdr);
1576 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1577 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD);
1578 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1579 0, MCLBYTES, BUS_DMASYNC_POSTREAD);
1580 m = rxq->rxq_mbufs[slot];
1581 KASSERT(m != NULL);
1582 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
1583 rxq->rxq_mbufs[slot] = NULL;
1584 virtio_dequeue_commit(vsc, vq, slot);
1585 m_set_rcvif(m, ifp);
1586 m->m_len = m->m_pkthdr.len = len;
1587
1588 mutex_exit(rxq->rxq_lock);
1589 if_percpuq_enqueue(ifp->if_percpuq, m);
1590 mutex_enter(rxq->rxq_lock);
1591
1592 if (rxq->rxq_stopping)
1593 break;
1594 }
1595
1596 if (dequeued)
1597 softint_schedule(rxq->rxq_softint);
1598
1599 return more;
1600 }
1601
1602 /* rx interrupt; call _dequeue above and schedule a softint */
1603 static int
1604 vioif_rx_intr(void *arg)
1605 {
1606 struct vioif_rxqueue *rxq = arg;
1607 struct virtqueue *vq = rxq->rxq_vq;
1608 struct virtio_softc *vsc = vq->vq_owner;
1609 struct vioif_softc *sc = device_private(virtio_child(vsc));
1610 u_int limit;
1611 bool more;
1612
1613 limit = sc->sc_rx_intr_process_limit;
1614
1615 if (atomic_load_relaxed(&rxq->rxq_active) == true)
1616 return 1;
1617
1618 mutex_enter(rxq->rxq_lock);
1619
1620 if (!rxq->rxq_stopping) {
1621 rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl;
1622
1623 virtio_stop_vq_intr(vsc, vq);
1624 atomic_store_relaxed(&rxq->rxq_active, true);
1625
1626 more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1627 if (more) {
1628 vioif_rx_sched_handle(sc, rxq);
1629 } else {
1630 atomic_store_relaxed(&rxq->rxq_active, false);
1631 virtio_start_vq_intr(vsc, vq);
1632 }
1633 }
1634
1635 mutex_exit(rxq->rxq_lock);
1636 return 1;
1637 }
1638
1639 static void
1640 vioif_rx_handle(void *xrxq)
1641 {
1642 struct vioif_rxqueue *rxq = xrxq;
1643 struct virtqueue *vq = rxq->rxq_vq;
1644 struct virtio_softc *vsc = vq->vq_owner;
1645 struct vioif_softc *sc = device_private(virtio_child(vsc));
1646 u_int limit;
1647 bool more;
1648
1649 limit = sc->sc_rx_process_limit;
1650
1651 mutex_enter(rxq->rxq_lock);
1652
1653 if (!rxq->rxq_stopping) {
1654 more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
1655 if (more) {
1656 vioif_rx_sched_handle(sc, rxq);
1657 } else {
1658 atomic_store_relaxed(&rxq->rxq_active, false);
1659 virtio_start_vq_intr(vsc, rxq->rxq_vq);
1660 }
1661 }
1662
1663 mutex_exit(rxq->rxq_lock);
1664 }
1665
1666 static void
1667 vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
1668 {
1669
1670 if (rxq->rxq_workqueue)
1671 vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work);
1672 else
1673 softint_schedule(rxq->rxq_handle_si);
1674 }
1675
1676 /* softint: enqueue receive requests for new incoming packets */
1677 static void
1678 vioif_rx_softint(void *arg)
1679 {
1680 struct vioif_rxqueue *rxq = arg;
1681
1682 vioif_populate_rx_mbufs(rxq);
1683 }
1684
1685 /* free all the mbufs; called from if_stop(disable) */
1686 static void
1687 vioif_rx_drain(struct vioif_rxqueue *rxq)
1688 {
1689 struct virtqueue *vq = rxq->rxq_vq;
1690 int i;
1691
1692 for (i = 0; i < vq->vq_num; i++) {
1693 if (rxq->rxq_mbufs[i] == NULL)
1694 continue;
1695 vioif_free_rx_mbuf(rxq, i);
1696 }
1697 }
1698
1699 /*
1700 * Transmition implementation
1701 */
1702 /* actual transmission is done in if_start */
1703 /* tx interrupt; dequeue and free mbufs */
1704 /*
1705 * tx interrupt is actually disabled; this should be called upon
1706 * tx vq full and watchdog
1707 */
1708
1709 static int
1710 vioif_tx_intr(void *arg)
1711 {
1712 struct vioif_txqueue *txq = arg;
1713 struct virtqueue *vq = txq->txq_vq;
1714 struct virtio_softc *vsc = vq->vq_owner;
1715 struct vioif_softc *sc = device_private(virtio_child(vsc));
1716 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1717 bool more;
1718 u_int limit;
1719
1720 limit = sc->sc_tx_intr_process_limit;
1721
1722 if (atomic_load_relaxed(&txq->txq_active) == true)
1723 return 1;
1724
1725 mutex_enter(txq->txq_lock);
1726
1727 if (!txq->txq_stopping) {
1728 txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl;
1729
1730 virtio_stop_vq_intr(vsc, vq);
1731 atomic_store_relaxed(&txq->txq_active, true);
1732
1733 more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1734 if (more) {
1735 vioif_tx_sched_handle(sc, txq);
1736 } else {
1737 atomic_store_relaxed(&txq->txq_active, false);
1738
1739 /* for ALTQ */
1740 if (txq == &sc->sc_txq[0]) {
1741 if_schedule_deferred_start(ifp);
1742 ifp->if_flags &= ~IFF_OACTIVE;
1743 }
1744 softint_schedule(txq->txq_deferred_transmit);
1745
1746 virtio_start_vq_intr(vsc, vq);
1747 }
1748 }
1749
1750 mutex_exit(txq->txq_lock);
1751
1752 return 1;
1753 }
1754
1755 static void
1756 vioif_tx_handle(void *xtxq)
1757 {
1758 struct vioif_txqueue *txq = xtxq;
1759 struct virtqueue *vq = txq->txq_vq;
1760 struct virtio_softc *vsc = vq->vq_owner;
1761 struct vioif_softc *sc = device_private(virtio_child(vsc));
1762 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1763 u_int limit;
1764 bool more;
1765
1766 limit = sc->sc_tx_process_limit;
1767
1768 mutex_enter(txq->txq_lock);
1769
1770 if (!txq->txq_stopping) {
1771 more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1772 if (more) {
1773 vioif_tx_sched_handle(sc, txq);
1774 } else {
1775 atomic_store_relaxed(&txq->txq_active, false);
1776
1777 /* for ALTQ */
1778 if (txq == &sc->sc_txq[0]) {
1779 if_schedule_deferred_start(ifp);
1780 ifp->if_flags &= ~IFF_OACTIVE;
1781 }
1782 softint_schedule(txq->txq_deferred_transmit);
1783
1784 virtio_start_vq_intr(vsc, txq->txq_vq);
1785 }
1786 }
1787
1788 mutex_exit(txq->txq_lock);
1789 }
1790
1791 static void
1792 vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq)
1793 {
1794
1795 if (txq->txq_workqueue)
1796 vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work);
1797 else
1798 softint_schedule(txq->txq_handle_si);
1799 }
1800
1801 static void
1802 vioif_tx_queue_clear(struct vioif_txqueue *txq)
1803 {
1804 struct virtqueue *vq = txq->txq_vq;
1805 struct virtio_softc *vsc = vq->vq_owner;
1806 struct vioif_softc *sc = device_private(virtio_child(vsc));
1807 u_int limit = UINT_MAX;
1808 bool more;
1809
1810 mutex_enter(txq->txq_lock);
1811 for (;;) {
1812 more = vioif_tx_deq_locked(sc, vsc, txq, limit);
1813 if (more == false)
1814 break;
1815 }
1816 mutex_exit(txq->txq_lock);
1817 }
1818
1819 static bool
1820 vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1821 struct vioif_txqueue *txq, u_int limit)
1822 {
1823 struct virtqueue *vq = txq->txq_vq;
1824 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1825 struct mbuf *m;
1826 int slot, len;
1827 bool more = false;
1828
1829 KASSERT(mutex_owned(txq->txq_lock));
1830
1831 if (virtio_vq_is_enqueued(vsc, vq) == false)
1832 return false;
1833
1834 for (;;) {
1835 if (limit-- == 0) {
1836 more = true;
1837 break;
1838 }
1839
1840 if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1841 break;
1842
1843 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1844 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE);
1845 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1846 0, txq->txq_dmamaps[slot]->dm_mapsize,
1847 BUS_DMASYNC_POSTWRITE);
1848 m = txq->txq_mbufs[slot];
1849 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
1850 txq->txq_mbufs[slot] = NULL;
1851 virtio_dequeue_commit(vsc, vq, slot);
1852 if_statinc(ifp, if_opackets);
1853 m_freem(m);
1854 }
1855
1856 return more;
1857 }
1858
1859 /* free all the mbufs already put on vq; called from if_stop(disable) */
1860 static void
1861 vioif_tx_drain(struct vioif_txqueue *txq)
1862 {
1863 struct virtqueue *vq = txq->txq_vq;
1864 struct virtio_softc *vsc = vq->vq_owner;
1865 int i;
1866
1867 KASSERT(txq->txq_stopping);
1868
1869 for (i = 0; i < vq->vq_num; i++) {
1870 if (txq->txq_mbufs[i] == NULL)
1871 continue;
1872 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
1873 m_freem(txq->txq_mbufs[i]);
1874 txq->txq_mbufs[i] = NULL;
1875 }
1876 }
1877
1878 /*
1879 * Control vq
1880 */
1881 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
1882 static void
1883 vioif_ctrl_acquire(struct vioif_softc *sc)
1884 {
1885 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1886
1887 mutex_enter(&ctrlq->ctrlq_wait_lock);
1888 while (ctrlq->ctrlq_inuse != FREE)
1889 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1890 ctrlq->ctrlq_inuse = INUSE;
1891 ctrlq->ctrlq_owner = curlwp;
1892 mutex_exit(&ctrlq->ctrlq_wait_lock);
1893 }
1894
1895 static void
1896 vioif_ctrl_release(struct vioif_softc *sc)
1897 {
1898 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1899
1900 KASSERT(ctrlq->ctrlq_inuse != FREE);
1901 KASSERT(ctrlq->ctrlq_owner == curlwp);
1902
1903 mutex_enter(&ctrlq->ctrlq_wait_lock);
1904 ctrlq->ctrlq_inuse = FREE;
1905 ctrlq->ctrlq_owner = NULL;
1906 cv_signal(&ctrlq->ctrlq_wait);
1907 mutex_exit(&ctrlq->ctrlq_wait_lock);
1908 }
1909
1910 static int
1911 vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
1912 struct vioif_ctrl_cmdspec *specs, int nspecs)
1913 {
1914 struct virtio_softc *vsc = sc->sc_virtio;
1915 int i, r, loaded;
1916
1917 loaded = 0;
1918 for (i = 0; i < nspecs; i++) {
1919 r = bus_dmamap_load(virtio_dmat(vsc),
1920 specs[i].dmamap, specs[i].buf, specs[i].bufsize,
1921 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1922 if (r) {
1923 aprint_error_dev(sc->sc_dev, "control command dmamap"
1924 " load failed, error code %d\n", r);
1925 goto err;
1926 }
1927 loaded++;
1928
1929 }
1930
1931 return r;
1932
1933 err:
1934 for (i = 0; i < loaded; i++) {
1935 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
1936 }
1937
1938 return r;
1939 }
1940
1941 static void
1942 vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
1943 struct vioif_ctrl_cmdspec *specs, int nspecs)
1944 {
1945 struct virtio_softc *vsc = sc->sc_virtio;
1946 int i;
1947
1948 for (i = 0; i < nspecs; i++) {
1949 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
1950 }
1951 }
1952
1953 static int
1954 vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
1955 struct vioif_ctrl_cmdspec *specs, int nspecs)
1956 {
1957 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1958 struct virtqueue *vq = ctrlq->ctrlq_vq;
1959 struct virtio_softc *vsc = sc->sc_virtio;
1960 int i, r, slot;
1961
1962 ctrlq->ctrlq_cmd->class = class;
1963 ctrlq->ctrlq_cmd->command = cmd;
1964
1965 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
1966 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
1967 for (i = 0; i < nspecs; i++) {
1968 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
1969 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
1970 }
1971 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
1972 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
1973
1974 r = virtio_enqueue_prep(vsc, vq, &slot);
1975 if (r != 0)
1976 panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1977 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
1978 if (r != 0)
1979 panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1980 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
1981 for (i = 0; i < nspecs; i++) {
1982 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
1983 }
1984 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
1985 virtio_enqueue_commit(vsc, vq, slot, true);
1986
1987 /* wait for done */
1988 mutex_enter(&ctrlq->ctrlq_wait_lock);
1989 while (ctrlq->ctrlq_inuse != DONE)
1990 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1991 mutex_exit(&ctrlq->ctrlq_wait_lock);
1992 /* already dequeueued */
1993
1994 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
1995 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
1996 for (i = 0; i < nspecs; i++) {
1997 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
1998 specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
1999 }
2000 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
2001 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
2002
2003 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
2004 r = 0;
2005 else {
2006 aprint_error_dev(sc->sc_dev, "failed setting rx mode\n");
2007 r = EIO;
2008 }
2009
2010 return r;
2011 }
2012
2013 static int
2014 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
2015 {
2016 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
2017 struct vioif_ctrl_cmdspec specs[1];
2018 int r;
2019
2020 if (!sc->sc_has_ctrl)
2021 return ENOTSUP;
2022
2023 vioif_ctrl_acquire(sc);
2024
2025 rx->onoff = onoff;
2026 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
2027 specs[0].buf = rx;
2028 specs[0].bufsize = sizeof(*rx);
2029
2030 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
2031 specs, __arraycount(specs));
2032
2033 vioif_ctrl_release(sc);
2034 return r;
2035 }
2036
2037 static int
2038 vioif_set_promisc(struct vioif_softc *sc, bool onoff)
2039 {
2040 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
2041 }
2042
2043 static int
2044 vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
2045 {
2046 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
2047 }
2048
2049 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
2050 static int
2051 vioif_set_rx_filter(struct vioif_softc *sc)
2052 {
2053 /* filter already set in ctrlq->ctrlq_mac_tbl */
2054 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
2055 struct vioif_ctrl_cmdspec specs[2];
2056 int nspecs = __arraycount(specs);
2057 int r;
2058
2059 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
2060 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
2061
2062 if (!sc->sc_has_ctrl)
2063 return ENOTSUP;
2064
2065 vioif_ctrl_acquire(sc);
2066
2067 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
2068 specs[0].buf = mac_tbl_uc;
2069 specs[0].bufsize = sizeof(*mac_tbl_uc)
2070 + (ETHER_ADDR_LEN * mac_tbl_uc->nentries);
2071
2072 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
2073 specs[1].buf = mac_tbl_mc;
2074 specs[1].bufsize = sizeof(*mac_tbl_mc)
2075 + (ETHER_ADDR_LEN * mac_tbl_mc->nentries);
2076
2077 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
2078 if (r != 0)
2079 goto out;
2080
2081 r = vioif_ctrl_send_command(sc,
2082 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
2083 specs, nspecs);
2084
2085 vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
2086
2087 out:
2088 vioif_ctrl_release(sc);
2089
2090 return r;
2091 }
2092
2093 static int
2094 vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
2095 {
2096 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
2097 struct vioif_ctrl_cmdspec specs[1];
2098 int r;
2099
2100 if (!sc->sc_has_ctrl)
2101 return ENOTSUP;
2102
2103 if (nvq_pairs <= 1)
2104 return EINVAL;
2105
2106 vioif_ctrl_acquire(sc);
2107
2108 mq->virtqueue_pairs = nvq_pairs;
2109 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
2110 specs[0].buf = mq;
2111 specs[0].bufsize = sizeof(*mq);
2112
2113 r = vioif_ctrl_send_command(sc,
2114 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
2115 specs, __arraycount(specs));
2116
2117 vioif_ctrl_release(sc);
2118
2119 return r;
2120 }
2121
2122 /* ctrl vq interrupt; wake up the command issuer */
2123 static int
2124 vioif_ctrl_intr(void *arg)
2125 {
2126 struct vioif_ctrlqueue *ctrlq = arg;
2127 struct virtqueue *vq = ctrlq->ctrlq_vq;
2128 struct virtio_softc *vsc = vq->vq_owner;
2129 int r, slot;
2130
2131 if (virtio_vq_is_enqueued(vsc, vq) == false)
2132 return 0;
2133
2134 r = virtio_dequeue(vsc, vq, &slot, NULL);
2135 if (r == ENOENT)
2136 return 0;
2137 virtio_dequeue_commit(vsc, vq, slot);
2138
2139 mutex_enter(&ctrlq->ctrlq_wait_lock);
2140 ctrlq->ctrlq_inuse = DONE;
2141 cv_signal(&ctrlq->ctrlq_wait);
2142 mutex_exit(&ctrlq->ctrlq_wait_lock);
2143
2144 return 1;
2145 }
2146
2147 /*
2148 * If IFF_PROMISC requested, set promiscuous
2149 * If multicast filter small enough (<=MAXENTRIES) set rx filter
2150 * If large multicast filter exist use ALLMULTI
2151 */
2152 /*
2153 * If setting rx filter fails fall back to ALLMULTI
2154 * If ALLMULTI fails fall back to PROMISC
2155 */
2156 static int
2157 vioif_rx_filter(struct vioif_softc *sc)
2158 {
2159 struct ethercom *ec = &sc->sc_ethercom;
2160 struct ifnet *ifp = &ec->ec_if;
2161 struct ether_multi *enm;
2162 struct ether_multistep step;
2163 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2164 int nentries;
2165 int promisc = 0, allmulti = 0, rxfilter = 0;
2166 int r;
2167
2168 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */
2169 ifp->if_flags |= IFF_PROMISC;
2170 return 0;
2171 }
2172
2173 if (ifp->if_flags & IFF_PROMISC) {
2174 promisc = 1;
2175 goto set;
2176 }
2177
2178 nentries = -1;
2179 ETHER_LOCK(ec);
2180 ETHER_FIRST_MULTI(step, ec, enm);
2181 while (nentries++, enm != NULL) {
2182 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
2183 allmulti = 1;
2184 goto set_unlock;
2185 }
2186 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2187 allmulti = 1;
2188 goto set_unlock;
2189 }
2190 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
2191 enm->enm_addrlo, ETHER_ADDR_LEN);
2192 ETHER_NEXT_MULTI(step, enm);
2193 }
2194 rxfilter = 1;
2195
2196 set_unlock:
2197 ETHER_UNLOCK(ec);
2198
2199 set:
2200 if (rxfilter) {
2201 ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
2202 ctrlq->ctrlq_mac_tbl_mc->nentries = nentries;
2203 r = vioif_set_rx_filter(sc);
2204 if (r != 0) {
2205 rxfilter = 0;
2206 allmulti = 1; /* fallback */
2207 }
2208 } else {
2209 /* remove rx filter */
2210 ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
2211 ctrlq->ctrlq_mac_tbl_mc->nentries = 0;
2212 r = vioif_set_rx_filter(sc);
2213 /* what to do on failure? */
2214 }
2215 if (allmulti) {
2216 r = vioif_set_allmulti(sc, true);
2217 if (r != 0) {
2218 allmulti = 0;
2219 promisc = 1; /* fallback */
2220 }
2221 } else {
2222 r = vioif_set_allmulti(sc, false);
2223 /* what to do on failure? */
2224 }
2225 if (promisc) {
2226 r = vioif_set_promisc(sc, true);
2227 } else {
2228 r = vioif_set_promisc(sc, false);
2229 }
2230
2231 return r;
2232 }
2233
2234 static bool
2235 vioif_is_link_up(struct vioif_softc *sc)
2236 {
2237 struct virtio_softc *vsc = sc->sc_virtio;
2238 uint16_t status;
2239
2240 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2241 status = virtio_read_device_config_2(vsc,
2242 VIRTIO_NET_CONFIG_STATUS);
2243 else
2244 status = VIRTIO_NET_S_LINK_UP;
2245
2246 return ((status & VIRTIO_NET_S_LINK_UP) != 0);
2247 }
2248
2249 /* change link status */
2250 static void
2251 vioif_update_link_status(struct vioif_softc *sc)
2252 {
2253 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2254 struct vioif_txqueue *txq;
2255 bool active, changed;
2256 int link, i;
2257
2258 active = vioif_is_link_up(sc);
2259 changed = false;
2260
2261 if (active) {
2262 if (!sc->sc_link_active)
2263 changed = true;
2264
2265 link = LINK_STATE_UP;
2266 sc->sc_link_active = true;
2267 } else {
2268 if (sc->sc_link_active)
2269 changed = true;
2270
2271 link = LINK_STATE_DOWN;
2272 sc->sc_link_active = false;
2273 }
2274
2275 if (changed) {
2276 for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2277 txq = &sc->sc_txq[i];
2278
2279 mutex_enter(txq->txq_lock);
2280 txq->txq_link_active = sc->sc_link_active;
2281 mutex_exit(txq->txq_lock);
2282 }
2283
2284 if_link_state_change(ifp, link);
2285 }
2286 }
2287
2288 static int
2289 vioif_config_change(struct virtio_softc *vsc)
2290 {
2291 struct vioif_softc *sc = device_private(virtio_child(vsc));
2292
2293 softint_schedule(sc->sc_ctl_softint);
2294 return 0;
2295 }
2296
2297 static void
2298 vioif_ctl_softint(void *arg)
2299 {
2300 struct vioif_softc *sc = arg;
2301 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2302
2303 vioif_update_link_status(sc);
2304 vioif_start(ifp);
2305 }
2306
2307 static struct workqueue *
2308 vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
2309 {
2310 struct workqueue *wq;
2311 int error;
2312
2313 error = workqueue_create(&wq, name, vioif_workq_work, NULL,
2314 prio, ipl, flags);
2315
2316 if (error)
2317 return NULL;
2318
2319 return wq;
2320 }
2321
2322 static void
2323 vioif_workq_destroy(struct workqueue *wq)
2324 {
2325
2326 workqueue_destroy(wq);
2327 }
2328
2329 static void
2330 vioif_workq_work(struct work *wk, void *context)
2331 {
2332 struct vioif_work *work;
2333
2334 work = container_of(wk, struct vioif_work, cookie);
2335
2336 atomic_store_relaxed(&work->added, 0);
2337 work->func(work->arg);
2338 }
2339
2340 static void
2341 vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
2342 {
2343
2344 memset(work, 0, sizeof(*work));
2345 work->func = func;
2346 work->arg = arg;
2347 }
2348
2349 static void
2350 vioif_work_add(struct workqueue *wq, struct vioif_work *work)
2351 {
2352
2353 if (atomic_load_relaxed(&work->added) != 0)
2354 return;
2355
2356 atomic_store_relaxed(&work->added, 1);
2357 kpreempt_disable();
2358 workqueue_enqueue(wq, &work->cookie, NULL);
2359 kpreempt_enable();
2360 }
2361
2362 static void
2363 vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
2364 {
2365
2366 workqueue_wait(wq, &work->cookie);
2367 }
2368
2369 static int
2370 vioif_setup_sysctl(struct vioif_softc *sc)
2371 {
2372 const char *devname;
2373 struct sysctllog **log;
2374 const struct sysctlnode *rnode, *rxnode, *txnode;
2375 int error;
2376
2377 log = &sc->sc_sysctllog;
2378 devname = device_xname(sc->sc_dev);
2379
2380 error = sysctl_createv(log, 0, NULL, &rnode,
2381 0, CTLTYPE_NODE, devname,
2382 SYSCTL_DESCR("virtio-net information and settings"),
2383 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
2384 if (error)
2385 goto out;
2386
2387 error = sysctl_createv(log, 0, &rnode, NULL,
2388 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
2389 SYSCTL_DESCR("Use workqueue for packet processing"),
2390 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
2391 if (error)
2392 goto out;
2393
2394 error = sysctl_createv(log, 0, &rnode, &rxnode,
2395 0, CTLTYPE_NODE, "rx",
2396 SYSCTL_DESCR("virtio-net information and settings for Rx"),
2397 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2398 if (error)
2399 goto out;
2400
2401 error = sysctl_createv(log, 0, &rxnode, NULL,
2402 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2403 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
2404 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2405 if (error)
2406 goto out;
2407
2408 error = sysctl_createv(log, 0, &rxnode, NULL,
2409 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2410 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
2411 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
2412 if (error)
2413 goto out;
2414
2415 error = sysctl_createv(log, 0, &rnode, &txnode,
2416 0, CTLTYPE_NODE, "tx",
2417 SYSCTL_DESCR("virtio-net information and settings for Tx"),
2418 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2419 if (error)
2420 goto out;
2421
2422 error = sysctl_createv(log, 0, &txnode, NULL,
2423 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2424 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
2425 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2426 if (error)
2427 goto out;
2428
2429 error = sysctl_createv(log, 0, &txnode, NULL,
2430 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2431 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
2432 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
2433
2434 out:
2435 if (error)
2436 sysctl_teardown(log);
2437
2438 return error;
2439 }
2440
2441 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
2442
2443 #ifdef _MODULE
2444 #include "ioconf.c"
2445 #endif
2446
2447 static int
2448 if_vioif_modcmd(modcmd_t cmd, void *opaque)
2449 {
2450 int error = 0;
2451
2452 #ifdef _MODULE
2453 switch (cmd) {
2454 case MODULE_CMD_INIT:
2455 error = config_init_component(cfdriver_ioconf_if_vioif,
2456 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2457 break;
2458 case MODULE_CMD_FINI:
2459 error = config_fini_component(cfdriver_ioconf_if_vioif,
2460 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2461 break;
2462 default:
2463 error = ENOTTY;
2464 break;
2465 }
2466 #endif
2467
2468 return error;
2469 }
2470